{"step": 1, "total_steps": 72479, "tokens": 524288, "loss": 14.47672438621521, "l_mix": 9.035724401473999, "l_a": 9.052121639251709, "l_c": 9.084503173828125, "z_loss": 0.12790902890264988, "lr": 1.3142857142857143e-06, "wd": 0.1, "grad_norm": 0.7286704778671265, "tokens_per_second": 6078.831172690608, "eta_seconds": 6251094.097614288, "data_wait_seconds": 8.892472952604294, "mix": {"fineweb_hq": 1.0, "dclm": 0.0, "cosmopedia_v2": 0.0, "finemath": 0.0}, "gates": {"gate_a2b/L1_absmean": 0.00999999, "gate_a2b/L1_absmax": 0.01000131, "gate_a2b/L1_rms": 0.00999999, "gate_a2b/L3_absmean": 0.01000009, "gate_a2b/L3_absmax": 0.01000131, "gate_a2b/L3_rms": 0.01000009, "gate_a2b/L5_absmean": 0.01000004, "gate_a2b/L5_absmax": 0.0100013, "gate_a2b/L5_rms": 0.01000004, "gate_c2b/L1_absmean": 0.00999988, "gate_c2b/L1_absmax": 0.01000131, "gate_c2b/L1_rms": 0.00999988, "gate_c2b/L3_absmean": 0.01000002, "gate_c2b/L3_absmax": 0.01000131, "gate_c2b/L3_rms": 0.01000002, "gate_c2b/L5_absmean": 0.00999987, "gate_c2b/L5_absmax": 0.01000131, "gate_c2b/L5_rms": 0.00999987, "gate_b2a/L7_absmean": 0.01000011, "gate_b2a/L7_absmax": 0.01000131, "gate_b2a/L7_rms": 0.01000011, "gate_b2a/L11_absmean": 0.01000013, "gate_b2a/L11_absmax": 0.01000131, "gate_b2a/L11_rms": 0.01000013, "gate_b2a/L14_absmean": 0.00999994, "gate_b2a/L14_absmax": 0.01000131, "gate_b2a/L14_rms": 0.00999994, "gate_b2c/L3_absmean": 0.00999996, "gate_b2c/L3_absmax": 0.01000131, "gate_b2c/L3_rms": 0.00999996, "gate_b2c/L5_absmean": 0.00999989, "gate_b2c/L5_absmax": 0.01000131, "gate_b2c/L5_rms": 0.00999989, "gate_b2c/L6_absmean": 0.00999999, "gate_b2c/L6_absmax": 0.01000131, "gate_b2c/L6_rms": 0.00999999}, "streams": {"cos_a2b/round0": 0.015846, "cos_c2b/round0": 0.004168, "cos_b2a/round0": -0.000901, "cos_b2c/round0": 0.00438, "rms_a/round0": 0.158341, "rms_b/round0": 0.164404, "rms_c/round0": 0.398572, "cos_a2b/round1": -0.000234, "cos_c2b/round1": 0.008409, "cos_b2a/round1": 0.005919, "cos_b2c/round1": -0.000681, "rms_a/round1": 0.162101, "rms_b/round1": 0.175618, "rms_c/round1": 0.403409, "cos_a2b/round2": -4.9e-05, "cos_c2b/round2": 0.014912, "cos_b2a/round2": -0.002536, "cos_b2c/round2": -0.006093, "rms_a/round2": 0.165381, "rms_b/round2": 0.181449, "rms_c/round2": 0.406072}, "alpha_mean": 0.5, "alpha_hist": [0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0], "alpha_by_type": {"special": {"share": 0.00078, "mean": 0.5, "std": 0.0}, "whitespace": {"share": 0.02481, "mean": 0.5, "std": 0.0}, "digit": {"share": 0.04126, "mean": 0.5, "std": 0.0}, "word_start": {"share": 0.56691, "mean": 0.5, "std": 0.0}, "word_cont": {"share": 0.25729, "mean": 0.5, "std": 0.0}, "punct": {"share": 0.10895, "mean": 0.5, "std": 0.0}}, "alpha_p05": 0.5, "alpha_p50": 0.5, "alpha_p95": 0.5}
{"step": 10, "total_steps": 72479, "tokens": 5242880, "loss": 14.420469641685486, "l_mix": 9.000654578208923, "l_a": 9.004850625991821, "l_c": 9.061158776283264, "z_loss": 0.1277289930731058, "lr": 1.3142857142857143e-05, "wd": 0.1, "grad_norm": 0.7595540285110474, "tokens_per_second": 149466.80578667182, "eta_seconds": 254201.1041985353, "data_wait_seconds": 3.865361213684082e-05, "mix": {"fineweb_hq": 0.5, "dclm": 0.2, "cosmopedia_v2": 0.2, "finemath": 0.1}}
{"step": 20, "total_steps": 72479, "tokens": 10485760, "loss": 14.175702333450317, "l_mix": 8.845036149024963, "l_a": 8.815389275550842, "l_c": 8.953457117080688, "z_loss": 0.1274268664419651, "lr": 2.6285714285714286e-05, "wd": 0.1, "grad_norm": 1.6760839223861694, "tokens_per_second": 1439901.073852328, "eta_seconds": 26383.329300785063, "data_wait_seconds": 2.0295381546020508e-05, "mix": {"fineweb_hq": 0.5, "dclm": 0.2, "cosmopedia_v2": 0.2, "finemath": 0.1}}
{"step": 30, "total_steps": 72479, "tokens": 15728640, "loss": 13.766064405441284, "l_mix": 8.582801580429077, "l_a": 8.577266335487366, "l_c": 8.700231313705444, "z_loss": 0.12927597016096115, "lr": 3.942857142857143e-05, "wd": 0.1, "grad_norm": 2.071235418319702, "tokens_per_second": 1427832.1312802117, "eta_seconds": 26602.666013646125, "data_wait_seconds": 1.71661376953125e-05, "mix": {"fineweb_hq": 0.5, "dclm": 0.2, "cosmopedia_v2": 0.2, "finemath": 0.1}}
{"step": 40, "total_steps": 72479, "tokens": 20971520, "loss": 13.566155195236206, "l_mix": 8.460710525512695, "l_a": 8.49457061290741, "l_c": 8.523529767990112, "z_loss": 0.14291252195835114, "lr": 5.257142857142857e-05, "wd": 0.1, "grad_norm": 1.325427770614624, "tokens_per_second": 1487663.7888007907, "eta_seconds": 25529.221533727643, "data_wait_seconds": 1.5437602996826172e-05, "mix": {"fineweb_hq": 0.525, "dclm": 0.2, "cosmopedia_v2": 0.2, "finemath": 0.075}}
{"step": 50, "total_steps": 72479, "tokens": 26214400, "loss": 13.193636536598206, "l_mix": 8.228643655776978, "l_a": 8.348215103149414, "l_c": 8.20169460773468, "z_loss": 0.19901667721569538, "lr": 6.571428571428571e-05, "wd": 0.1, "grad_norm": 1.6887521743774414, "tokens_per_second": 1484538.3860517787, "eta_seconds": 25579.436617326734, "data_wait_seconds": 2.136826515197754e-05, "mix": {"fineweb_hq": 0.52, "dclm": 0.2, "cosmopedia_v2": 0.2, "finemath": 0.08}}
{"step": 60, "total_steps": 72479, "tokens": 31457280, "loss": 12.845388174057007, "l_mix": 7.998206853866577, "l_a": 8.19615650177002, "l_c": 7.961012542247772, "z_loss": 0.3053499832749367, "lr": 7.885714285714286e-05, "wd": 0.1, "grad_norm": 1.7961777448654175, "tokens_per_second": 1340977.4064173535, "eta_seconds": 28313.983882427216, "data_wait_seconds": 2.9087066650390625e-05, "mix": {"fineweb_hq": 0.51667, "dclm": 0.2, "cosmopedia_v2": 0.2, "finemath": 0.08333}}
{"step": 70, "total_steps": 72479, "tokens": 36700160, "loss": 12.451049447059631, "l_mix": 7.7357577085494995, "l_a": 8.033122062683105, "l_c": 7.68434464931488, "z_loss": 0.5127934142947197, "lr": 9.2e-05, "wd": 0.1, "grad_norm": 1.8067258596420288, "tokens_per_second": 1487282.452952593, "eta_seconds": 25525.191745948792, "data_wait_seconds": 1.6033649444580078e-05, "mix": {"fineweb_hq": 0.51429, "dclm": 0.2, "cosmopedia_v2": 0.2, "finemath": 0.08571}}
{"step": 80, "total_steps": 72479, "tokens": 41943040, "loss": 12.05958890914917, "l_mix": 7.471342921257019, "l_a": 7.86543607711792, "l_c": 7.428434371948242, "z_loss": 0.845537967979908, "lr": 0.00010514285714285714, "wd": 0.1, "grad_norm": 1.5622421503067017, "tokens_per_second": 1484305.612223691, "eta_seconds": 25572.851439356804, "data_wait_seconds": 2.142786979675293e-05, "mix": {"fineweb_hq": 0.5125, "dclm": 0.2, "cosmopedia_v2": 0.2, "finemath": 0.0875}}
{"step": 90, "total_steps": 72479, "tokens": 47185920, "loss": 11.705622673034668, "l_mix": 7.235036790370941, "l_a": 7.69447785615921, "l_c": 7.20705509185791, "z_loss": 1.2569920420646667, "lr": 0.00011828571428571429, "wd": 0.1, "grad_norm": 1.1561428308486938, "tokens_per_second": 1457214.800634382, "eta_seconds": 26044.673726534842, "data_wait_seconds": 1.6480684280395508e-05, "mix": {"fineweb_hq": 0.51111, "dclm": 0.21111, "cosmopedia_v2": 0.2, "finemath": 0.07778}}
{"step": 100, "total_steps": 72479, "tokens": 52428800, "loss": 11.387068271636963, "l_mix": 7.028275370597839, "l_a": 7.5174137353897095, "l_c": 7.011336863040924, "z_loss": 1.673080936074257, "lr": 0.00013142857142857143, "wd": 0.1, "grad_norm": 0.850379228591919, "tokens_per_second": 1484803.5144650675, "eta_seconds": 25557.213989806176, "data_wait_seconds": 1.5348196029663086e-05, "mix": {"fineweb_hq": 0.51, "dclm": 0.21, "cosmopedia_v2": 0.2, "finemath": 0.08}, "gates": {"gate_a2b/L1_absmean": 0.01013556, "gate_a2b/L1_absmax": 0.01433333, "gate_a2b/L1_rms": 0.01026403, "gate_a2b/L3_absmean": 0.01039405, "gate_a2b/L3_absmax": 0.01443803, "gate_a2b/L3_rms": 0.01053591, "gate_a2b/L5_absmean": 0.0102346, "gate_a2b/L5_absmax": 0.01424643, "gate_a2b/L5_rms": 0.01039003, "gate_c2b/L1_absmean": 0.01022123, "gate_c2b/L1_absmax": 0.01413538, "gate_c2b/L1_rms": 0.01034611, "gate_c2b/L3_absmean": 0.01051268, "gate_c2b/L3_absmax": 0.01436089, "gate_c2b/L3_rms": 0.01065706, "gate_c2b/L5_absmean": 0.01037257, "gate_c2b/L5_absmax": 0.01445691, "gate_c2b/L5_rms": 0.01053338, "gate_b2a/L7_absmean": 0.01109544, "gate_b2a/L7_absmax": 0.01469284, "gate_b2a/L7_rms": 0.01118721, "gate_b2a/L11_absmean": 0.01083107, "gate_b2a/L11_absmax": 0.01434811, "gate_b2a/L11_rms": 0.01090923, "gate_b2a/L14_absmean": 0.01099423, "gate_b2a/L14_absmax": 0.01351194, "gate_b2a/L14_rms": 0.01106725, "gate_b2c/L3_absmean": 0.01101439, "gate_b2c/L3_absmax": 0.015636, "gate_b2c/L3_rms": 0.01114003, "gate_b2c/L5_absmean": 0.0108079, "gate_b2c/L5_absmax": 0.014256, "gate_b2c/L5_rms": 0.01092292, "gate_b2c/L6_absmean": 0.01119966, "gate_b2c/L6_absmax": 0.01463067, "gate_b2c/L6_rms": 0.01129277}, "streams": {"cos_a2b/round0": 0.106919, "cos_c2b/round0": 0.074518, "cos_b2a/round0": 0.133089, "cos_b2c/round0": 0.128717, "rms_a/round0": 0.189662, "rms_b/round0": 0.290253, "rms_c/round0": 0.450034, "cos_a2b/round1": 0.279801, "cos_c2b/round1": 0.136151, "cos_b2a/round1": 0.423385, "cos_b2c/round1": 0.356429, "rms_a/round1": 0.313055, "rms_b/round1": 1.324186, "rms_c/round1": 0.634113, "cos_a2b/round2": 0.38531, "cos_c2b/round2": 0.243886, "cos_b2a/round2": 0.59263, "cos_b2c/round2": 0.594362, "rms_a/round2": 0.540956, "rms_b/round2": 1.986939, "rms_c/round2": 0.911743}, "alpha_mean": 0.09509287430057611, "alpha_hist": [0.7341, 0.26576, 6e-05, 2e-05, 3e-05, 2e-05, 0.0, 0.0, 2e-05, 0.0], "alpha_by_type": {"special": {"share": 0.00056, "mean": 0.13322, "std": 0.03105}, "whitespace": {"share": 0.02145, "mean": 0.11607, "std": 0.03047}, "digit": {"share": 0.03352, "mean": 0.1207, "std": 0.02781}, "word_start": {"share": 0.53787, "mean": 0.10033, "std": 0.0276}, "word_cont": {"share": 0.29454, "mean": 0.08377, "std": 0.01296}, "punct": {"share": 0.11205, "mean": 0.08786, "std": 0.01724}}, "alpha_p05": 0.07696, "alpha_p50": 0.0827, "alpha_p95": 0.15431}
{"step": 110, "total_steps": 72479, "tokens": 57671680, "loss": 11.081153869628906, "l_mix": 6.83515989780426, "l_a": 7.329842686653137, "l_c": 6.822786569595337, "z_loss": 2.05376398563385, "lr": 0.00014457142857142859, "wd": 0.1, "grad_norm": 0.6866495609283447, "tokens_per_second": 1481329.487947737, "eta_seconds": 25613.611678361893, "data_wait_seconds": 1.5527009963989258e-05, "mix": {"fineweb_hq": 0.50909, "dclm": 0.20909, "cosmopedia_v2": 0.2, "finemath": 0.08182}}
{"step": 120, "total_steps": 72479, "tokens": 62914560, "loss": 10.787169098854065, "l_mix": 6.6487011313438416, "l_a": 7.155963242053986, "l_c": 6.638160765171051, "z_loss": 2.304818719625473, "lr": 0.00015771428571428571, "wd": 0.1, "grad_norm": 0.6770825982093811, "tokens_per_second": 1373276.4941215145, "eta_seconds": 27625.13998775482, "data_wait_seconds": 3.364682197570801e-05, "mix": {"fineweb_hq": 0.50833, "dclm": 0.20833, "cosmopedia_v2": 0.2, "finemath": 0.08333}}
{"step": 130, "total_steps": 72479, "tokens": 68157440, "loss": 10.552145957946777, "l_mix": 6.501602292060852, "l_a": 7.007596850395203, "l_c": 6.493339240550995, "z_loss": 2.6260264813899994, "lr": 0.00017085714285714284, "wd": 0.1, "grad_norm": 0.5527415871620178, "tokens_per_second": 1457122.4908566365, "eta_seconds": 26031.931254935265, "data_wait_seconds": 2.512335777282715e-05, "mix": {"fineweb_hq": 0.50769, "dclm": 0.20769, "cosmopedia_v2": 0.2, "finemath": 0.08462}}
{"step": 140, "total_steps": 72479, "tokens": 73400320, "loss": 9.794955372810364, "l_mix": 6.0228551030159, "l_a": 6.555157542228699, "l_c": 6.01742285490036, "z_loss": 3.2594143748283386, "lr": 0.000184, "wd": 0.1, "grad_norm": 1.034790277481079, "tokens_per_second": 1341620.3732520454, "eta_seconds": 28269.15153357983, "data_wait_seconds": 2.1219253540039062e-05, "mix": {"fineweb_hq": 0.50714, "dclm": 0.20714, "cosmopedia_v2": 0.2, "finemath": 0.08571}}
{"step": 150, "total_steps": 72479, "tokens": 78643200, "loss": 10.087844371795654, "l_mix": 6.217134714126587, "l_a": 6.690563678741455, "l_c": 6.210737407207489, "z_loss": 3.192536562681198, "lr": 0.00019714285714285713, "wd": 0.1, "grad_norm": 0.5715301036834717, "tokens_per_second": 1364840.4286608743, "eta_seconds": 27784.36654987335, "data_wait_seconds": 1.558661460876465e-05, "mix": {"fineweb_hq": 0.51333, "dclm": 0.20667, "cosmopedia_v2": 0.2, "finemath": 0.08}}
{"step": 160, "total_steps": 72479, "tokens": 83886080, "loss": 9.941380262374878, "l_mix": 6.130853772163391, "l_a": 6.575091600418091, "l_c": 6.125538885593414, "z_loss": 3.3714952766895294, "lr": 0.00021028571428571429, "wd": 0.1, "grad_norm": 0.7287636399269104, "tokens_per_second": 1489882.2250407054, "eta_seconds": 25448.980620574952, "data_wait_seconds": 1.6391277313232422e-05, "mix": {"fineweb_hq": 0.5125, "dclm": 0.20625, "cosmopedia_v2": 0.2, "finemath": 0.08125}}
{"step": 170, "total_steps": 72479, "tokens": 89128960, "loss": 9.817246198654175, "l_mix": 6.0559515953063965, "l_a": 6.485471248626709, "l_c": 6.050968825817108, "z_loss": 3.6260419487953186, "lr": 0.00022342857142857144, "wd": 0.1, "grad_norm": 0.4940676987171173, "tokens_per_second": 1474559.9890136719, "eta_seconds": 25709.866858220103, "data_wait_seconds": 3.4183263778686523e-05, "mix": {"fineweb_hq": 0.51176, "dclm": 0.20588, "cosmopedia_v2": 0.2, "finemath": 0.08235}}
{"step": 180, "total_steps": 72479, "tokens": 94371840, "loss": 9.65708339214325, "l_mix": 5.956133842468262, "l_a": 6.382104158401489, "l_c": 5.953053116798401, "z_loss": 4.021556526422501, "lr": 0.00023657142857142857, "wd": 0.1, "grad_norm": 0.5197019577026367, "tokens_per_second": 1419888.8066112527, "eta_seconds": 26696.10319871902, "data_wait_seconds": 2.053380012512207e-05, "mix": {"fineweb_hq": 0.51111, "dclm": 0.20556, "cosmopedia_v2": 0.2, "finemath": 0.08333}}
{"step": 190, "total_steps": 72479, "tokens": 99614720, "loss": 9.515849471092224, "l_mix": 5.873618543148041, "l_a": 6.26884526014328, "l_c": 5.870530784130096, "z_loss": 4.181336939334869, "lr": 0.00024971428571428573, "wd": 0.1, "grad_norm": 0.6289177536964417, "tokens_per_second": 1477482.83900801, "eta_seconds": 25651.90893008709, "data_wait_seconds": 2.1457672119140625e-05, "mix": {"fineweb_hq": 0.51053, "dclm": 0.20526, "cosmopedia_v2": 0.2, "finemath": 0.08421}}
{"step": 200, "total_steps": 72479, "tokens": 104857600, "loss": 9.170726418495178, "l_mix": 5.662435472011566, "l_a": 6.032914221286774, "l_c": 5.659927546977997, "z_loss": 4.384070813655853, "lr": 0.00026285714285714286, "wd": 0.1, "grad_norm": 0.7797325849533081, "tokens_per_second": 1438729.1422804983, "eta_seconds": 26339.226222896577, "data_wait_seconds": 1.7076730728149414e-05, "mix": {"fineweb_hq": 0.51, "dclm": 0.21, "cosmopedia_v2": 0.2, "finemath": 0.08}, "gates": {"gate_a2b/L1_absmean": 0.00979484, "gate_a2b/L1_absmax": 0.01483283, "gate_a2b/L1_rms": 0.01001528, "gate_a2b/L3_absmean": 0.01008041, "gate_a2b/L3_absmax": 0.0171812, "gate_a2b/L3_rms": 0.01033485, "gate_a2b/L5_absmean": 0.01028938, "gate_a2b/L5_absmax": 0.01815658, "gate_a2b/L5_rms": 0.01062237, "gate_c2b/L1_absmean": 0.01049698, "gate_c2b/L1_absmax": 0.01679583, "gate_c2b/L1_rms": 0.01077283, "gate_c2b/L3_absmean": 0.01058276, "gate_c2b/L3_absmax": 0.01956983, "gate_c2b/L3_rms": 0.01096777, "gate_c2b/L5_absmean": 0.01114407, "gate_c2b/L5_absmax": 0.02164241, "gate_c2b/L5_rms": 0.01154733, "gate_b2a/L7_absmean": 0.01394307, "gate_b2a/L7_absmax": 0.02373539, "gate_b2a/L7_rms": 0.01423168, "gate_b2a/L11_absmean": 0.01236894, "gate_b2a/L11_absmax": 0.02055072, "gate_b2a/L11_rms": 0.01257768, "gate_b2a/L14_absmean": 0.01225588, "gate_b2a/L14_absmax": 0.01780462, "gate_b2a/L14_rms": 0.01240383, "gate_b2c/L3_absmean": 0.01252163, "gate_b2c/L3_absmax": 0.02123424, "gate_b2c/L3_rms": 0.01277306, "gate_b2c/L5_absmean": 0.01093341, "gate_b2c/L5_absmax": 0.02095394, "gate_b2c/L5_rms": 0.01110754, "gate_b2c/L6_absmean": 0.01116803, "gate_b2c/L6_absmax": 0.02179631, "gate_b2c/L6_rms": 0.01131304}, "streams": {"cos_a2b/round0": 0.104077, "cos_c2b/round0": 0.100406, "cos_b2a/round0": 0.247712, "cos_b2c/round0": 0.160157, "rms_a/round0": 0.198766, "rms_b/round0": 0.323337, "rms_c/round0": 0.452429, "cos_a2b/round1": 0.1562, "cos_c2b/round1": 0.231499, "cos_b2a/round1": 0.385927, "cos_b2c/round1": 0.26453, "rms_a/round1": 0.324184, "rms_b/round1": 0.668408, "rms_c/round1": 0.671388, "cos_a2b/round2": 0.182005, "cos_c2b/round2": 0.310679, "cos_b2a/round2": 0.445053, "cos_b2c/round2": 0.340077, "rms_a/round2": 0.456267, "rms_b/round2": 0.98426, "rms_c/round2": 0.840075}, "alpha_mean": 0.01153868652348411, "alpha_hist": [0.99931, 0.00067, 2e-05, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], "alpha_by_type": {"special": {"share": 0.00052, "mean": 0.00751, "std": 0.00495}, "whitespace": {"share": 0.08296, "mean": 0.007, "std": 0.00668}, "digit": {"share": 0.03935, "mean": 0.00373, "std": 0.00367}, "word_start": {"share": 0.5468, "mean": 0.01332, "std": 0.01665}, "word_cont": {"share": 0.23117, "mean": 0.01341, "std": 0.01505}, "punct": {"share": 0.0992, "mean": 0.0043, "std": 0.00647}}, "alpha_p05": 0.00113, "alpha_p50": 0.00639, "alpha_p95": 0.04814}
{"step": 210, "total_steps": 72479, "tokens": 110100480, "loss": 9.125158309936523, "l_mix": 5.6367645263671875, "l_a": 5.991681337356567, "l_c": 5.634811699390411, "z_loss": 4.456553399562836, "lr": 0.000276, "wd": 0.1, "grad_norm": 0.664954662322998, "tokens_per_second": 1392851.344018926, "eta_seconds": 27203.02466928959, "data_wait_seconds": 2.0265579223632812e-05, "mix": {"fineweb_hq": 0.50952, "dclm": 0.20952, "cosmopedia_v2": 0.2, "finemath": 0.08095}}
{"step": 220, "total_steps": 72479, "tokens": 115343360, "loss": 8.971947073936462, "l_mix": 5.544267892837524, "l_a": 5.880756914615631, "l_c": 5.543157875537872, "z_loss": 5.047135412693024, "lr": 0.00028914285714285717, "wd": 0.1, "grad_norm": 0.6585619449615479, "tokens_per_second": 1484719.1041438754, "eta_seconds": 25516.292264485357, "data_wait_seconds": 1.6182661056518555e-05, "mix": {"fineweb_hq": 0.50909, "dclm": 0.20909, "cosmopedia_v2": 0.2, "finemath": 0.08182}}
{"step": 230, "total_steps": 72479, "tokens": 120586240, "loss": 8.970234990119934, "l_mix": 5.546203911304474, "l_a": 5.867521286010742, "l_c": 5.544302225112915, "z_loss": 4.8400439620018005, "lr": 0.0003022857142857143, "wd": 0.1, "grad_norm": 0.6546902656555176, "tokens_per_second": 1472142.1668284289, "eta_seconds": 25730.72395148277, "data_wait_seconds": 2.905726432800293e-05, "mix": {"fineweb_hq": 0.5087, "dclm": 0.2087, "cosmopedia_v2": 0.2, "finemath": 0.08261}}
{"step": 240, "total_steps": 72479, "tokens": 125829120, "loss": 8.848293781280518, "l_mix": 5.473823547363281, "l_a": 5.773420691490173, "l_c": 5.47300660610199, "z_loss": 5.418484151363373, "lr": 0.00031542857142857143, "wd": 0.1, "grad_norm": 0.639414370059967, "tokens_per_second": 1449892.6013152015, "eta_seconds": 26121.96296308041, "data_wait_seconds": 2.0384788513183594e-05, "mix": {"fineweb_hq": 0.50833, "dclm": 0.20833, "cosmopedia_v2": 0.2, "finemath": 0.08333}}
{"step": 250, "total_steps": 72479, "tokens": 131072000, "loss": 7.9802767634391785, "l_mix": 4.923585057258606, "l_a": 5.263879835605621, "l_c": 4.9230154156684875, "z_loss": 6.229677438735962, "lr": 0.00032857142857142856, "wd": 0.1, "grad_norm": 1.22578763961792, "tokens_per_second": 1389449.9564795261, "eta_seconds": 27254.524551534654, "data_wait_seconds": 2.8908252716064453e-05, "mix": {"fineweb_hq": 0.508, "dclm": 0.208, "cosmopedia_v2": 0.2, "finemath": 0.084}}
{"step": 260, "total_steps": 72479, "tokens": 136314880, "loss": 8.807690858840942, "l_mix": 5.449946820735931, "l_a": 5.741418480873108, "l_c": 5.449292898178101, "z_loss": 5.303723871707916, "lr": 0.0003417142857142857, "wd": 0.1, "grad_norm": 0.5515539646148682, "tokens_per_second": 1369444.3854538521, "eta_seconds": 27648.84465129375, "data_wait_seconds": 3.299117088317871e-05, "mix": {"fineweb_hq": 0.51154, "dclm": 0.20769, "cosmopedia_v2": 0.2, "finemath": 0.08077}}
{"step": 270, "total_steps": 72479, "tokens": 141557760, "loss": 8.706616759300232, "l_mix": 5.390162527561188, "l_a": 5.663214921951294, "l_c": 5.389654457569122, "z_loss": 5.929699420928955, "lr": 0.00035485714285714287, "wd": 0.1, "grad_norm": 0.5379797220230103, "tokens_per_second": 1421829.7737652874, "eta_seconds": 26626.47307753563, "data_wait_seconds": 2.0116567611694336e-05, "mix": {"fineweb_hq": 0.51111, "dclm": 0.20741, "cosmopedia_v2": 0.2, "finemath": 0.08148}}
{"step": 280, "total_steps": 72479, "tokens": 146800640, "loss": 8.741459965705872, "l_mix": 5.412259578704834, "l_a": 5.683949291706085, "l_c": 5.411688208580017, "z_loss": 5.089005351066589, "lr": 0.000368, "wd": 0.1, "grad_norm": 0.6953883767127991, "tokens_per_second": 1475776.4861509337, "eta_seconds": 25649.595089244845, "data_wait_seconds": 2.1070241928100586e-05, "mix": {"fineweb_hq": 0.51071, "dclm": 0.20714, "cosmopedia_v2": 0.2, "finemath": 0.08214}}
{"step": 290, "total_steps": 72479, "tokens": 152043520, "loss": 8.644548416137695, "l_mix": 5.353215873241425, "l_a": 5.616191565990448, "l_c": 5.352876782417297, "z_loss": 6.118154168128967, "lr": 0.0003811428571428572, "wd": 0.1, "grad_norm": 0.6905578970909119, "tokens_per_second": 1433417.0205044257, "eta_seconds": 26403.918671679497, "data_wait_seconds": 1.8388032913208008e-05, "mix": {"fineweb_hq": 0.51034, "dclm": 0.2069, "cosmopedia_v2": 0.2, "finemath": 0.08276}}
{"step": 300, "total_steps": 72479, "tokens": 157286400, "loss": 8.540663242340088, "l_mix": 5.290314495563507, "l_a": 5.542400538921356, "l_c": 5.290163159370422, "z_loss": 5.7972341775894165, "lr": 0.00039428571428571426, "wd": 0.1, "grad_norm": 0.6885882019996643, "tokens_per_second": 1450280.731250324, "eta_seconds": 26093.281622362134, "data_wait_seconds": 1.5497207641601562e-05, "mix": {"fineweb_hq": 0.51, "dclm": 0.20667, "cosmopedia_v2": 0.2, "finemath": 0.08333}, "gates": {"gate_a2b/L1_absmean": 0.00925463, "gate_a2b/L1_absmax": 0.01857348, "gate_a2b/L1_rms": 0.00973708, "gate_a2b/L3_absmean": 0.01074761, "gate_a2b/L3_absmax": 0.02197265, "gate_a2b/L3_rms": 0.01143381, "gate_a2b/L5_absmean": 0.0111122, "gate_a2b/L5_absmax": 0.02276217, "gate_a2b/L5_rms": 0.0117991, "gate_c2b/L1_absmean": 0.00989807, "gate_c2b/L1_absmax": 0.01786134, "gate_c2b/L1_rms": 0.01046235, "gate_c2b/L3_absmean": 0.01144208, "gate_c2b/L3_absmax": 0.02187608, "gate_c2b/L3_rms": 0.01220491, "gate_c2b/L5_absmean": 0.01230574, "gate_c2b/L5_absmax": 0.02758425, "gate_c2b/L5_rms": 0.0130415, "gate_b2a/L7_absmean": 0.01580198, "gate_b2a/L7_absmax": 0.02900244, "gate_b2a/L7_rms": 0.01628156, "gate_b2a/L11_absmean": 0.01266482, "gate_b2a/L11_absmax": 0.02206676, "gate_b2a/L11_rms": 0.01304285, "gate_b2a/L14_absmean": 0.01272352, "gate_b2a/L14_absmax": 0.01992847, "gate_b2a/L14_rms": 0.01303097, "gate_b2c/L3_absmean": 0.01364935, "gate_b2c/L3_absmax": 0.02578929, "gate_b2c/L3_rms": 0.01422756, "gate_b2c/L5_absmean": 0.01186352, "gate_b2c/L5_absmax": 0.02348641, "gate_b2c/L5_rms": 0.01238259, "gate_b2c/L6_absmean": 0.01222641, "gate_b2c/L6_absmax": 0.02188151, "gate_b2c/L6_rms": 0.01255417}, "streams": {"cos_a2b/round0": 0.112355, "cos_c2b/round0": 0.132304, "cos_b2a/round0": 0.182491, "cos_b2c/round0": 0.106509, "rms_a/round0": 0.196194, "rms_b/round0": 0.487802, "rms_c/round0": 0.436052, "cos_a2b/round1": 0.198495, "cos_c2b/round1": 0.285704, "cos_b2a/round1": 0.214273, "cos_b2c/round1": 0.16255, "rms_a/round1": 0.2697, "rms_b/round1": 1.428381, "rms_c/round1": 0.56802, "cos_a2b/round2": 0.163417, "cos_c2b/round2": 0.264896, "cos_b2a/round2": 0.290885, "cos_b2c/round2": 0.214335, "rms_a/round2": 0.338563, "rms_b/round2": 1.769678, "rms_c/round2": 0.663389}, "alpha_mean": 0.008171218394239677, "alpha_hist": [0.99796, 0.00195, 9e-05, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], "alpha_by_type": {"special": {"share": 0.00121, "mean": 0.00737, "std": 0.00618}, "whitespace": {"share": 0.02536, "mean": 0.00792, "std": 0.01029}, "digit": {"share": 0.02975, "mean": 0.00106, "std": 0.00094}, "word_start": {"share": 0.59625, "mean": 0.01103, "std": 0.01697}, "word_cont": {"share": 0.24405, "mean": 0.00404, "std": 0.00731}, "punct": {"share": 0.10338, "mean": 0.00353, "std": 0.00921}}, "alpha_p05": 0.00026, "alpha_p50": 0.00205, "alpha_p95": 0.03581}
{"step": 310, "total_steps": 72479, "tokens": 162529280, "loss": 7.636134684085846, "l_mix": 4.7360358238220215, "l_a": 4.9282273054122925, "l_c": 4.735965073108673, "z_loss": 8.410756826400757, "lr": 0.0004074285714285714, "wd": 0.1, "grad_norm": 1.492295503616333, "tokens_per_second": 1423346.2403292446, "eta_seconds": 26583.37065143585, "data_wait_seconds": 2.294778823852539e-05, "mix": {"fineweb_hq": 0.50968, "dclm": 0.20645, "cosmopedia_v2": 0.2, "finemath": 0.08387}}
{"step": 320, "total_steps": 72479, "tokens": 167772160, "loss": 8.362236499786377, "l_mix": 5.191720545291901, "l_a": 5.374464452266693, "l_c": 5.191770792007446, "z_loss": 6.453270256519318, "lr": 0.00042057142857142857, "wd": 0.1, "grad_norm": 0.7655479311943054, "tokens_per_second": 1450872.4576859733, "eta_seconds": 26075.412481355666, "data_wait_seconds": 3.17990779876709e-05, "mix": {"fineweb_hq": 0.50938, "dclm": 0.20938, "cosmopedia_v2": 0.2, "finemath": 0.08125}}
{"step": 330, "total_steps": 72479, "tokens": 173015040, "loss": 8.30090606212616, "l_mix": 5.157762169837952, "l_a": 5.316825866699219, "l_c": 5.1580952405929565, "z_loss": 6.673392951488495, "lr": 0.0004337142857142857, "wd": 0.1, "grad_norm": 0.5964782238006592, "tokens_per_second": 1472174.8871391385, "eta_seconds": 25694.538904619218, "data_wait_seconds": 1.940131187438965e-05, "mix": {"fineweb_hq": 0.50909, "dclm": 0.20909, "cosmopedia_v2": 0.2, "finemath": 0.08182}}
{"step": 340, "total_steps": 72479, "tokens": 178257920, "loss": 8.228565454483032, "l_mix": 5.10895836353302, "l_a": 5.286672115325928, "l_c": 5.109717011451721, "z_loss": 6.902952611446381, "lr": 0.0004468571428571429, "wd": 0.1, "grad_norm": 0.5975745916366577, "tokens_per_second": 1489562.104228415, "eta_seconds": 25391.094419384, "data_wait_seconds": 1.901388168334961e-05, "mix": {"fineweb_hq": 0.50882, "dclm": 0.20882, "cosmopedia_v2": 0.2, "finemath": 0.08235}}
{"step": 350, "total_steps": 72479, "tokens": 183500800, "loss": 7.994722843170166, "l_mix": 4.964408367872238, "l_a": 5.133355528116226, "l_c": 4.965399086475372, "z_loss": 6.879830241203308, "lr": 0.00045999999999999996, "wd": 0.1, "grad_norm": 0.8005017042160034, "tokens_per_second": 1395766.9264953367, "eta_seconds": 27093.61314854622, "data_wait_seconds": 2.3871660232543945e-05, "mix": {"fineweb_hq": 0.50857, "dclm": 0.20857, "cosmopedia_v2": 0.2, "finemath": 0.08286}}
{"step": 360, "total_steps": 72479, "tokens": 188743680, "loss": 8.082355856895447, "l_mix": 5.018507361412048, "l_a": 5.190502762794495, "l_c": 5.0197436809539795, "z_loss": 7.7420796155929565, "lr": 0.00047314285714285714, "wd": 0.1, "grad_norm": 0.7534477710723877, "tokens_per_second": 1333506.55730937, "eta_seconds": 28354.660923671723, "data_wait_seconds": 4.583597183227539e-05, "mix": {"fineweb_hq": 0.50833, "dclm": 0.20833, "cosmopedia_v2": 0.2, "finemath": 0.08333}}
{"step": 370, "total_steps": 72479, "tokens": 193986560, "loss": 8.16573053598404, "l_mix": 5.069485187530518, "l_a": 5.2469847202301025, "l_c": 5.071630001068115, "z_loss": 6.606931209564209, "lr": 0.00048628571428571427, "wd": 0.1, "grad_norm": 0.48127368092536926, "tokens_per_second": 1424965.47819625, "eta_seconds": 26531.087223148344, "data_wait_seconds": 1.7315149307250977e-05, "mix": {"fineweb_hq": 0.51081, "dclm": 0.20811, "cosmopedia_v2": 0.2, "finemath": 0.08108}}
{"step": 380, "total_steps": 72479, "tokens": 199229440, "loss": 8.094369649887085, "l_mix": 5.027669787406921, "l_a": 5.189493834972382, "l_c": 5.030614495277405, "z_loss": 6.670691132545471, "lr": 0.0004994285714285715, "wd": 0.1, "grad_norm": 0.6883058547973633, "tokens_per_second": 1315027.7833620186, "eta_seconds": 28745.126901698113, "data_wait_seconds": 2.6345252990722656e-05, "mix": {"fineweb_hq": 0.51053, "dclm": 0.20789, "cosmopedia_v2": 0.2, "finemath": 0.08158}}
{"step": 390, "total_steps": 72479, "tokens": 204472320, "loss": 8.10543030500412, "l_mix": 5.036073386669159, "l_a": 5.186490535736084, "l_c": 5.042501926422119, "z_loss": 6.5927141308784485, "lr": 0.0005125714285714285, "wd": 0.1, "grad_norm": 0.6855636835098267, "tokens_per_second": 1344117.7351262781, "eta_seconds": 28119.11236960888, "data_wait_seconds": 1.665949821472168e-05, "mix": {"fineweb_hq": 0.51026, "dclm": 0.20769, "cosmopedia_v2": 0.2, "finemath": 0.08205}}
{"step": 400, "total_steps": 72479, "tokens": 209715200, "loss": 8.015348851680756, "l_mix": 4.977989077568054, "l_a": 5.138700366020203, "l_c": 4.9833531975746155, "z_loss": 7.434688448905945, "lr": 0.0005257142857142857, "wd": 0.1, "grad_norm": 0.5640770196914673, "tokens_per_second": 1458676.4164248279, "eta_seconds": 25907.15413403511, "data_wait_seconds": 1.895427703857422e-05, "mix": {"fineweb_hq": 0.51, "dclm": 0.2075, "cosmopedia_v2": 0.2, "finemath": 0.0825}, "gates": {"gate_a2b/L1_absmean": 0.00912658, "gate_a2b/L1_absmax": 0.02123137, "gate_a2b/L1_rms": 0.01009651, "gate_a2b/L3_absmean": 0.01319675, "gate_a2b/L3_absmax": 0.03059324, "gate_a2b/L3_rms": 0.01446615, "gate_a2b/L5_absmean": 0.01422933, "gate_a2b/L5_absmax": 0.0354929, "gate_a2b/L5_rms": 0.01561502, "gate_c2b/L1_absmean": 0.00962936, "gate_c2b/L1_absmax": 0.02354888, "gate_c2b/L1_rms": 0.01066719, "gate_c2b/L3_absmean": 0.01382783, "gate_c2b/L3_absmax": 0.03103713, "gate_c2b/L3_rms": 0.01499703, "gate_c2b/L5_absmean": 0.01406144, "gate_c2b/L5_absmax": 0.036971, "gate_c2b/L5_rms": 0.01540085, "gate_b2a/L7_absmean": 0.01608248, "gate_b2a/L7_absmax": 0.0332886, "gate_b2a/L7_rms": 0.01687199, "gate_b2a/L11_absmean": 0.01263921, "gate_b2a/L11_absmax": 0.02498833, "gate_b2a/L11_rms": 0.01330274, "gate_b2a/L14_absmean": 0.01257548, "gate_b2a/L14_absmax": 0.02124501, "gate_b2a/L14_rms": 0.01308631, "gate_b2c/L3_absmean": 0.01356912, "gate_b2c/L3_absmax": 0.03108533, "gate_b2c/L3_rms": 0.01467324, "gate_b2c/L5_absmean": 0.01278995, "gate_b2c/L5_absmax": 0.02782106, "gate_b2c/L5_rms": 0.01365378, "gate_b2c/L6_absmean": 0.01260394, "gate_b2c/L6_absmax": 0.02466126, "gate_b2c/L6_rms": 0.01317412}, "streams": {"cos_a2b/round0": 0.119153, "cos_c2b/round0": 0.138743, "cos_b2a/round0": 0.127584, "cos_b2c/round0": 0.065472, "rms_a/round0": 0.202461, "rms_b/round0": 0.83232, "rms_c/round0": 0.423421, "cos_a2b/round1": 0.264372, "cos_c2b/round1": 0.311853, "cos_b2a/round1": 0.175352, "cos_b2c/round1": 0.153613, "rms_a/round1": 0.263918, "rms_b/round1": 1.888618, "rms_c/round1": 0.537699, "cos_a2b/round2": 0.276778, "cos_c2b/round2": 0.238078, "cos_b2a/round2": 0.280402, "cos_b2c/round2": 0.22268, "rms_a/round2": 0.323402, "rms_b/round2": 2.135755, "rms_c/round2": 0.62138}, "alpha_mean": 0.11341925068360448, "alpha_hist": [0.66769, 0.13159, 0.08278, 0.04536, 0.02983, 0.01846, 0.01851, 0.00549, 0.00027, 0.0], "alpha_by_type": {"special": {"share": 0.00096, "mean": 0.48517, "std": 0.2039}, "whitespace": {"share": 0.02934, "mean": 0.35316, "std": 0.24013}, "digit": {"share": 0.0433, "mean": 0.05081, "std": 0.06548}, "word_start": {"share": 0.57228, "mean": 0.14477, "std": 0.15854}, "word_cont": {"share": 0.244, "mean": 0.02559, "std": 0.08362}, "punct": {"share": 0.11011, "mean": 0.10258, "std": 0.11909}}, "alpha_p05": 0.00112, "alpha_p50": 0.04085, "alpha_p95": 0.44577}
{"step": 410, "total_steps": 72479, "tokens": 214958080, "loss": 8.006594061851501, "l_mix": 4.974054455757141, "l_a": 5.126687526702881, "l_c": 4.979224979877472, "z_loss": 7.655277073383331, "lr": 0.0005388571428571428, "wd": 0.1, "grad_norm": 0.8473768830299377, "tokens_per_second": 1422217.095652563, "eta_seconds": 26567.61192612648, "data_wait_seconds": 1.5974044799804688e-05, "mix": {"fineweb_hq": 0.50976, "dclm": 0.20732, "cosmopedia_v2": 0.2, "finemath": 0.08293}}
{"step": 420, "total_steps": 72479, "tokens": 220200960, "loss": 7.155505120754242, "l_mix": 4.433825254440308, "l_a": 4.617964506149292, "l_c": 4.451139450073242, "z_loss": 9.486081004142761, "lr": 0.000552, "wd": 0.1, "grad_norm": 0.863793671131134, "tokens_per_second": 1463531.5978734226, "eta_seconds": 25814.043951559066, "data_wait_seconds": 2.2798776626586914e-05, "mix": {"fineweb_hq": 0.50952, "dclm": 0.20714, "cosmopedia_v2": 0.2, "finemath": 0.08333}}
{"step": 430, "total_steps": 72479, "tokens": 225443840, "loss": 6.909706950187683, "l_mix": 4.2835418581962585, "l_a": 4.454911649227142, "l_c": 4.295877993106842, "z_loss": 9.281957745552063, "lr": 0.0005651428571428572, "wd": 0.1, "grad_norm": 0.9977641701698303, "tokens_per_second": 1489619.0135100717, "eta_seconds": 25358.44787788391, "data_wait_seconds": 1.996755599975586e-05, "mix": {"fineweb_hq": 0.5093, "dclm": 0.20698, "cosmopedia_v2": 0.2, "finemath": 0.08372}}
{"step": 440, "total_steps": 72479, "tokens": 230686720, "loss": 7.759162247180939, "l_mix": 4.814213335514069, "l_a": 4.969196140766144, "l_c": 4.8446972370147705, "z_loss": 7.8079522252082825, "lr": 0.0005782857142857143, "wd": 0.1, "grad_norm": 0.6866968870162964, "tokens_per_second": 1460427.0336145042, "eta_seconds": 25861.7393150568, "data_wait_seconds": 1.633167266845703e-05, "mix": {"fineweb_hq": 0.50909, "dclm": 0.20909, "cosmopedia_v2": 0.2, "finemath": 0.08182}}
{"step": 450, "total_steps": 72479, "tokens": 235929600, "loss": 7.721240818500519, "l_mix": 4.790996313095093, "l_a": 4.934426486492157, "l_c": 4.830242931842804, "z_loss": 8.436092495918274, "lr": 0.0005914285714285714, "wd": 0.1, "grad_norm": 0.5204446911811829, "tokens_per_second": 1411640.2505618897, "eta_seconds": 26751.81607847214, "data_wait_seconds": 2.2649765014648438e-05, "mix": {"fineweb_hq": 0.50889, "dclm": 0.20889, "cosmopedia_v2": 0.2, "finemath": 0.08222}}
{"step": 460, "total_steps": 72479, "tokens": 241172480, "loss": 7.707168459892273, "l_mix": 4.783446788787842, "l_a": 4.919252634048462, "l_c": 4.823724210262299, "z_loss": 8.285942077636719, "lr": 0.0006045714285714286, "wd": 0.1, "grad_norm": 0.6343427896499634, "tokens_per_second": 1485478.942023228, "eta_seconds": 25418.534321713447, "data_wait_seconds": 2.5272369384765625e-05, "mix": {"fineweb_hq": 0.5087, "dclm": 0.2087, "cosmopedia_v2": 0.2, "finemath": 0.08261}}
{"step": 470, "total_steps": 72479, "tokens": 246415360, "loss": 7.515861213207245, "l_mix": 4.6619139313697815, "l_a": 4.813498258590698, "l_c": 4.696555435657501, "z_loss": 9.309856295585632, "lr": 0.0006177142857142857, "wd": 0.1, "grad_norm": 0.5376386046409607, "tokens_per_second": 1454217.0529594591, "eta_seconds": 25961.3614866972, "data_wait_seconds": 2.0116567611694336e-05, "mix": {"fineweb_hq": 0.50851, "dclm": 0.20851, "cosmopedia_v2": 0.2, "finemath": 0.08298}}
{"step": 480, "total_steps": 72479, "tokens": 251658240, "loss": 7.694688558578491, "l_mix": 4.770952582359314, "l_a": 4.941956162452698, "l_c": 4.800941288471222, "z_loss": 8.66654884815216, "lr": 0.0006308571428571429, "wd": 0.1, "grad_norm": 0.47729483246803284, "tokens_per_second": 1482170.9696476972, "eta_seconds": 25468.189895105363, "data_wait_seconds": 1.5527009963989258e-05, "mix": {"fineweb_hq": 0.51042, "dclm": 0.20833, "cosmopedia_v2": 0.2, "finemath": 0.08125}}
{"step": 490, "total_steps": 72479, "tokens": 256901120, "loss": 7.677704751491547, "l_mix": 4.759032130241394, "l_a": 4.926066517829895, "l_c": 4.799805462360382, "z_loss": 9.109945178031921, "lr": 0.000644, "wd": 0.1, "grad_norm": 0.7182350158691406, "tokens_per_second": 1296446.827348124, "eta_seconds": 29112.623854541776, "data_wait_seconds": 2.1398067474365234e-05, "mix": {"fineweb_hq": 0.5102, "dclm": 0.20816, "cosmopedia_v2": 0.2, "finemath": 0.08163}}
{"step": 500, "total_steps": 72479, "tokens": 262144000, "loss": 7.659057259559631, "l_mix": 4.748191177845001, "l_a": 4.921480119228363, "l_c": 4.778606355190277, "z_loss": 8.400300979614258, "lr": 0.0006571428571428571, "wd": 0.1, "grad_norm": 0.5006059408187866, "tokens_per_second": 1304689.82413335, "eta_seconds": 28924.672557377817, "data_wait_seconds": 1.564621925354004e-05, "mix": {"fineweb_hq": 0.51, "dclm": 0.208, "cosmopedia_v2": 0.2, "finemath": 0.082}, "gates": {"gate_a2b/L1_absmean": 0.00911614, "gate_a2b/L1_absmax": 0.0273011, "gate_a2b/L1_rms": 0.01064124, "gate_a2b/L3_absmean": 0.01707566, "gate_a2b/L3_absmax": 0.04492782, "gate_a2b/L3_rms": 0.01897563, "gate_a2b/L5_absmean": 0.0183104, "gate_a2b/L5_absmax": 0.05013235, "gate_a2b/L5_rms": 0.02047768, "gate_c2b/L1_absmean": 0.00913685, "gate_c2b/L1_absmax": 0.02687647, "gate_c2b/L1_rms": 0.01069981, "gate_c2b/L3_absmean": 0.01894072, "gate_c2b/L3_absmax": 0.04160669, "gate_c2b/L3_rms": 0.02075596, "gate_c2b/L5_absmean": 0.01972991, "gate_c2b/L5_absmax": 0.04703695, "gate_c2b/L5_rms": 0.02179375, "gate_b2a/L7_absmean": 0.0160981, "gate_b2a/L7_absmax": 0.03635528, "gate_b2a/L7_rms": 0.01731696, "gate_b2a/L11_absmean": 0.01261898, "gate_b2a/L11_absmax": 0.02666332, "gate_b2a/L11_rms": 0.01374069, "gate_b2a/L14_absmean": 0.01292155, "gate_b2a/L14_absmax": 0.02358837, "gate_b2a/L14_rms": 0.01358033, "gate_b2c/L3_absmean": 0.013046, "gate_b2c/L3_absmax": 0.03518847, "gate_b2c/L3_rms": 0.01493182, "gate_b2c/L5_absmean": 0.01364084, "gate_b2c/L5_absmax": 0.03569882, "gate_b2c/L5_rms": 0.01496807, "gate_b2c/L6_absmean": 0.01303823, "gate_b2c/L6_absmax": 0.02626981, "gate_b2c/L6_rms": 0.0138584}, "streams": {"cos_a2b/round0": 0.1548, "cos_c2b/round0": 0.17364, "cos_b2a/round0": 0.138069, "cos_b2c/round0": 0.082134, "rms_a/round0": 0.225396, "rms_b/round0": 0.663784, "rms_c/round0": 0.42441, "cos_a2b/round1": 0.335291, "cos_c2b/round1": 0.335025, "cos_b2a/round1": 0.143903, "cos_b2c/round1": 0.125052, "rms_a/round1": 0.289333, "rms_b/round1": 1.628569, "rms_c/round1": 0.532996, "cos_a2b/round2": 0.206509, "cos_c2b/round2": 0.245506, "cos_b2a/round2": 0.197939, "cos_b2c/round2": 0.172866, "rms_a/round2": 0.346048, "rms_b/round2": 1.754593, "rms_c/round2": 0.611284}, "alpha_mean": 0.27539911557301444, "alpha_hist": [0.26801, 0.17375, 0.15532, 0.1246, 0.10036, 0.08092, 0.06155, 0.03217, 0.00333, 0.0], "alpha_by_type": {"special": {"share": 0.00093, "mean": 0.62774, "std": 0.12608}, "whitespace": {"share": 0.02753, "mean": 0.47737, "std": 0.21933}, "digit": {"share": 0.0506, "mean": 0.2355, "std": 0.13432}, "word_start": {"share": 0.56148, "mean": 0.32628, "std": 0.1991}, "word_cont": {"share": 0.24683, "mean": 0.10822, "std": 0.13676}, "punct": {"share": 0.11264, "mean": 0.35376, "std": 0.20424}}, "alpha_p05": 0.01396, "alpha_p50": 0.23441, "alpha_p95": 0.66454}
{"step": 510, "total_steps": 72479, "tokens": 267386880, "loss": 7.582219421863556, "l_mix": 4.698551535606384, "l_a": 4.876089453697205, "l_c": 4.733270168304443, "z_loss": 8.599031448364258, "lr": 0.0006702857142857143, "wd": 0.1, "grad_norm": 0.4348697364330292, "tokens_per_second": 1248267.5464610031, "eta_seconds": 30227.8811773777, "data_wait_seconds": 1.773238182067871e-05, "mix": {"fineweb_hq": 0.5098, "dclm": 0.20784, "cosmopedia_v2": 0.2, "finemath": 0.08235}}
{"step": 520, "total_steps": 72479, "tokens": 272629760, "loss": 7.54559463262558, "l_mix": 4.67432564496994, "l_a": 4.847856879234314, "l_c": 4.720065712928772, "z_loss": 8.919731855392456, "lr": 0.0006834285714285714, "wd": 0.1, "grad_norm": 0.5915257334709167, "tokens_per_second": 1206935.5279049433, "eta_seconds": 31258.703816175464, "data_wait_seconds": 2.1249055862426758e-05, "mix": {"fineweb_hq": 0.50962, "dclm": 0.20769, "cosmopedia_v2": 0.2, "finemath": 0.08269}}
{"step": 530, "total_steps": 72479, "tokens": 277872640, "loss": 6.5138044357299805, "l_mix": 4.026084840297699, "l_a": 4.189317584037781, "l_c": 4.099274396896362, "z_loss": 11.419251680374146, "lr": 0.0006965714285714287, "wd": 0.1, "grad_norm": 0.9339258670806885, "tokens_per_second": 1390540.3128183065, "eta_seconds": 27127.582684421537, "data_wait_seconds": 2.4527311325073242e-05, "mix": {"fineweb_hq": 0.50943, "dclm": 0.20755, "cosmopedia_v2": 0.2, "finemath": 0.08302}}
{"step": 540, "total_steps": 72479, "tokens": 283115520, "loss": 6.589920520782471, "l_mix": 4.067920058965683, "l_a": 4.245975315570831, "l_c": 4.1572476625442505, "z_loss": 10.3354994058609, "lr": 0.0007097142857142857, "wd": 0.1, "grad_norm": 1.0875746011734009, "tokens_per_second": 1457557.2020871416, "eta_seconds": 25876.68900952339, "data_wait_seconds": 1.874566078186035e-05, "mix": {"fineweb_hq": 0.50926, "dclm": 0.20741, "cosmopedia_v2": 0.2, "finemath": 0.08333}}
{"step": 550, "total_steps": 72479, "tokens": 288358400, "loss": 6.5149224400520325, "l_mix": 4.021407961845398, "l_a": 4.2499688267707825, "l_c": 4.05795630812645, "z_loss": 11.368615865707397, "lr": 0.0007228571428571428, "wd": 0.1, "grad_norm": 0.8756926655769348, "tokens_per_second": 1415271.4187395659, "eta_seconds": 26646.13377523422, "data_wait_seconds": 1.990795135498047e-05, "mix": {"fineweb_hq": 0.50909, "dclm": 0.20727, "cosmopedia_v2": 0.2, "finemath": 0.08364}}
{"step": 560, "total_steps": 72479, "tokens": 293601280, "loss": 7.467288494110107, "l_mix": 4.617526650428772, "l_a": 4.836296916007996, "l_c": 4.659860551357269, "z_loss": 9.145410060882568, "lr": 0.000736, "wd": 0.1, "grad_norm": 0.47158533334732056, "tokens_per_second": 1369258.2394675952, "eta_seconds": 27537.733632087708, "data_wait_seconds": 2.065300941467285e-05, "mix": {"fineweb_hq": 0.51071, "dclm": 0.20714, "cosmopedia_v2": 0.2, "finemath": 0.08214}}
{"step": 570, "total_steps": 72479, "tokens": 298844160, "loss": 7.419131636619568, "l_mix": 4.586887240409851, "l_a": 4.7921154499053955, "l_c": 4.645599544048309, "z_loss": 9.297641277313232, "lr": 0.0007491428571428571, "wd": 0.1, "grad_norm": 0.5330434441566467, "tokens_per_second": 1485541.0592396974, "eta_seconds": 25378.649454021455, "data_wait_seconds": 1.9609928131103516e-05, "mix": {"fineweb_hq": 0.51053, "dclm": 0.20702, "cosmopedia_v2": 0.2, "finemath": 0.08246}}
{"step": 580, "total_steps": 72479, "tokens": 304087040, "loss": 7.317393124103546, "l_mix": 4.522009193897247, "l_a": 4.733172237873077, "l_c": 4.581643283367157, "z_loss": 9.391452193260193, "lr": 0.0007622857142857144, "wd": 0.1, "grad_norm": 0.45536351203918457, "tokens_per_second": 1454471.1720790523, "eta_seconds": 25917.174321246148, "data_wait_seconds": 2.7835369110107422e-05, "mix": {"fineweb_hq": 0.51034, "dclm": 0.2069, "cosmopedia_v2": 0.2, "finemath": 0.08276}}
{"step": 590, "total_steps": 72479, "tokens": 309329920, "loss": 7.23209673166275, "l_mix": 4.468189537525177, "l_a": 4.686687707901001, "l_c": 4.5229862332344055, "z_loss": 10.048566460609436, "lr": 0.0007754285714285714, "wd": 0.1, "grad_norm": 0.5544047355651855, "tokens_per_second": 1461898.991062367, "eta_seconds": 25781.904401350024, "data_wait_seconds": 1.6361474990844727e-05, "mix": {"fineweb_hq": 0.51017, "dclm": 0.20678, "cosmopedia_v2": 0.2, "finemath": 0.08305}}
{"step": 600, "total_steps": 72479, "tokens": 314572800, "loss": 7.29384446144104, "l_mix": 4.505487620830536, "l_a": 4.728361546993256, "l_c": 4.562880218029022, "z_loss": 9.843735098838806, "lr": 0.0007885714285714285, "wd": 0.1, "grad_norm": 0.717620849609375, "tokens_per_second": 1480707.2787487153, "eta_seconds": 25450.875870513915, "data_wait_seconds": 1.4722347259521484e-05, "mix": {"fineweb_hq": 0.51, "dclm": 0.20833, "cosmopedia_v2": 0.2, "finemath": 0.08167}, "gates": {"gate_a2b/L1_absmean": 0.00951174, "gate_a2b/L1_absmax": 0.02648037, "gate_a2b/L1_rms": 0.01135209, "gate_a2b/L3_absmean": 0.02272609, "gate_a2b/L3_absmax": 0.06354133, "gate_a2b/L3_rms": 0.0251126, "gate_a2b/L5_absmean": 0.02547642, "gate_a2b/L5_absmax": 0.08136226, "gate_a2b/L5_rms": 0.02877224, "gate_c2b/L1_absmean": 0.0088055, "gate_c2b/L1_absmax": 0.02760023, "gate_c2b/L1_rms": 0.01080732, "gate_c2b/L3_absmean": 0.0274119, "gate_c2b/L3_absmax": 0.05749251, "gate_c2b/L3_rms": 0.02972617, "gate_c2b/L5_absmean": 0.03468698, "gate_c2b/L5_absmax": 0.07775985, "gate_c2b/L5_rms": 0.03792173, "gate_b2a/L7_absmean": 0.01623694, "gate_b2a/L7_absmax": 0.04324465, "gate_b2a/L7_rms": 0.01811443, "gate_b2a/L11_absmean": 0.0125775, "gate_b2a/L11_absmax": 0.03038116, "gate_b2a/L11_rms": 0.01415874, "gate_b2a/L14_absmean": 0.01313599, "gate_b2a/L14_absmax": 0.02667063, "gate_b2a/L14_rms": 0.01407922, "gate_b2c/L3_absmean": 0.01245341, "gate_b2c/L3_absmax": 0.04217337, "gate_b2c/L3_rms": 0.01514241, "gate_b2c/L5_absmean": 0.01418168, "gate_b2c/L5_absmax": 0.03944426, "gate_b2c/L5_rms": 0.01597241, "gate_b2c/L6_absmean": 0.01301625, "gate_b2c/L6_absmax": 0.02700233, "gate_b2c/L6_rms": 0.014114}, "streams": {"cos_a2b/round0": 0.157129, "cos_c2b/round0": 0.185136, "cos_b2a/round0": 0.138104, "cos_b2c/round0": 0.080712, "rms_a/round0": 0.250687, "rms_b/round0": 0.695683, "rms_c/round0": 0.432455, "cos_a2b/round1": 0.343478, "cos_c2b/round1": 0.335437, "cos_b2a/round1": 0.139192, "cos_b2c/round1": 0.129591, "rms_a/round1": 0.321358, "rms_b/round1": 1.673807, "rms_c/round1": 0.554337, "cos_a2b/round2": 0.229668, "cos_c2b/round2": 0.313157, "cos_b2a/round2": 0.189362, "cos_b2c/round2": 0.170532, "rms_a/round2": 0.390227, "rms_b/round2": 1.712543, "rms_c/round2": 0.644681}, "alpha_mean": 0.3232998523880166, "alpha_hist": [0.19986, 0.15622, 0.1593, 0.13931, 0.10397, 0.08401, 0.08699, 0.06133, 0.00894, 6e-05], "alpha_by_type": {"special": {"share": 0.00101, "mean": 0.6396, "std": 0.14381}, "whitespace": {"share": 0.02318, "mean": 0.54781, "std": 0.19939}, "digit": {"share": 0.02954, "mean": 0.32894, "std": 0.16177}, "word_start": {"share": 0.58574, "mean": 0.37113, "std": 0.20418}, "word_cont": {"share": 0.24428, "mean": 0.15199, "std": 0.16022}, "punct": {"share": 0.11626, "mean": 0.39335, "std": 0.24791}}, "alpha_p05": 0.02187, "alpha_p50": 0.27944, "alpha_p95": 0.71701}
{"step": 610, "total_steps": 72479, "tokens": 319815680, "loss": 7.262825965881348, "l_mix": 4.484340667724609, "l_a": 4.712722420692444, "l_c": 4.545749306678772, "z_loss": 9.436321496963501, "lr": 0.0008017142857142857, "wd": 0.1, "grad_norm": 0.5072914958000183, "tokens_per_second": 1463067.6175276975, "eta_seconds": 25754.144115138053, "data_wait_seconds": 1.9550323486328125e-05, "mix": {"fineweb_hq": 0.50984, "dclm": 0.2082, "cosmopedia_v2": 0.2, "finemath": 0.08197}}
{"step": 620, "total_steps": 72479, "tokens": 325058560, "loss": 7.1504247188568115, "l_mix": 4.414298057556152, "l_a": 4.6423821449279785, "l_c": 4.474579095840454, "z_loss": 10.382166266441345, "lr": 0.0008148571428571428, "wd": 0.1, "grad_norm": 0.4578287601470947, "tokens_per_second": 1463922.5823827202, "eta_seconds": 25735.52170407772, "data_wait_seconds": 1.8030405044555664e-05, "mix": {"fineweb_hq": 0.50968, "dclm": 0.20806, "cosmopedia_v2": 0.2, "finemath": 0.08226}}
{"step": 630, "total_steps": 72479, "tokens": 330301440, "loss": 7.3093244433403015, "l_mix": 4.510521233081818, "l_a": 4.7528557777404785, "l_c": 4.573092997074127, "z_loss": 10.185728311538696, "lr": 0.0008280000000000001, "wd": 0.1, "grad_norm": 0.5709524750709534, "tokens_per_second": 1371945.071588697, "eta_seconds": 27457.052976894378, "data_wait_seconds": 2.104043960571289e-05, "mix": {"fineweb_hq": 0.50952, "dclm": 0.20794, "cosmopedia_v2": 0.2, "finemath": 0.08254}}
{"step": 640, "total_steps": 72479, "tokens": 335544320, "loss": 7.262300670146942, "l_mix": 4.482975602149963, "l_a": 4.705637514591217, "l_c": 4.5553863644599915, "z_loss": 10.178205966949463, "lr": 0.0008411428571428571, "wd": 0.1, "grad_norm": 0.4564012885093689, "tokens_per_second": 1389256.1383131058, "eta_seconds": 27111.14573712349, "data_wait_seconds": 3.4362077713012695e-05, "mix": {"fineweb_hq": 0.50938, "dclm": 0.20781, "cosmopedia_v2": 0.2, "finemath": 0.08281}}
{"step": 650, "total_steps": 72479, "tokens": 340787200, "loss": 6.227408468723297, "l_mix": 3.83123779296875, "l_a": 4.092753529548645, "l_c": 3.8903577029705048, "z_loss": 12.373312830924988, "lr": 0.0008542857142857142, "wd": 0.1, "grad_norm": 1.0204155445098877, "tokens_per_second": 1319122.506404635, "eta_seconds": 28548.58632853031, "data_wait_seconds": 3.1322240829467773e-05, "mix": {"fineweb_hq": 0.50923, "dclm": 0.20769, "cosmopedia_v2": 0.2, "finemath": 0.08308}}
{"step": 660, "total_steps": 72479, "tokens": 346030080, "loss": 6.21194326877594, "l_mix": 3.8282848596572876, "l_a": 4.038806259632111, "l_c": 3.902426928281784, "z_loss": 12.884539723396301, "lr": 0.0008674285714285714, "wd": 0.1, "grad_norm": 0.8151845335960388, "tokens_per_second": 1445344.666068773, "eta_seconds": 26051.806711554527, "data_wait_seconds": 2.6792287826538086e-05, "mix": {"fineweb_hq": 0.50909, "dclm": 0.20758, "cosmopedia_v2": 0.2, "finemath": 0.08333}}
{"step": 670, "total_steps": 72479, "tokens": 351272960, "loss": 7.04491400718689, "l_mix": 4.344374597072601, "l_a": 4.5813546776771545, "l_c": 4.416974186897278, "z_loss": 10.407228112220764, "lr": 0.0008805714285714285, "wd": 0.1, "grad_norm": 0.3614371716976166, "tokens_per_second": 1442914.727497051, "eta_seconds": 26092.045686793328, "data_wait_seconds": 2.0056962966918945e-05, "mix": {"fineweb_hq": 0.51045, "dclm": 0.20746, "cosmopedia_v2": 0.2, "finemath": 0.08209}}
{"step": 680, "total_steps": 72479, "tokens": 356515840, "loss": 7.105412662029266, "l_mix": 4.3795424699783325, "l_a": 4.626061141490936, "l_c": 4.456656754016876, "z_loss": 10.548259854316711, "lr": 0.0008937142857142858, "wd": 0.1, "grad_norm": 0.463230162858963, "tokens_per_second": 1469804.3653944472, "eta_seconds": 25611.132337260246, "data_wait_seconds": 1.4901161193847656e-05, "mix": {"fineweb_hq": 0.51029, "dclm": 0.20735, "cosmopedia_v2": 0.2, "finemath": 0.08235}}
{"step": 690, "total_steps": 72479, "tokens": 361758720, "loss": 7.009087145328522, "l_mix": 4.317391276359558, "l_a": 4.575866222381592, "l_c": 4.392760634422302, "z_loss": 11.076057195663452, "lr": 0.0009068571428571428, "wd": 0.1, "grad_norm": 0.4798468053340912, "tokens_per_second": 1476249.4554261295, "eta_seconds": 25495.76637854576, "data_wait_seconds": 1.6480684280395508e-05, "mix": {"fineweb_hq": 0.51014, "dclm": 0.20725, "cosmopedia_v2": 0.2, "finemath": 0.08261}}
{"step": 700, "total_steps": 72479, "tokens": 367001600, "loss": 7.0116278529167175, "l_mix": 4.31846559047699, "l_a": 4.58609938621521, "l_c": 4.387661635875702, "z_loss": 10.33922791481018, "lr": 0.0009199999999999999, "wd": 0.1, "grad_norm": 0.5966548919677734, "tokens_per_second": 1389468.6564725777, "eta_seconds": 27084.35931727886, "data_wait_seconds": 2.1219253540039062e-05, "mix": {"fineweb_hq": 0.51, "dclm": 0.20714, "cosmopedia_v2": 0.2, "finemath": 0.08286}, "gates": {"gate_a2b/L1_absmean": 0.00943234, "gate_a2b/L1_absmax": 0.03133131, "gate_a2b/L1_rms": 0.0117903, "gate_a2b/L3_absmean": 0.03098884, "gate_a2b/L3_absmax": 0.07606009, "gate_a2b/L3_rms": 0.03365894, "gate_a2b/L5_absmean": 0.03877776, "gate_a2b/L5_absmax": 0.11644544, "gate_a2b/L5_rms": 0.04330036, "gate_c2b/L1_absmean": 0.00818602, "gate_c2b/L1_absmax": 0.03544301, "gate_c2b/L1_rms": 0.01068548, "gate_c2b/L3_absmean": 0.04137238, "gate_c2b/L3_absmax": 0.08693354, "gate_c2b/L3_rms": 0.04423244, "gate_c2b/L5_absmean": 0.06570216, "gate_c2b/L5_absmax": 0.1410007, "gate_c2b/L5_rms": 0.07036769, "gate_b2a/L7_absmean": 0.01639332, "gate_b2a/L7_absmax": 0.05044279, "gate_b2a/L7_rms": 0.01903023, "gate_b2a/L11_absmean": 0.01297581, "gate_b2a/L11_absmax": 0.03526942, "gate_b2a/L11_rms": 0.01501033, "gate_b2a/L14_absmean": 0.01421901, "gate_b2a/L14_absmax": 0.03127528, "gate_b2a/L14_rms": 0.01541953, "gate_b2c/L3_absmean": 0.01242112, "gate_b2c/L3_absmax": 0.04789572, "gate_b2c/L3_rms": 0.01584871, "gate_b2c/L5_absmean": 0.01405554, "gate_b2c/L5_absmax": 0.03821365, "gate_b2c/L5_rms": 0.01622241, "gate_b2c/L6_absmean": 0.01219612, "gate_b2c/L6_absmax": 0.02802932, "gate_b2c/L6_rms": 0.01355431}, "streams": {"cos_a2b/round0": 0.18101, "cos_c2b/round0": 0.221803, "cos_b2a/round0": 0.145148, "cos_b2c/round0": 0.109445, "rms_a/round0": 0.273252, "rms_b/round0": 0.703441, "rms_c/round0": 0.447279, "cos_a2b/round1": 0.381718, "cos_c2b/round1": 0.373114, "cos_b2a/round1": 0.124358, "cos_b2c/round1": 0.131537, "rms_a/round1": 0.352068, "rms_b/round1": 1.519692, "rms_c/round1": 0.584945, "cos_a2b/round2": 0.223684, "cos_c2b/round2": 0.375073, "cos_b2a/round2": 0.187033, "cos_b2c/round2": 0.157813, "rms_a/round2": 0.435926, "rms_b/round2": 1.62224, "rms_c/round2": 0.685164}, "alpha_mean": 0.34583803094020027, "alpha_hist": [0.17825, 0.16089, 0.14725, 0.12602, 0.10962, 0.08902, 0.09273, 0.07497, 0.02029, 0.00096], "alpha_by_type": {"special": {"share": 0.00098, "mean": 0.67327, "std": 0.0954}, "whitespace": {"share": 0.02484, "mean": 0.56906, "std": 0.19995}, "digit": {"share": 0.02673, "mean": 0.36837, "std": 0.21708}, "word_start": {"share": 0.61363, "mean": 0.38364, "std": 0.21329}, "word_cont": {"share": 0.23244, "mean": 0.15448, "std": 0.1683}, "punct": {"share": 0.10138, "mean": 0.49194, "std": 0.21913}}, "alpha_p05": 0.03397, "alpha_p50": 0.29503, "alpha_p95": 0.74945}
{"step": 710, "total_steps": 72479, "tokens": 372244480, "loss": 7.003850936889648, "l_mix": 4.312046825885773, "l_a": 4.5798468589782715, "l_c": 4.389058947563171, "z_loss": 11.322470307350159, "lr": 0.0009331428571428571, "wd": 0.1, "grad_norm": 0.7068897485733032, "tokens_per_second": 1345835.1192247975, "eta_seconds": 27958.570061445233, "data_wait_seconds": 2.41696834564209e-05, "mix": {"fineweb_hq": 0.50986, "dclm": 0.20704, "cosmopedia_v2": 0.2, "finemath": 0.0831}}
{"step": 720, "total_steps": 72479, "tokens": 377487360, "loss": 6.685384929180145, "l_mix": 4.112654805183411, "l_a": 4.362689197063446, "l_c": 4.209090203046799, "z_loss": 11.961074829101562, "lr": 0.0009462857142857143, "wd": 0.1, "grad_norm": 0.44269436597824097, "tokens_per_second": 1341468.718580896, "eta_seconds": 28045.66522564888, "data_wait_seconds": 2.3186206817626953e-05, "mix": {"fineweb_hq": 0.50972, "dclm": 0.20833, "cosmopedia_v2": 0.2, "finemath": 0.08194}}
{"step": 730, "total_steps": 72479, "tokens": 382730240, "loss": 6.930765211582184, "l_mix": 4.263782858848572, "l_a": 4.5291460156440735, "l_c": 4.356947422027588, "z_loss": 11.542965650558472, "lr": 0.0009594285714285715, "wd": 0.1, "grad_norm": 0.47072193026542664, "tokens_per_second": 1385957.0789905246, "eta_seconds": 27141.633952617645, "data_wait_seconds": 1.704692840576172e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20822, "cosmopedia_v2": 0.2, "finemath": 0.08219}}
{"step": 740, "total_steps": 72479, "tokens": 387973120, "loss": 6.934702754020691, "l_mix": 4.262257814407349, "l_a": 4.551520764827728, "l_c": 4.352697968482971, "z_loss": 11.79190182685852, "lr": 0.0009725714285714285, "wd": 0.1, "grad_norm": 0.43337056040763855, "tokens_per_second": 1467514.5208672485, "eta_seconds": 25629.65905766487, "data_wait_seconds": 2.193450927734375e-05, "mix": {"fineweb_hq": 0.50946, "dclm": 0.20811, "cosmopedia_v2": 0.2, "finemath": 0.08243}}
{"step": 750, "total_steps": 72479, "tokens": 393216000, "loss": 6.740262866020203, "l_mix": 4.138554692268372, "l_a": 4.438139736652374, "l_c": 4.230253338813782, "z_loss": 11.902667760848999, "lr": 0.0009857142857142857, "wd": 0.1, "grad_norm": 0.4555477797985077, "tokens_per_second": 1410448.8043939858, "eta_seconds": 26662.898954463006, "data_wait_seconds": 1.958012580871582e-05, "mix": {"fineweb_hq": 0.50933, "dclm": 0.208, "cosmopedia_v2": 0.2, "finemath": 0.08267}}
{"step": 760, "total_steps": 72479, "tokens": 398458880, "loss": 6.831307470798492, "l_mix": 4.195277810096741, "l_a": 4.470397889614105, "l_c": 4.312382280826569, "z_loss": 11.955536842346191, "lr": 0.000998857142857143, "wd": 0.1, "grad_norm": 0.47143030166625977, "tokens_per_second": 1423388.896837372, "eta_seconds": 26416.821963095663, "data_wait_seconds": 2.2143125534057617e-05, "mix": {"fineweb_hq": 0.50921, "dclm": 0.20789, "cosmopedia_v2": 0.2, "finemath": 0.08289}}
{"step": 770, "total_steps": 72479, "tokens": 403701760, "loss": 5.5773597955703735, "l_mix": 3.3927195072174072, "l_a": 3.8041904866695404, "l_c": 3.473253697156906, "z_loss": 14.068711042404175, "lr": 0.0010119999999999999, "wd": 0.1, "grad_norm": 1.0434753894805908, "tokens_per_second": 1384510.3050499149, "eta_seconds": 27154.84894180298, "data_wait_seconds": 2.0325183868408203e-05, "mix": {"fineweb_hq": 0.50909, "dclm": 0.20779, "cosmopedia_v2": 0.2, "finemath": 0.08312}}
{"step": 780, "total_steps": 72479, "tokens": 408944640, "loss": 6.689542770385742, "l_mix": 4.094645917415619, "l_a": 4.451086580753326, "l_c": 4.194641530513763, "z_loss": 11.784537553787231, "lr": 0.001025142857142857, "wd": 0.1, "grad_norm": 0.407647043466568, "tokens_per_second": 1448013.3194429125, "eta_seconds": 25960.344982504845, "data_wait_seconds": 2.4318695068359375e-05, "mix": {"fineweb_hq": 0.51026, "dclm": 0.20769, "cosmopedia_v2": 0.2, "finemath": 0.08205}}
{"step": 790, "total_steps": 72479, "tokens": 414187520, "loss": 6.647393226623535, "l_mix": 4.060894072055817, "l_a": 4.456596612930298, "l_c": 4.161184668540955, "z_loss": 11.646533131599426, "lr": 0.0010382857142857142, "wd": 0.1, "grad_norm": 0.4686763882637024, "tokens_per_second": 1471723.6328688697, "eta_seconds": 25538.546499204636, "data_wait_seconds": 1.582503318786621e-05, "mix": {"fineweb_hq": 0.51013, "dclm": 0.20759, "cosmopedia_v2": 0.2, "finemath": 0.08228}}
{"step": 800, "total_steps": 72479, "tokens": 419430400, "loss": 6.645305097103119, "l_mix": 4.06000217795372, "l_a": 4.448550164699554, "l_c": 4.165169298648834, "z_loss": 11.870200395584106, "lr": 0.0010514285714285714, "wd": 0.1, "grad_norm": 0.4797481298446655, "tokens_per_second": 1478641.2268329326, "eta_seconds": 25415.522623085977, "data_wait_seconds": 1.5228986740112305e-05, "mix": {"fineweb_hq": 0.51, "dclm": 0.2075, "cosmopedia_v2": 0.2, "finemath": 0.0825}, "gates": {"gate_a2b/L1_absmean": 0.00990143, "gate_a2b/L1_absmax": 0.0397533, "gate_a2b/L1_rms": 0.01278254, "gate_a2b/L3_absmean": 0.04065287, "gate_a2b/L3_absmax": 0.09248795, "gate_a2b/L3_rms": 0.04370068, "gate_a2b/L5_absmean": 0.05236755, "gate_a2b/L5_absmax": 0.16247088, "gate_a2b/L5_rms": 0.05830738, "gate_c2b/L1_absmean": 0.00766241, "gate_c2b/L1_absmax": 0.03535262, "gate_c2b/L1_rms": 0.01057259, "gate_c2b/L3_absmean": 0.06151372, "gate_c2b/L3_absmax": 0.12542078, "gate_c2b/L3_rms": 0.06502327, "gate_c2b/L5_absmean": 0.11828034, "gate_c2b/L5_absmax": 0.23120473, "gate_c2b/L5_rms": 0.12564236, "gate_b2a/L7_absmean": 0.01636915, "gate_b2a/L7_absmax": 0.05849764, "gate_b2a/L7_rms": 0.01980571, "gate_b2a/L11_absmean": 0.0135861, "gate_b2a/L11_absmax": 0.04165039, "gate_b2a/L11_rms": 0.01626761, "gate_b2a/L14_absmean": 0.01879722, "gate_b2a/L14_absmax": 0.03804953, "gate_b2a/L14_rms": 0.02029336, "gate_b2c/L3_absmean": 0.01281155, "gate_b2c/L3_absmax": 0.05221343, "gate_b2c/L3_rms": 0.01719103, "gate_b2c/L5_absmean": 0.01505497, "gate_b2c/L5_absmax": 0.0408311, "gate_b2c/L5_rms": 0.01735592, "gate_b2c/L6_absmean": 0.01190263, "gate_b2c/L6_absmax": 0.02805746, "gate_b2c/L6_rms": 0.01369973}, "streams": {"cos_a2b/round0": 0.183531, "cos_c2b/round0": 0.228287, "cos_b2a/round0": 0.130791, "cos_b2c/round0": 0.104413, "rms_a/round0": 0.301489, "rms_b/round0": 0.822753, "rms_c/round0": 0.472808, "cos_a2b/round1": 0.337466, "cos_c2b/round1": 0.300992, "cos_b2a/round1": 0.113481, "cos_b2c/round1": 0.11461, "rms_a/round1": 0.394208, "rms_b/round1": 1.366976, "rms_c/round1": 0.627774, "cos_a2b/round2": 0.20112, "cos_c2b/round2": 0.511826, "cos_b2a/round2": 0.164946, "cos_b2c/round2": 0.145018, "rms_a/round2": 0.498189, "rms_b/round2": 1.963362, "rms_c/round2": 0.761561}, "alpha_mean": 0.3946119143717848, "alpha_hist": [0.13304, 0.11533, 0.13086, 0.14365, 0.13292, 0.11749, 0.10031, 0.08878, 0.0352, 0.00243], "alpha_by_type": {"special": {"share": 0.0011, "mean": 0.71026, "std": 0.11855}, "whitespace": {"share": 0.03517, "mean": 0.57326, "std": 0.18078}, "digit": {"share": 0.05794, "mean": 0.51681, "std": 0.23436}, "word_start": {"share": 0.49179, "mean": 0.45614, "std": 0.19883}, "word_cont": {"share": 0.29443, "mean": 0.20782, "std": 0.17229}, "punct": {"share": 0.11957, "mean": 0.48686, "std": 0.2367}}, "alpha_p05": 0.04672, "alpha_p50": 0.37114, "alpha_p95": 0.76731}
{"step": 810, "total_steps": 72479, "tokens": 424673280, "loss": 6.521934270858765, "l_mix": 3.980577737092972, "l_a": 4.376259207725525, "l_c": 4.091027230024338, "z_loss": 11.70432198047638, "lr": 0.0010645714285714286, "wd": 0.1, "grad_norm": 0.5098571181297302, "tokens_per_second": 1429411.1431476094, "eta_seconds": 26287.18605709076, "data_wait_seconds": 1.6450881958007812e-05, "mix": {"fineweb_hq": 0.50988, "dclm": 0.20741, "cosmopedia_v2": 0.2, "finemath": 0.08272}}
{"step": 820, "total_steps": 72479, "tokens": 429916160, "loss": 6.574698030948639, "l_mix": 4.011743605136871, "l_a": 4.418120384216309, "l_c": 4.121261775493622, "z_loss": 11.396878361701965, "lr": 0.0010777142857142856, "wd": 0.1, "grad_norm": 0.44843605160713196, "tokens_per_second": 1462464.542295005, "eta_seconds": 25689.48012444973, "data_wait_seconds": 1.5288591384887695e-05, "mix": {"fineweb_hq": 0.50976, "dclm": 0.20732, "cosmopedia_v2": 0.2, "finemath": 0.08293}}
{"step": 830, "total_steps": 72479, "tokens": 435159040, "loss": 6.584240674972534, "l_mix": 4.015634894371033, "l_a": 4.425634443759918, "l_c": 4.132119119167328, "z_loss": 12.796822667121887, "lr": 0.001090857142857143, "wd": 0.1, "grad_norm": 0.44623711705207825, "tokens_per_second": 1441763.692743278, "eta_seconds": 26054.69335999489, "data_wait_seconds": 2.0116567611694336e-05, "mix": {"fineweb_hq": 0.50964, "dclm": 0.20843, "cosmopedia_v2": 0.2, "finemath": 0.08193}}
{"step": 840, "total_steps": 72479, "tokens": 440401920, "loss": 6.47639387845993, "l_mix": 3.947252094745636, "l_a": 4.347894787788391, "l_c": 4.0783069133758545, "z_loss": 12.810539603233337, "lr": 0.001104, "wd": 0.1, "grad_norm": 0.5287054777145386, "tokens_per_second": 1466997.8054318505, "eta_seconds": 25602.947661495207, "data_wait_seconds": 1.665949821472168e-05, "mix": {"fineweb_hq": 0.50952, "dclm": 0.20833, "cosmopedia_v2": 0.2, "finemath": 0.08214}}
{"step": 850, "total_steps": 72479, "tokens": 445644800, "loss": 6.468699634075165, "l_mix": 3.9459377825260162, "l_a": 4.330409824848175, "l_c": 4.074492871761322, "z_loss": 12.910410165786743, "lr": 0.0011171428571428571, "wd": 0.1, "grad_norm": 0.4117547869682312, "tokens_per_second": 1467506.0985625132, "eta_seconds": 25590.507043743135, "data_wait_seconds": 1.9490718841552734e-05, "mix": {"fineweb_hq": 0.50941, "dclm": 0.20824, "cosmopedia_v2": 0.2, "finemath": 0.08235}}
{"step": 860, "total_steps": 72479, "tokens": 450887680, "loss": 6.449058711528778, "l_mix": 3.9338099360466003, "l_a": 4.320351660251617, "l_c": 4.0593381524086, "z_loss": 13.417844772338867, "lr": 0.0011302857142857143, "wd": 0.1, "grad_norm": 0.4571038484573364, "tokens_per_second": 1413465.1316507685, "eta_seconds": 26565.19883737564, "data_wait_seconds": 2.7477741241455078e-05, "mix": {"fineweb_hq": 0.5093, "dclm": 0.20814, "cosmopedia_v2": 0.2, "finemath": 0.08256}}
{"step": 870, "total_steps": 72479, "tokens": 456130560, "loss": 6.352102637290955, "l_mix": 3.8744071424007416, "l_a": 4.258363783359528, "l_c": 3.996204197406769, "z_loss": 13.25143027305603, "lr": 0.0011434285714285713, "wd": 0.1, "grad_norm": 0.4359203577041626, "tokens_per_second": 1386605.6168674533, "eta_seconds": 27076.004117751123, "data_wait_seconds": 3.87728214263916e-05, "mix": {"fineweb_hq": 0.5092, "dclm": 0.20805, "cosmopedia_v2": 0.2, "finemath": 0.08276}}
{"step": 880, "total_steps": 72479, "tokens": 461373440, "loss": 5.381496012210846, "l_mix": 3.2230261266231537, "l_a": 3.8457659482955933, "l_c": 3.344182014465332, "z_loss": 14.854621648788452, "lr": 0.0011565714285714287, "wd": 0.1, "grad_norm": 0.6548170447349548, "tokens_per_second": 1422257.9367504849, "eta_seconds": 26393.592569971082, "data_wait_seconds": 2.6464462280273438e-05, "mix": {"fineweb_hq": 0.50909, "dclm": 0.20795, "cosmopedia_v2": 0.2, "finemath": 0.08295}}
{"step": 890, "total_steps": 72479, "tokens": 466616320, "loss": 6.337773561477661, "l_mix": 3.8609989285469055, "l_a": 4.2672794461250305, "l_c": 3.984454393386841, "z_loss": 12.543791055679321, "lr": 0.0011697142857142859, "wd": 0.1, "grad_norm": 0.3653961420059204, "tokens_per_second": 1397932.8203133065, "eta_seconds": 26849.111120796206, "data_wait_seconds": 2.2917985916137695e-05, "mix": {"fineweb_hq": 0.51011, "dclm": 0.20787, "cosmopedia_v2": 0.2, "finemath": 0.08202}}
{"step": 900, "total_steps": 72479, "tokens": 471859200, "loss": 6.2442240715026855, "l_mix": 3.7994192838668823, "l_a": 4.225140810012817, "l_c": 3.919976770877838, "z_loss": 12.693126559257507, "lr": 0.0011828571428571428, "wd": 0.1, "grad_norm": 0.4645359218120575, "tokens_per_second": 1419791.5398441656, "eta_seconds": 26432.05688922405, "data_wait_seconds": 1.7315149307250977e-05, "mix": {"fineweb_hq": 0.51, "dclm": 0.20778, "cosmopedia_v2": 0.2, "finemath": 0.08222}, "gates": {"gate_a2b/L1_absmean": 0.01116289, "gate_a2b/L1_absmax": 0.05001055, "gate_a2b/L1_rms": 0.01492365, "gate_a2b/L3_absmean": 0.05081557, "gate_a2b/L3_absmax": 0.1178806, "gate_a2b/L3_rms": 0.05449674, "gate_a2b/L5_absmean": 0.0659221, "gate_a2b/L5_absmax": 0.1877325, "gate_a2b/L5_rms": 0.07331392, "gate_c2b/L1_absmean": 0.007228, "gate_c2b/L1_absmax": 0.04232685, "gate_c2b/L1_rms": 0.01043346, "gate_c2b/L3_absmean": 0.0823045, "gate_c2b/L3_absmax": 0.15260158, "gate_c2b/L3_rms": 0.0865229, "gate_c2b/L5_absmean": 0.1734073, "gate_c2b/L5_absmax": 0.25536183, "gate_c2b/L5_rms": 0.17711997, "gate_b2a/L7_absmean": 0.01658584, "gate_b2a/L7_absmax": 0.06540465, "gate_b2a/L7_rms": 0.02096788, "gate_b2a/L11_absmean": 0.01572776, "gate_b2a/L11_absmax": 0.05017896, "gate_b2a/L11_rms": 0.01893538, "gate_b2a/L14_absmean": 0.02780063, "gate_b2a/L14_absmax": 0.05516936, "gate_b2a/L14_rms": 0.02906957, "gate_b2c/L3_absmean": 0.01367279, "gate_b2c/L3_absmax": 0.06226667, "gate_b2c/L3_rms": 0.01878617, "gate_b2c/L5_absmean": 0.01730248, "gate_b2c/L5_absmax": 0.04629221, "gate_b2c/L5_rms": 0.01982697, "gate_b2c/L6_absmean": 0.01173986, "gate_b2c/L6_absmax": 0.03433859, "gate_b2c/L6_rms": 0.01391588}, "streams": {"cos_a2b/round0": 0.200342, "cos_c2b/round0": 0.260014, "cos_b2a/round0": 0.143468, "cos_b2c/round0": 0.127961, "rms_a/round0": 0.328006, "rms_b/round0": 0.726431, "rms_c/round0": 0.505545, "cos_a2b/round1": 0.377517, "cos_c2b/round1": 0.346566, "cos_b2a/round1": 0.118371, "cos_b2c/round1": 0.128287, "rms_a/round1": 0.429266, "rms_b/round1": 1.193278, "rms_c/round1": 0.687852, "cos_a2b/round2": 0.113917, "cos_c2b/round2": 0.600706, "cos_b2a/round2": 0.178126, "cos_b2c/round2": 0.115408, "rms_a/round2": 0.552606, "rms_b/round2": 2.043603, "rms_c/round2": 0.855244}, "alpha_mean": 0.4182490709692992, "alpha_hist": [0.10492, 0.10329, 0.11813, 0.13586, 0.15034, 0.14635, 0.11899, 0.08542, 0.03444, 0.00226], "alpha_by_type": {"special": {"share": 0.00114, "mean": 0.59957, "std": 0.08876}, "whitespace": {"share": 0.02834, "mean": 0.5415, "std": 0.15519}, "digit": {"share": 0.04546, "mean": 0.61544, "std": 0.1783}, "word_start": {"share": 0.55927, "mean": 0.45659, "std": 0.19124}, "word_cont": {"share": 0.26421, "mean": 0.23257, "std": 0.19358}, "punct": {"share": 0.10159, "mean": 0.56539, "std": 0.19816}}, "alpha_p05": 0.06562, "alpha_p50": 0.43326, "alpha_p95": 0.77965}
{"step": 910, "total_steps": 72479, "tokens": 477102080, "loss": 6.2102115750312805, "l_mix": 3.7713714838027954, "l_a": 4.232681334018707, "l_c": 3.8925886154174805, "z_loss": 12.589778304100037, "lr": 0.001196, "wd": 0.1, "grad_norm": 0.4648093283176422, "tokens_per_second": 1414201.5171167057, "eta_seconds": 26532.82959878445, "data_wait_seconds": 2.2977590560913086e-05, "mix": {"fineweb_hq": 0.50989, "dclm": 0.20769, "cosmopedia_v2": 0.2, "finemath": 0.08242}}
{"step": 920, "total_steps": 72479, "tokens": 482344960, "loss": 6.265037775039673, "l_mix": 3.8116839826107025, "l_a": 4.230927646160126, "l_c": 3.942580372095108, "z_loss": 13.011967301368713, "lr": 0.0012091428571428572, "wd": 0.1, "grad_norm": 0.4226607084274292, "tokens_per_second": 1418940.72616992, "eta_seconds": 26440.516013145447, "data_wait_seconds": 1.5169382095336914e-05, "mix": {"fineweb_hq": 0.50978, "dclm": 0.20761, "cosmopedia_v2": 0.2, "finemath": 0.08261}}
{"step": 930, "total_steps": 72479, "tokens": 487587840, "loss": 6.234752118587494, "l_mix": 3.790621429681778, "l_a": 4.214082211256027, "l_c": 3.9289447963237762, "z_loss": 12.224709868431091, "lr": 0.0012222857142857142, "wd": 0.1, "grad_norm": 0.6113654971122742, "tokens_per_second": 1463598.1274608106, "eta_seconds": 25630.179082751274, "data_wait_seconds": 1.531839370727539e-05, "mix": {"fineweb_hq": 0.50968, "dclm": 0.20753, "cosmopedia_v2": 0.2, "finemath": 0.0828}}
{"step": 940, "total_steps": 72479, "tokens": 492830720, "loss": 5.372154712677002, "l_mix": 3.2554340958595276, "l_a": 3.672394961118698, "l_c": 3.379379987716675, "z_loss": 11.881154179573059, "lr": 0.0012354285714285713, "wd": 0.1, "grad_norm": 0.5128132104873657, "tokens_per_second": 1474146.700347557, "eta_seconds": 25443.220286798478, "data_wait_seconds": 2.09808349609375e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20745, "cosmopedia_v2": 0.20106, "finemath": 0.08191}}
{"step": 950, "total_steps": 72479, "tokens": 498073600, "loss": 5.408142507076263, "l_mix": 3.2811094522476196, "l_a": 3.6786636114120483, "l_c": 3.4075613617897034, "z_loss": 11.655279397964478, "lr": 0.0012485714285714285, "wd": 0.1, "grad_norm": 0.572892963886261, "tokens_per_second": 1442142.9424323482, "eta_seconds": 26004.215843367576, "data_wait_seconds": 2.4437904357910156e-05, "mix": {"fineweb_hq": 0.50947, "dclm": 0.20737, "cosmopedia_v2": 0.20105, "finemath": 0.08211}}
{"step": 960, "total_steps": 72479, "tokens": 503316480, "loss": 5.375696837902069, "l_mix": 3.2648712396621704, "l_a": 3.652023822069168, "l_c": 3.380322754383087, "z_loss": 11.216099381446838, "lr": 0.0012617142857142857, "wd": 0.1, "grad_norm": 0.5016360878944397, "tokens_per_second": 1435443.991742792, "eta_seconds": 26121.92024745941, "data_wait_seconds": 2.8848648071289062e-05, "mix": {"fineweb_hq": 0.50938, "dclm": 0.20729, "cosmopedia_v2": 0.20104, "finemath": 0.08229}}
{"step": 970, "total_steps": 72479, "tokens": 508559360, "loss": 6.254091501235962, "l_mix": 3.8160582184791565, "l_a": 4.172868728637695, "l_c": 3.9492015540599823, "z_loss": 14.121851801872253, "lr": 0.0012748571428571427, "wd": 0.1, "grad_norm": 0.3801896274089813, "tokens_per_second": 1425823.8117657555, "eta_seconds": 26294.490443086623, "data_wait_seconds": 1.806020736694336e-05, "mix": {"fineweb_hq": 0.50928, "dclm": 0.20825, "cosmopedia_v2": 0.2, "finemath": 0.08247}}
{"step": 980, "total_steps": 72479, "tokens": 513802240, "loss": 6.30641770362854, "l_mix": 3.8580631613731384, "l_a": 4.14331328868866, "l_c": 4.013183206319809, "z_loss": 14.05527663230896, "lr": 0.001288, "wd": 0.1, "grad_norm": 0.5944151878356934, "tokens_per_second": 1468828.5061343815, "eta_seconds": 25521.064954447746, "data_wait_seconds": 1.6301870346069336e-05, "mix": {"fineweb_hq": 0.50918, "dclm": 0.20816, "cosmopedia_v2": 0.2, "finemath": 0.08265}}
{"step": 990, "total_steps": 72479, "tokens": 519045120, "loss": 6.247102677822113, "l_mix": 3.808476209640503, "l_a": 4.1754617393016815, "l_c": 3.948407828807831, "z_loss": 14.656912207603455, "lr": 0.0013011428571428573, "wd": 0.1, "grad_norm": 0.6503790020942688, "tokens_per_second": 1349691.529423087, "eta_seconds": 27769.919285202028, "data_wait_seconds": 1.6927719116210938e-05, "mix": {"fineweb_hq": 0.50909, "dclm": 0.20808, "cosmopedia_v2": 0.2, "finemath": 0.08283}}
{"step": 1000, "total_steps": 72479, "tokens": 524288000, "loss": 6.087492823600769, "l_mix": 3.71190482378006, "l_a": 4.08178973197937, "l_c": 3.8323563039302826, "z_loss": 13.44077467918396, "lr": 0.0013142857142857142, "wd": 0.1, "grad_norm": 0.3505009710788727, "tokens_per_second": 1399730.9639569588, "eta_seconds": 26773.41783313751, "data_wait_seconds": 1.8924474716186523e-05, "mix": {"fineweb_hq": 0.51, "dclm": 0.208, "cosmopedia_v2": 0.2, "finemath": 0.082}, "gates": {"gate_a2b/L1_absmean": 0.01245556, "gate_a2b/L1_absmax": 0.05871973, "gate_a2b/L1_rms": 0.01712995, "gate_a2b/L3_absmean": 0.06105513, "gate_a2b/L3_absmax": 0.13791254, "gate_a2b/L3_rms": 0.065444, "gate_a2b/L5_absmean": 0.07947979, "gate_a2b/L5_absmax": 0.20783219, "gate_a2b/L5_rms": 0.08864237, "gate_c2b/L1_absmean": 0.00718221, "gate_c2b/L1_absmax": 0.04812836, "gate_c2b/L1_rms": 0.0109469, "gate_c2b/L3_absmean": 0.09865765, "gate_c2b/L3_absmax": 0.18104616, "gate_c2b/L3_rms": 0.10377927, "gate_c2b/L5_absmean": 0.21601105, "gate_c2b/L5_absmax": 0.33075106, "gate_c2b/L5_rms": 0.21900757, "gate_b2a/L7_absmean": 0.01726876, "gate_b2a/L7_absmax": 0.07349096, "gate_b2a/L7_rms": 0.02252133, "gate_b2a/L11_absmean": 0.0182777, "gate_b2a/L11_absmax": 0.05655135, "gate_b2a/L11_rms": 0.02214494, "gate_b2a/L14_absmean": 0.0408146, "gate_b2a/L14_absmax": 0.08067977, "gate_b2a/L14_rms": 0.04192536, "gate_b2c/L3_absmean": 0.01526827, "gate_b2c/L3_absmax": 0.07270216, "gate_b2c/L3_rms": 0.02130216, "gate_b2c/L5_absmean": 0.02057379, "gate_b2c/L5_absmax": 0.05653514, "gate_b2c/L5_rms": 0.02359712, "gate_b2c/L6_absmean": 0.01182655, "gate_b2c/L6_absmax": 0.03632318, "gate_b2c/L6_rms": 0.01451038}, "streams": {"cos_a2b/round0": 0.216765, "cos_c2b/round0": 0.281722, "cos_b2a/round0": 0.141055, "cos_b2c/round0": 0.135664, "rms_a/round0": 0.36958, "rms_b/round0": 0.758342, "rms_c/round0": 0.554806, "cos_a2b/round1": 0.362862, "cos_c2b/round1": 0.361598, "cos_b2a/round1": 0.120492, "cos_b2c/round1": 0.138448, "rms_a/round1": 0.487702, "rms_b/round1": 1.135962, "rms_c/round1": 0.767496, "cos_a2b/round2": 0.075841, "cos_c2b/round2": 0.637299, "cos_b2a/round2": 0.188481, "cos_b2c/round2": 0.065684, "rms_a/round2": 0.641, "rms_b/round2": 2.570906, "rms_c/round2": 0.972012}, "alpha_mean": 0.4436682818914477, "alpha_hist": [0.09814, 0.09061, 0.10899, 0.1261, 0.14067, 0.14091, 0.1272, 0.11311, 0.05084, 0.00342], "alpha_by_type": {"special": {"share": 0.0014, "mean": 0.69724, "std": 0.11811}, "whitespace": {"share": 0.02556, "mean": 0.60862, "std": 0.19309}, "digit": {"share": 0.02762, "mean": 0.34168, "std": 0.20347}, "word_start": {"share": 0.582, "mean": 0.50422, "std": 0.1927}, "word_cont": {"share": 0.25418, "mean": 0.24093, "std": 0.2013}, "punct": {"share": 0.10924, "mean": 0.57674, "std": 0.22326}}, "alpha_p05": 0.04796, "alpha_p50": 0.46247, "alpha_p95": 0.80069}
{"step": 1010, "total_steps": 72479, "tokens": 529530880, "loss": 6.080958664417267, "l_mix": 3.711884379386902, "l_a": 4.051787495613098, "l_c": 3.840531975030899, "z_loss": 13.783693432807922, "lr": 0.0013274285714285714, "wd": 0.1, "grad_norm": 0.44850969314575195, "tokens_per_second": 1433319.386298107, "eta_seconds": 26142.351404857636, "data_wait_seconds": 1.8537044525146484e-05, "mix": {"fineweb_hq": 0.5099, "dclm": 0.20792, "cosmopedia_v2": 0.2, "finemath": 0.08218}}
{"step": 1020, "total_steps": 72479, "tokens": 534773760, "loss": 6.088631570339203, "l_mix": 3.717721849679947, "l_a": 4.041320621967316, "l_c": 3.8570267260074615, "z_loss": 14.054298043251038, "lr": 0.0013405714285714286, "wd": 0.1, "grad_norm": 0.39400458335876465, "tokens_per_second": 1367073.3851729871, "eta_seconds": 27405.329222512242, "data_wait_seconds": 3.0428171157836914e-05, "mix": {"fineweb_hq": 0.5098, "dclm": 0.20784, "cosmopedia_v2": 0.2, "finemath": 0.08235}}
{"step": 1030, "total_steps": 72479, "tokens": 540016640, "loss": 6.244139075279236, "l_mix": 3.821575105190277, "l_a": 4.113885700702667, "l_c": 3.9566533863544464, "z_loss": 14.020394325256348, "lr": 0.0013537142857142856, "wd": 0.1, "grad_norm": 0.4255017638206482, "tokens_per_second": 1385931.660206932, "eta_seconds": 27028.643898940085, "data_wait_seconds": 1.8656253814697266e-05, "mix": {"fineweb_hq": 0.50971, "dclm": 0.20777, "cosmopedia_v2": 0.2, "finemath": 0.08252}}
{"step": 1040, "total_steps": 72479, "tokens": 545259520, "loss": 6.094502747058868, "l_mix": 3.7273691296577454, "l_a": 4.0190048813819885, "l_c": 3.866843521595001, "z_loss": 13.79008936882019, "lr": 0.0013668571428571427, "wd": 0.1, "grad_norm": 0.41363203525543213, "tokens_per_second": 1448132.8013638514, "eta_seconds": 25864.071580123902, "data_wait_seconds": 3.069639205932617e-05, "mix": {"fineweb_hq": 0.50962, "dclm": 0.20769, "cosmopedia_v2": 0.2, "finemath": 0.08269}}
{"step": 1050, "total_steps": 72479, "tokens": 550502400, "loss": 4.5695845782756805, "l_mix": 2.745070159435272, "l_a": 3.2404484152793884, "l_c": 2.835627019405365, "z_loss": 16.91746425628662, "lr": 0.00138, "wd": 0.1, "grad_norm": 1.1335121393203735, "tokens_per_second": 1379976.0226537078, "eta_seconds": 27137.694378185275, "data_wait_seconds": 3.135204315185547e-05, "mix": {"fineweb_hq": 0.50952, "dclm": 0.20762, "cosmopedia_v2": 0.2, "finemath": 0.08286}}
{"step": 1060, "total_steps": 72479, "tokens": 555745280, "loss": 5.169286370277405, "l_mix": 3.145343631505966, "l_a": 3.4700282216072083, "l_c": 3.272274911403656, "z_loss": 12.516805171966553, "lr": 0.0013931428571428573, "wd": 0.1, "grad_norm": 0.4431310296058655, "tokens_per_second": 1354967.3480673435, "eta_seconds": 27634.706272006035, "data_wait_seconds": 1.9550323486328125e-05, "mix": {"fineweb_hq": 0.50943, "dclm": 0.20755, "cosmopedia_v2": 0.20094, "finemath": 0.08208}}
{"step": 1070, "total_steps": 72479, "tokens": 560988160, "loss": 5.1868321895599365, "l_mix": 3.1558437049388885, "l_a": 3.4864489138126373, "l_c": 3.2793784737586975, "z_loss": 12.402408123016357, "lr": 0.0014062857142857143, "wd": 0.1, "grad_norm": 0.4985334277153015, "tokens_per_second": 1396177.5840072248, "eta_seconds": 26815.272083473206, "data_wait_seconds": 2.6851892471313477e-05, "mix": {"fineweb_hq": 0.50935, "dclm": 0.20748, "cosmopedia_v2": 0.20093, "finemath": 0.08224}}
{"step": 1080, "total_steps": 72479, "tokens": 566231040, "loss": 5.1416760087013245, "l_mix": 3.129388242959976, "l_a": 3.434536933898926, "l_c": 3.268891215324402, "z_loss": 12.59196925163269, "lr": 0.0014194285714285715, "wd": 0.1, "grad_norm": 0.47517502307891846, "tokens_per_second": 1348837.4116997742, "eta_seconds": 27752.521235918997, "data_wait_seconds": 1.761317253112793e-05, "mix": {"fineweb_hq": 0.50926, "dclm": 0.20741, "cosmopedia_v2": 0.20093, "finemath": 0.08241}}
{"step": 1090, "total_steps": 72479, "tokens": 571473920, "loss": 5.170536279678345, "l_mix": 3.1470480263233185, "l_a": 3.4530737102031708, "l_c": 3.2874774038791656, "z_loss": 13.227647066116333, "lr": 0.0014325714285714287, "wd": 0.1, "grad_norm": 0.5198989510536194, "tokens_per_second": 1388860.1554837448, "eta_seconds": 26949.002665400505, "data_wait_seconds": 2.47955322265625e-05, "mix": {"fineweb_hq": 0.50917, "dclm": 0.20734, "cosmopedia_v2": 0.20092, "finemath": 0.08257}}
{"step": 1100, "total_steps": 72479, "tokens": 576716800, "loss": 5.938996911048889, "l_mix": 3.6350124776363373, "l_a": 3.893671452999115, "l_c": 3.7815820574760437, "z_loss": 14.081932306289673, "lr": 0.0014457142857142856, "wd": 0.1, "grad_norm": 0.40300071239471436, "tokens_per_second": 1467112.4147500948, "eta_seconds": 25508.033860087395, "data_wait_seconds": 2.440810203552246e-05, "mix": {"fineweb_hq": 0.51, "dclm": 0.20727, "cosmopedia_v2": 0.2, "finemath": 0.08273}, "gates": {"gate_a2b/L1_absmean": 0.01475668, "gate_a2b/L1_absmax": 0.06676084, "gate_a2b/L1_rms": 0.02045253, "gate_a2b/L3_absmean": 0.0719962, "gate_a2b/L3_absmax": 0.1623376, "gate_a2b/L3_rms": 0.07704837, "gate_a2b/L5_absmean": 0.1023422, "gate_a2b/L5_absmax": 0.26587099, "gate_a2b/L5_rms": 0.11338127, "gate_c2b/L1_absmean": 0.00889932, "gate_c2b/L1_absmax": 0.0542606, "gate_c2b/L1_rms": 0.01293515, "gate_c2b/L3_absmean": 0.11559039, "gate_c2b/L3_absmax": 0.22233015, "gate_c2b/L3_rms": 0.12125662, "gate_c2b/L5_absmean": 0.2464484, "gate_c2b/L5_absmax": 0.38759336, "gate_c2b/L5_rms": 0.25057006, "gate_b2a/L7_absmean": 0.01855446, "gate_b2a/L7_absmax": 0.08580059, "gate_b2a/L7_rms": 0.02437124, "gate_b2a/L11_absmean": 0.02242209, "gate_b2a/L11_absmax": 0.06915791, "gate_b2a/L11_rms": 0.0270466, "gate_b2a/L14_absmean": 0.05427494, "gate_b2a/L14_absmax": 0.09158218, "gate_b2a/L14_rms": 0.05525628, "gate_b2c/L3_absmean": 0.0174644, "gate_b2c/L3_absmax": 0.08336708, "gate_b2c/L3_rms": 0.02436648, "gate_b2c/L5_absmean": 0.02430725, "gate_b2c/L5_absmax": 0.06237812, "gate_b2c/L5_rms": 0.02760912, "gate_b2c/L6_absmean": 0.01368372, "gate_b2c/L6_absmax": 0.04326123, "gate_b2c/L6_rms": 0.01676803}, "streams": {"cos_a2b/round0": 0.229824, "cos_c2b/round0": 0.300422, "cos_b2a/round0": 0.138655, "cos_b2c/round0": 0.138997, "rms_a/round0": 0.416589, "rms_b/round0": 0.841934, "rms_c/round0": 0.607188, "cos_a2b/round1": 0.374397, "cos_c2b/round1": 0.379755, "cos_b2a/round1": 0.128248, "cos_b2c/round1": 0.141089, "rms_a/round1": 0.555322, "rms_b/round1": 1.165195, "rms_c/round1": 0.851564, "cos_a2b/round2": 0.015094, "cos_c2b/round2": 0.658777, "cos_b2a/round2": 0.188039, "cos_b2c/round2": 0.011823, "rms_a/round2": 0.738767, "rms_b/round2": 3.106222, "rms_c/round2": 1.105034}, "alpha_mean": 0.4081105236789071, "alpha_hist": [0.10699, 0.10316, 0.12253, 0.14371, 0.15627, 0.14824, 0.11803, 0.07307, 0.02634, 0.00166], "alpha_by_type": {"special": {"share": 0.00085, "mean": 0.61969, "std": 0.11077}, "whitespace": {"share": 0.02052, "mean": 0.52327, "std": 0.14984}, "digit": {"share": 0.02174, "mean": 0.40483, "std": 0.19709}, "word_start": {"share": 0.58427, "mean": 0.46071, "std": 0.18602}, "word_cont": {"share": 0.2533, "mean": 0.23158, "std": 0.1917}, "punct": {"share": 0.11931, "mean": 0.50455, "std": 0.23355}}, "alpha_p05": 0.03832, "alpha_p50": 0.40592, "alpha_p95": 0.74651}
{"step": 1110, "total_steps": 72479, "tokens": 581959680, "loss": 6.031332850456238, "l_mix": 3.6948876678943634, "l_a": 3.9465320706367493, "l_c": 3.8366349935531616, "z_loss": 14.950928449630737, "lr": 0.0014588571428571428, "wd": 0.1, "grad_norm": 0.4173354506492615, "tokens_per_second": 1436178.2307411777, "eta_seconds": 26053.806882095338, "data_wait_seconds": 3.135204315185547e-05, "mix": {"fineweb_hq": 0.50991, "dclm": 0.20721, "cosmopedia_v2": 0.2, "finemath": 0.08288}}
{"step": 1120, "total_steps": 72479, "tokens": 587202560, "loss": 5.961114346981049, "l_mix": 3.650132119655609, "l_a": 3.9026466012001038, "l_c": 3.7959351241588593, "z_loss": 14.076560616493225, "lr": 0.001472, "wd": 0.1, "grad_norm": 0.3329419195652008, "tokens_per_second": 1429791.7302163034, "eta_seconds": 26166.515445113182, "data_wait_seconds": 1.5497207641601562e-05, "mix": {"fineweb_hq": 0.50982, "dclm": 0.20804, "cosmopedia_v2": 0.2, "finemath": 0.08214}}
{"step": 1130, "total_steps": 72479, "tokens": 592445440, "loss": 5.916339218616486, "l_mix": 3.622564971446991, "l_a": 3.880922019481659, "l_c": 3.760087698698044, "z_loss": 14.713804006576538, "lr": 0.001485142857142857, "wd": 0.1, "grad_norm": 0.32683029770851135, "tokens_per_second": 1320741.9858127744, "eta_seconds": 28323.037287998202, "data_wait_seconds": 2.3156404495239258e-05, "mix": {"fineweb_hq": 0.50973, "dclm": 0.20796, "cosmopedia_v2": 0.2, "finemath": 0.0823}}
{"step": 1140, "total_steps": 72479, "tokens": 597688320, "loss": 5.878834068775177, "l_mix": 3.599827080965042, "l_a": 3.857087790966034, "l_c": 3.734703153371811, "z_loss": 14.695742011070251, "lr": 0.0014982857142857142, "wd": 0.1, "grad_norm": 0.39818060398101807, "tokens_per_second": 1274577.3800855041, "eta_seconds": 29344.771228790283, "data_wait_seconds": 1.9103288650512695e-05, "mix": {"fineweb_hq": 0.50965, "dclm": 0.20789, "cosmopedia_v2": 0.2, "finemath": 0.08246}}
{"step": 1150, "total_steps": 72479, "tokens": 602931200, "loss": 5.877926051616669, "l_mix": 3.6029000878334045, "l_a": 3.843712657690048, "l_c": 3.7348760962486267, "z_loss": 14.491909623146057, "lr": 0.0015114285714285713, "wd": 0.1, "grad_norm": 0.3440513610839844, "tokens_per_second": 1404521.794161492, "eta_seconds": 26626.100718021393, "data_wait_seconds": 3.713369369506836e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20783, "cosmopedia_v2": 0.2, "finemath": 0.08261}}
{"step": 1160, "total_steps": 72479, "tokens": 608174080, "loss": 4.446840345859528, "l_mix": 2.7035052478313446, "l_a": 3.005829483270645, "l_c": 2.7997858822345734, "z_loss": 16.504907369613647, "lr": 0.0015245714285714287, "wd": 0.1, "grad_norm": 0.6227682828903198, "tokens_per_second": 1345816.9987019338, "eta_seconds": 27783.640649557114, "data_wait_seconds": 2.4557113647460938e-05, "mix": {"fineweb_hq": 0.50948, "dclm": 0.20776, "cosmopedia_v2": 0.2, "finemath": 0.08276}}
{"step": 1170, "total_steps": 72479, "tokens": 613416960, "loss": 5.03556215763092, "l_mix": 3.0685075521469116, "l_a": 3.339915841817856, "l_c": 3.2125303149223328, "z_loss": 13.2063969373703, "lr": 0.0015377142857142857, "wd": 0.1, "grad_norm": 0.430227667093277, "tokens_per_second": 1460793.2651645876, "eta_seconds": 25593.253941917417, "data_wait_seconds": 1.7255544662475586e-05, "mix": {"fineweb_hq": 0.5094, "dclm": 0.20769, "cosmopedia_v2": 0.20085, "finemath": 0.08205}}
{"step": 1180, "total_steps": 72479, "tokens": 618659840, "loss": 4.983793139457703, "l_mix": 3.0358621180057526, "l_a": 3.3047618865966797, "l_c": 3.1840836107730865, "z_loss": 12.77125370502472, "lr": 0.0015508571428571429, "wd": 0.1, "grad_norm": 0.38061028718948364, "tokens_per_second": 1297167.4536707003, "eta_seconds": 28817.567081427573, "data_wait_seconds": 2.154707908630371e-05, "mix": {"fineweb_hq": 0.50932, "dclm": 0.20763, "cosmopedia_v2": 0.20085, "finemath": 0.0822}}
{"step": 1190, "total_steps": 72479, "tokens": 623902720, "loss": 5.031120538711548, "l_mix": 3.065935254096985, "l_a": 3.3377332985401154, "l_c": 3.2083948254585266, "z_loss": 13.468223333358765, "lr": 0.001564, "wd": 0.1, "grad_norm": 0.4714663624763489, "tokens_per_second": 1410967.7266213815, "eta_seconds": 26489.597548413276, "data_wait_seconds": 1.58846378326416e-05, "mix": {"fineweb_hq": 0.50924, "dclm": 0.20756, "cosmopedia_v2": 0.20084, "finemath": 0.08235}}
{"step": 1200, "total_steps": 72479, "tokens": 629145600, "loss": 5.026270270347595, "l_mix": 3.066525399684906, "l_a": 3.3245679140090942, "l_c": 3.203511565923691, "z_loss": 13.210240602493286, "lr": 0.001577142857142857, "wd": 0.1, "grad_norm": 0.4940112233161926, "tokens_per_second": 1404719.626326812, "eta_seconds": 26603.68920004368, "data_wait_seconds": 1.5437602996826172e-05, "mix": {"fineweb_hq": 0.50917, "dclm": 0.2075, "cosmopedia_v2": 0.20083, "finemath": 0.0825}, "gates": {"gate_a2b/L1_absmean": 0.01885112, "gate_a2b/L1_absmax": 0.07409813, "gate_a2b/L1_rms": 0.0248959, "gate_a2b/L3_absmean": 0.08491232, "gate_a2b/L3_absmax": 0.18655416, "gate_a2b/L3_rms": 0.09033721, "gate_a2b/L5_absmean": 0.13282859, "gate_a2b/L5_absmax": 0.35320762, "gate_a2b/L5_rms": 0.14625855, "gate_c2b/L1_absmean": 0.01088899, "gate_c2b/L1_absmax": 0.07443252, "gate_c2b/L1_rms": 0.01511824, "gate_c2b/L3_absmean": 0.13287416, "gate_c2b/L3_absmax": 0.26302233, "gate_c2b/L3_rms": 0.13911033, "gate_c2b/L5_absmean": 0.27060884, "gate_c2b/L5_absmax": 0.4232637, "gate_c2b/L5_rms": 0.27607343, "gate_b2a/L7_absmean": 0.02010588, "gate_b2a/L7_absmax": 0.09058128, "gate_b2a/L7_rms": 0.0264639, "gate_b2a/L11_absmean": 0.02470524, "gate_b2a/L11_absmax": 0.07399223, "gate_b2a/L11_rms": 0.02942195, "gate_b2a/L14_absmean": 0.06642839, "gate_b2a/L14_absmax": 0.10830078, "gate_b2a/L14_rms": 0.06744806, "gate_b2c/L3_absmean": 0.01792487, "gate_b2c/L3_absmax": 0.0877085, "gate_b2c/L3_rms": 0.02521176, "gate_b2c/L5_absmean": 0.02683558, "gate_b2c/L5_absmax": 0.07051279, "gate_b2c/L5_rms": 0.03040267, "gate_b2c/L6_absmean": 0.01791581, "gate_b2c/L6_absmax": 0.05252454, "gate_b2c/L6_rms": 0.02168585}, "streams": {"cos_a2b/round0": 0.240799, "cos_c2b/round0": 0.299671, "cos_b2a/round0": 0.129304, "cos_b2c/round0": 0.120287, "rms_a/round0": 0.464338, "rms_b/round0": 0.940116, "rms_c/round0": 0.655338, "cos_a2b/round1": 0.351255, "cos_c2b/round1": 0.350284, "cos_b2a/round1": 0.140787, "cos_b2c/round1": 0.145613, "rms_a/round1": 0.625903, "rms_b/round1": 1.278144, "rms_c/round1": 0.933877, "cos_a2b/round2": 0.055909, "cos_c2b/round2": 0.635412, "cos_b2a/round2": 0.200394, "cos_b2c/round2": 0.010573, "rms_a/round2": 0.854561, "rms_b/round2": 2.815666, "rms_c/round2": 1.194363}, "alpha_mean": 0.41873330473987735, "alpha_hist": [0.10909, 0.14597, 0.13513, 0.12419, 0.11307, 0.09715, 0.09102, 0.08488, 0.07402, 0.02548], "alpha_by_type": {"special": {"share": 0.00105, "mean": 0.62162, "std": 0.09441}, "whitespace": {"share": 0.03485, "mean": 0.52955, "std": 0.18992}, "digit": {"share": 0.00922, "mean": 0.43492, "std": 0.25213}, "word_start": {"share": 0.56766, "mean": 0.45928, "std": 0.23507}, "word_cont": {"share": 0.26994, "mean": 0.22674, "std": 0.18824}, "punct": {"share": 0.11728, "mean": 0.62836, "std": 0.23826}}, "alpha_p05": 0.05321, "alpha_p50": 0.413, "alpha_p95": 0.85537}
{"step": 1210, "total_steps": 72479, "tokens": 634388480, "loss": 5.869579017162323, "l_mix": 3.5997613072395325, "l_a": 3.8194886445999146, "l_c": 3.741740494966507, "z_loss": 14.488547444343567, "lr": 0.0015902857142857142, "wd": 0.1, "grad_norm": 0.3809581398963928, "tokens_per_second": 1473392.5839166595, "eta_seconds": 25360.166652035714, "data_wait_seconds": 1.576542854309082e-05, "mix": {"fineweb_hq": 0.50992, "dclm": 0.20744, "cosmopedia_v2": 0.2, "finemath": 0.08264}}
{"step": 1220, "total_steps": 72479, "tokens": 639631360, "loss": 5.864354193210602, "l_mix": 3.5962474942207336, "l_a": 3.8061127364635468, "l_c": 3.7489747405052185, "z_loss": 15.80320405960083, "lr": 0.0016034285714285714, "wd": 0.1, "grad_norm": 0.4266788363456726, "tokens_per_second": 1466468.2513327948, "eta_seconds": 25476.336468958856, "data_wait_seconds": 1.570582389831543e-05, "mix": {"fineweb_hq": 0.50984, "dclm": 0.20738, "cosmopedia_v2": 0.2, "finemath": 0.08279}}
{"step": 1230, "total_steps": 72479, "tokens": 644874240, "loss": 5.83337128162384, "l_mix": 3.5785616040229797, "l_a": 3.791440635919571, "l_c": 3.7196032404899597, "z_loss": 14.963855981826782, "lr": 0.0016165714285714284, "wd": 0.1, "grad_norm": 0.3175868093967438, "tokens_per_second": 1474662.8282151315, "eta_seconds": 25331.211309647562, "data_wait_seconds": 2.637505531311035e-05, "mix": {"fineweb_hq": 0.50976, "dclm": 0.20732, "cosmopedia_v2": 0.20081, "finemath": 0.08211}}
{"step": 1240, "total_steps": 72479, "tokens": 650117120, "loss": 4.989529132843018, "l_mix": 3.0449275374412537, "l_a": 3.282997488975525, "l_c": 3.194280058145523, "z_loss": 14.181613206863403, "lr": 0.0016297142857142856, "wd": 0.1, "grad_norm": 0.556206464767456, "tokens_per_second": 1448122.0252500025, "eta_seconds": 25791.85467851162, "data_wait_seconds": 1.65402889251709e-05, "mix": {"fineweb_hq": 0.50968, "dclm": 0.20726, "cosmopedia_v2": 0.20081, "finemath": 0.08226}}
{"step": 1250, "total_steps": 72479, "tokens": 655360000, "loss": 5.937387704849243, "l_mix": 3.647969275712967, "l_a": 3.8344940543174744, "l_c": 3.7917048633098602, "z_loss": 15.587069630622864, "lr": 0.001642857142857143, "wd": 0.1, "grad_norm": 0.3464168608188629, "tokens_per_second": 1367655.2005126756, "eta_seconds": 27305.500639343263, "data_wait_seconds": 3.263354301452637e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.208, "cosmopedia_v2": 0.2, "finemath": 0.0824}}
{"step": 1260, "total_steps": 72479, "tokens": 660602880, "loss": 5.903826057910919, "l_mix": 3.626911222934723, "l_a": 3.802415192127228, "l_c": 3.7818334698677063, "z_loss": 16.40093469619751, "lr": 0.0016560000000000001, "wd": 0.1, "grad_norm": 0.4028671681880951, "tokens_per_second": 1458899.7687094454, "eta_seconds": 25594.127761793134, "data_wait_seconds": 2.014636993408203e-05, "mix": {"fineweb_hq": 0.50952, "dclm": 0.20794, "cosmopedia_v2": 0.2, "finemath": 0.08254}}
{"step": 1270, "total_steps": 72479, "tokens": 665845760, "loss": 5.940282225608826, "l_mix": 3.6483230888843536, "l_a": 3.8344300985336304, "l_c": 3.7999468445777893, "z_loss": 16.460004568099976, "lr": 0.001669142857142857, "wd": 0.1, "grad_norm": 0.36802053451538086, "tokens_per_second": 1379209.4407123395, "eta_seconds": 27069.147795796394, "data_wait_seconds": 3.293156623840332e-05, "mix": {"fineweb_hq": 0.50945, "dclm": 0.20787, "cosmopedia_v2": 0.2, "finemath": 0.08268}}
{"step": 1280, "total_steps": 72479, "tokens": 671088640, "loss": 4.885102272033691, "l_mix": 2.981016129255295, "l_a": 3.2250560224056244, "l_c": 3.1173046827316284, "z_loss": 13.778794765472412, "lr": 0.0016822857142857143, "wd": 0.1, "grad_norm": 0.3690691590309143, "tokens_per_second": 1409896.3647703773, "eta_seconds": 26476.25899658203, "data_wait_seconds": 1.633167266845703e-05, "mix": {"fineweb_hq": 0.50938, "dclm": 0.20781, "cosmopedia_v2": 0.20078, "finemath": 0.08203}}
{"step": 1290, "total_steps": 72479, "tokens": 676331520, "loss": 4.994629502296448, "l_mix": 3.0496011674404144, "l_a": 3.2660791873931885, "l_c": 3.2127594649791718, "z_loss": 13.767391443252563, "lr": 0.0016954285714285715, "wd": 0.1, "grad_norm": 0.4944177269935608, "tokens_per_second": 1478641.8233834477, "eta_seconds": 25241.771091389655, "data_wait_seconds": 1.9490718841552734e-05, "mix": {"fineweb_hq": 0.5093, "dclm": 0.20775, "cosmopedia_v2": 0.20078, "finemath": 0.08217}}
{"step": 1300, "total_steps": 72479, "tokens": 681574400, "loss": 4.888859450817108, "l_mix": 2.9829451739788055, "l_a": 3.216620832681656, "l_c": 3.1318823993206024, "z_loss": 13.631524920463562, "lr": 0.0017085714285714284, "wd": 0.1, "grad_norm": 0.39498788118362427, "tokens_per_second": 1450353.0444837504, "eta_seconds": 25730.49072012901, "data_wait_seconds": 1.8924474716186523e-05, "mix": {"fineweb_hq": 0.50923, "dclm": 0.20769, "cosmopedia_v2": 0.20077, "finemath": 0.08231}, "gates": {"gate_a2b/L1_absmean": 0.01932151, "gate_a2b/L1_absmax": 0.07897829, "gate_a2b/L1_rms": 0.02719204, "gate_a2b/L3_absmean": 0.09895001, "gate_a2b/L3_absmax": 0.20990537, "gate_a2b/L3_rms": 0.10499016, "gate_a2b/L5_absmean": 0.15714671, "gate_a2b/L5_absmax": 0.39790446, "gate_a2b/L5_rms": 0.17069486, "gate_c2b/L1_absmean": 0.00881154, "gate_c2b/L1_absmax": 0.08402794, "gate_c2b/L1_rms": 0.01482685, "gate_c2b/L3_absmean": 0.15030903, "gate_c2b/L3_absmax": 0.30974331, "gate_c2b/L3_rms": 0.15737873, "gate_c2b/L5_absmean": 0.29219475, "gate_c2b/L5_absmax": 0.46789342, "gate_c2b/L5_rms": 0.2990326, "gate_b2a/L7_absmean": 0.02141641, "gate_b2a/L7_absmax": 0.09210557, "gate_b2a/L7_rms": 0.02824822, "gate_b2a/L11_absmean": 0.02804459, "gate_b2a/L11_absmax": 0.08312479, "gate_b2a/L11_rms": 0.03304514, "gate_b2a/L14_absmean": 0.07803722, "gate_b2a/L14_absmax": 0.11728936, "gate_b2a/L14_rms": 0.07908255, "gate_b2c/L3_absmean": 0.01926842, "gate_b2c/L3_absmax": 0.09593373, "gate_b2c/L3_rms": 0.02761935, "gate_b2c/L5_absmean": 0.02991891, "gate_b2c/L5_absmax": 0.07616935, "gate_b2c/L5_rms": 0.03385821, "gate_b2c/L6_absmean": 0.02176755, "gate_b2c/L6_absmax": 0.06936692, "gate_b2c/L6_rms": 0.02599648}, "streams": {"cos_a2b/round0": 0.245899, "cos_c2b/round0": 0.30579, "cos_b2a/round0": 0.126042, "cos_b2c/round0": 0.115413, "rms_a/round0": 0.524703, "rms_b/round0": 1.012817, "rms_c/round0": 0.71123, "cos_a2b/round1": 0.363625, "cos_c2b/round1": 0.35889, "cos_b2a/round1": 0.144884, "cos_b2c/round1": 0.151349, "rms_a/round1": 0.714604, "rms_b/round1": 1.386955, "rms_c/round1": 1.027725, "cos_a2b/round2": 0.080084, "cos_c2b/round2": 0.627675, "cos_b2a/round2": 0.190256, "cos_b2c/round2": 0.011709, "rms_a/round2": 0.995542, "rms_b/round2": 2.975546, "rms_c/round2": 1.330271}, "alpha_mean": 0.43893480349913716, "alpha_hist": [0.07799, 0.14795, 0.1425, 0.12372, 0.10936, 0.0966, 0.0937, 0.09023, 0.08994, 0.02802], "alpha_by_type": {"special": {"share": 0.00104, "mean": 0.70748, "std": 0.12667}, "whitespace": {"share": 0.03253, "mean": 0.64017, "std": 0.20011}, "digit": {"share": 0.01105, "mean": 0.51262, "std": 0.24888}, "word_start": {"share": 0.56581, "mean": 0.46029, "std": 0.23761}, "word_cont": {"share": 0.26753, "mean": 0.26353, "std": 0.19059}, "punct": {"share": 0.12204, "mean": 0.66185, "std": 0.22384}}, "alpha_p05": 0.08542, "alpha_p50": 0.42465, "alpha_p95": 0.87343}
{"step": 1310, "total_steps": 72479, "tokens": 686817280, "loss": 4.904236912727356, "l_mix": 2.9939560294151306, "l_a": 3.211266368627548, "l_c": 3.1517302095890045, "z_loss": 13.817800045013428, "lr": 0.0017217142857142856, "wd": 0.1, "grad_norm": 0.43660712242126465, "tokens_per_second": 1343682.9364349372, "eta_seconds": 27769.239052033423, "data_wait_seconds": 4.374980926513672e-05, "mix": {"fineweb_hq": 0.50916, "dclm": 0.20763, "cosmopedia_v2": 0.20076, "finemath": 0.08244}}
{"step": 1320, "total_steps": 72479, "tokens": 692060160, "loss": 5.711215078830719, "l_mix": 3.5073467195034027, "l_a": 3.6887383460998535, "l_c": 3.6524685323238373, "z_loss": 15.063009142875671, "lr": 0.0017348571428571428, "wd": 0.1, "grad_norm": 0.3716929852962494, "tokens_per_second": 1472772.1897687416, "eta_seconds": 25331.690842056276, "data_wait_seconds": 2.1070241928100586e-05, "mix": {"fineweb_hq": 0.50985, "dclm": 0.20758, "cosmopedia_v2": 0.2, "finemath": 0.08258}}
{"step": 1330, "total_steps": 72479, "tokens": 697303040, "loss": 5.727864742279053, "l_mix": 3.5153368711471558, "l_a": 3.713371515274048, "l_c": 3.6563976109027863, "z_loss": 15.971121788024902, "lr": 0.001748, "wd": 0.1, "grad_norm": 0.31232181191444397, "tokens_per_second": 1459987.4103489905, "eta_seconds": 25549.92368261814, "data_wait_seconds": 1.5527009963989258e-05, "mix": {"fineweb_hq": 0.50977, "dclm": 0.20752, "cosmopedia_v2": 0.2, "finemath": 0.08271}}
{"step": 1340, "total_steps": 72479, "tokens": 702545920, "loss": 5.856395781040192, "l_mix": 3.5946485102176666, "l_a": 3.788729190826416, "l_c": 3.7453853487968445, "z_loss": 15.127310872077942, "lr": 0.001761142857142857, "wd": 0.1, "grad_norm": 0.4855881929397583, "tokens_per_second": 1346353.5728025495, "eta_seconds": 27702.473395872115, "data_wait_seconds": 1.9788742065429688e-05, "mix": {"fineweb_hq": 0.5097, "dclm": 0.20746, "cosmopedia_v2": 0.20075, "finemath": 0.08209}}
{"step": 1350, "total_steps": 72479, "tokens": 707788800, "loss": 4.870872437953949, "l_mix": 2.973212957382202, "l_a": 3.200259745121002, "l_c": 3.120503157377243, "z_loss": 14.305179476737976, "lr": 0.0017742857142857144, "wd": 0.1, "grad_norm": 0.40044185519218445, "tokens_per_second": 1484616.8622147671, "eta_seconds": 25118.99339225292, "data_wait_seconds": 2.524256706237793e-05, "mix": {"fineweb_hq": 0.50963, "dclm": 0.20741, "cosmopedia_v2": 0.20074, "finemath": 0.08222}}
{"step": 1360, "total_steps": 72479, "tokens": 713031680, "loss": 4.890243113040924, "l_mix": 2.9857206642627716, "l_a": 3.2292498648166656, "l_c": 3.1143039762973785, "z_loss": 14.561643123626709, "lr": 0.0017874285714285715, "wd": 0.1, "grad_norm": 0.5479589700698853, "tokens_per_second": 1351804.5142613254, "eta_seconds": 27583.01061923504, "data_wait_seconds": 1.9371509552001953e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20735, "cosmopedia_v2": 0.20074, "finemath": 0.08235}}
{"step": 1370, "total_steps": 72479, "tokens": 718274560, "loss": 4.904634058475494, "l_mix": 2.9952603578567505, "l_a": 3.1891542077064514, "l_c": 3.1705360114574432, "z_loss": 14.664152979850769, "lr": 0.0018005714285714285, "wd": 0.1, "grad_norm": 0.5977428555488586, "tokens_per_second": 1450696.4394401335, "eta_seconds": 25699.1017406702, "data_wait_seconds": 2.765655517578125e-05, "mix": {"fineweb_hq": 0.50949, "dclm": 0.2073, "cosmopedia_v2": 0.20073, "finemath": 0.08248}}
{"step": 1380, "total_steps": 72479, "tokens": 723517440, "loss": 4.883216679096222, "l_mix": 2.9801122844219208, "l_a": 3.1998632550239563, "l_c": 3.139032483100891, "z_loss": 14.354623317718506, "lr": 0.0018137142857142857, "wd": 0.1, "grad_norm": 0.4525318145751953, "tokens_per_second": 1387589.7628584325, "eta_seconds": 26864.101703381537, "data_wait_seconds": 1.665949821472168e-05, "mix": {"fineweb_hq": 0.50942, "dclm": 0.20725, "cosmopedia_v2": 0.20072, "finemath": 0.08261}}
{"step": 1390, "total_steps": 72479, "tokens": 728760320, "loss": 5.827172756195068, "l_mix": 3.583150953054428, "l_a": 3.746425122022629, "l_c": 3.728091299533844, "z_loss": 16.667330741882324, "lr": 0.0018268571428571429, "wd": 0.1, "grad_norm": 0.33236250281333923, "tokens_per_second": 1473295.1532733338, "eta_seconds": 25297.788803005216, "data_wait_seconds": 1.850724220275879e-05, "mix": {"fineweb_hq": 0.50935, "dclm": 0.20791, "cosmopedia_v2": 0.2, "finemath": 0.08273}}
{"step": 1400, "total_steps": 72479, "tokens": 734003200, "loss": 4.799883186817169, "l_mix": 2.929626852273941, "l_a": 3.1492990255355835, "l_c": 3.080458253622055, "z_loss": 13.290782451629639, "lr": 0.0018399999999999998, "wd": 0.1, "grad_norm": 0.43416285514831543, "tokens_per_second": 1293989.3655982648, "eta_seconds": 28799.20634801388, "data_wait_seconds": 1.665949821472168e-05, "mix": {"fineweb_hq": 0.50929, "dclm": 0.20786, "cosmopedia_v2": 0.20071, "finemath": 0.08214}, "gates": {"gate_a2b/L1_absmean": 0.02198642, "gate_a2b/L1_absmax": 0.1098597, "gate_a2b/L1_rms": 0.03167738, "gate_a2b/L3_absmean": 0.11532038, "gate_a2b/L3_absmax": 0.25914672, "gate_a2b/L3_rms": 0.12241285, "gate_a2b/L5_absmean": 0.18188484, "gate_a2b/L5_absmax": 0.42249209, "gate_a2b/L5_rms": 0.19461635, "gate_c2b/L1_absmean": 0.01009063, "gate_c2b/L1_absmax": 0.10148076, "gate_c2b/L1_rms": 0.01786462, "gate_c2b/L3_absmean": 0.16697231, "gate_c2b/L3_absmax": 0.35111475, "gate_c2b/L3_rms": 0.17481104, "gate_c2b/L5_absmean": 0.31546527, "gate_c2b/L5_absmax": 0.49617723, "gate_c2b/L5_rms": 0.32358816, "gate_b2a/L7_absmean": 0.02279081, "gate_b2a/L7_absmax": 0.09636173, "gate_b2a/L7_rms": 0.0302641, "gate_b2a/L11_absmean": 0.03108911, "gate_b2a/L11_absmax": 0.08881575, "gate_b2a/L11_rms": 0.03609622, "gate_b2a/L14_absmean": 0.09069626, "gate_b2a/L14_absmax": 0.13504925, "gate_b2a/L14_rms": 0.09183763, "gate_b2c/L3_absmean": 0.02028426, "gate_b2c/L3_absmax": 0.11885573, "gate_b2c/L3_rms": 0.02917209, "gate_b2c/L5_absmean": 0.03239271, "gate_b2c/L5_absmax": 0.08552705, "gate_b2c/L5_rms": 0.03627414, "gate_b2c/L6_absmean": 0.02683493, "gate_b2c/L6_absmax": 0.10105796, "gate_b2c/L6_rms": 0.03105335}, "streams": {"cos_a2b/round0": 0.259183, "cos_c2b/round0": 0.311566, "cos_b2a/round0": 0.129938, "cos_b2c/round0": 0.115891, "rms_a/round0": 0.595575, "rms_b/round0": 1.129649, "rms_c/round0": 0.769843, "cos_a2b/round1": 0.349836, "cos_c2b/round1": 0.343506, "cos_b2a/round1": 0.155312, "cos_b2c/round1": 0.157898, "rms_a/round1": 0.828346, "rms_b/round1": 1.550299, "rms_c/round1": 1.132889, "cos_a2b/round2": 0.101245, "cos_c2b/round2": 0.609828, "cos_b2a/round2": 0.185724, "cos_b2c/round2": 0.030814, "rms_a/round2": 1.173142, "rms_b/round2": 3.28042, "rms_c/round2": 1.47551}, "alpha_mean": 0.46183301463990034, "alpha_hist": [0.07219, 0.13577, 0.13177, 0.11902, 0.11176, 0.10321, 0.08559, 0.08478, 0.11101, 0.04491], "alpha_by_type": {"special": {"share": 0.00111, "mean": 0.75915, "std": 0.11593}, "whitespace": {"share": 0.03299, "mean": 0.70167, "std": 0.21204}, "digit": {"share": 0.01074, "mean": 0.47338, "std": 0.25154}, "word_start": {"share": 0.57634, "mean": 0.49006, "std": 0.24425}, "word_cont": {"share": 0.26247, "mean": 0.27845, "std": 0.20095}, "punct": {"share": 0.11635, "mean": 0.66377, "std": 0.24526}}, "alpha_p05": 0.0827, "alpha_p50": 0.43446, "alpha_p95": 0.89404}
{"step": 1410, "total_steps": 72479, "tokens": 739246080, "loss": 4.72502475976944, "l_mix": 2.8832011818885803, "l_a": 3.0847056210041046, "l_c": 3.049962282180786, "z_loss": 14.231585621833801, "lr": 0.001853142857142857, "wd": 0.1, "grad_norm": 0.44243767857551575, "tokens_per_second": 1454832.304763437, "eta_seconds": 25611.628055000307, "data_wait_seconds": 3.287196159362793e-05, "mix": {"fineweb_hq": 0.50922, "dclm": 0.2078, "cosmopedia_v2": 0.20071, "finemath": 0.08227}}
{"step": 1420, "total_steps": 72479, "tokens": 744488960, "loss": 4.826941192150116, "l_mix": 2.945580005645752, "l_a": 3.162296324968338, "l_c": 3.104576826095581, "z_loss": 12.99206268787384, "lr": 0.0018662857142857142, "wd": 0.1, "grad_norm": 0.3811546266078949, "tokens_per_second": 1301222.8321137927, "eta_seconds": 28631.053861451146, "data_wait_seconds": 2.2172927856445312e-05, "mix": {"fineweb_hq": 0.50915, "dclm": 0.20775, "cosmopedia_v2": 0.2007, "finemath": 0.08239}}
{"step": 1430, "total_steps": 72479, "tokens": 749731840, "loss": 5.757248222827911, "l_mix": 3.5374181270599365, "l_a": 3.7007655203342438, "l_c": 3.6932437121868134, "z_loss": 16.272287368774414, "lr": 0.0018794285714285716, "wd": 0.1, "grad_norm": 0.32070058584213257, "tokens_per_second": 1328005.7788789286, "eta_seconds": 28049.68073515892, "data_wait_seconds": 1.65402889251709e-05, "mix": {"fineweb_hq": 0.50979, "dclm": 0.20769, "cosmopedia_v2": 0.2, "finemath": 0.08252}}
{"step": 1440, "total_steps": 72479, "tokens": 754974720, "loss": 5.674987018108368, "l_mix": 3.487843871116638, "l_a": 3.6505517661571503, "l_c": 3.634561449289322, "z_loss": 16.0899817943573, "lr": 0.0018925714285714286, "wd": 0.1, "grad_norm": 0.3950078785419464, "tokens_per_second": 1472315.246176674, "eta_seconds": 25296.82099585533, "data_wait_seconds": 1.901388168334961e-05, "mix": {"fineweb_hq": 0.50972, "dclm": 0.20764, "cosmopedia_v2": 0.2, "finemath": 0.08264}}
{"step": 1450, "total_steps": 72479, "tokens": 760217600, "loss": 5.733676314353943, "l_mix": 3.52585306763649, "l_a": 3.689560741186142, "l_c": 3.6643960177898407, "z_loss": 16.360766887664795, "lr": 0.0019057142857142858, "wd": 0.1, "grad_norm": 0.37057679891586304, "tokens_per_second": 1366226.077738125, "eta_seconds": 27257.313382315635, "data_wait_seconds": 1.5348196029663086e-05, "mix": {"fineweb_hq": 0.50966, "dclm": 0.20759, "cosmopedia_v2": 0.20069, "finemath": 0.08207}}
{"step": 1460, "total_steps": 72479, "tokens": 765460480, "loss": 4.812945246696472, "l_mix": 2.9400192499160767, "l_a": 3.1407527923583984, "l_c": 3.0978047847747803, "z_loss": 13.586862444877625, "lr": 0.001918857142857143, "wd": 0.1, "grad_norm": 0.35002437233924866, "tokens_per_second": 1437298.471873105, "eta_seconds": 25905.829721975326, "data_wait_seconds": 1.633167266845703e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20753, "cosmopedia_v2": 0.20068, "finemath": 0.08219}}
{"step": 1470, "total_steps": 72479, "tokens": 770703360, "loss": 4.82649564743042, "l_mix": 2.9464896619319916, "l_a": 3.1440958976745605, "l_c": 3.1181194484233856, "z_loss": 13.413221001625061, "lr": 0.001932, "wd": 0.1, "grad_norm": 0.4614658057689667, "tokens_per_second": 1372798.289902217, "eta_seconds": 27119.18194088936, "data_wait_seconds": 2.5540590286254883e-05, "mix": {"fineweb_hq": 0.50952, "dclm": 0.20748, "cosmopedia_v2": 0.20068, "finemath": 0.08231}}
{"step": 1480, "total_steps": 72479, "tokens": 775946240, "loss": 4.770778059959412, "l_mix": 2.912609964609146, "l_a": 3.1098656356334686, "l_c": 3.079248607158661, "z_loss": 14.33746325969696, "lr": 0.001945142857142857, "wd": 0.1, "grad_norm": 0.378388911485672, "tokens_per_second": 1367468.3500664544, "eta_seconds": 27221.04954764843, "data_wait_seconds": 0.00011438131332397461, "mix": {"fineweb_hq": 0.50946, "dclm": 0.20743, "cosmopedia_v2": 0.20068, "finemath": 0.08243}}
{"step": 1490, "total_steps": 72479, "tokens": 781189120, "loss": 4.782551944255829, "l_mix": 2.9216631948947906, "l_a": 3.1279785335063934, "l_c": 3.070319265127182, "z_loss": 13.994028329849243, "lr": 0.0019582857142857145, "wd": 0.1, "grad_norm": 0.4353851079940796, "tokens_per_second": 1322718.3683590346, "eta_seconds": 28138.023726224896, "data_wait_seconds": 3.4421682357788086e-05, "mix": {"fineweb_hq": 0.5094, "dclm": 0.20738, "cosmopedia_v2": 0.20067, "finemath": 0.08255}}
{"step": 1500, "total_steps": 72479, "tokens": 786432000, "loss": 4.7583677768707275, "l_mix": 2.909101903438568, "l_a": 3.101900190114975, "l_c": 3.05749174952507, "z_loss": 14.481935501098633, "lr": 0.0019714285714285715, "wd": 0.1, "grad_norm": 0.40112248063087463, "tokens_per_second": 1413846.9091646723, "eta_seconds": 26320.69830954075, "data_wait_seconds": 2.753734588623047e-05, "mix": {"fineweb_hq": 0.50933, "dclm": 0.20733, "cosmopedia_v2": 0.20067, "finemath": 0.08267}, "gates": {"gate_a2b/L1_absmean": 0.0236725, "gate_a2b/L1_absmax": 0.14513971, "gate_a2b/L1_rms": 0.03619603, "gate_a2b/L3_absmean": 0.13465872, "gate_a2b/L3_absmax": 0.30005085, "gate_a2b/L3_rms": 0.14282395, "gate_a2b/L5_absmean": 0.20796342, "gate_a2b/L5_absmax": 0.44952688, "gate_a2b/L5_rms": 0.22059962, "gate_c2b/L1_absmean": 0.01022606, "gate_c2b/L1_absmax": 0.1363138, "gate_c2b/L1_rms": 0.02028015, "gate_c2b/L3_absmean": 0.18278401, "gate_c2b/L3_absmax": 0.38689911, "gate_c2b/L3_rms": 0.19119461, "gate_c2b/L5_absmean": 0.33592328, "gate_c2b/L5_absmax": 0.52759987, "gate_c2b/L5_rms": 0.34501979, "gate_b2a/L7_absmean": 0.02410925, "gate_b2a/L7_absmax": 0.10171875, "gate_b2a/L7_rms": 0.03207137, "gate_b2a/L11_absmean": 0.03490746, "gate_b2a/L11_absmax": 0.09498088, "gate_b2a/L11_rms": 0.03989695, "gate_b2a/L14_absmean": 0.10223724, "gate_b2a/L14_absmax": 0.14300582, "gate_b2a/L14_rms": 0.10332544, "gate_b2c/L3_absmean": 0.02117704, "gate_b2c/L3_absmax": 0.14606541, "gate_b2c/L3_rms": 0.03086617, "gate_b2c/L5_absmean": 0.036266, "gate_b2c/L5_absmax": 0.08782549, "gate_b2c/L5_rms": 0.0399503, "gate_b2c/L6_absmean": 0.03306918, "gate_b2c/L6_absmax": 0.12602493, "gate_b2c/L6_rms": 0.0374434}, "streams": {"cos_a2b/round0": 0.277501, "cos_c2b/round0": 0.319176, "cos_b2a/round0": 0.136238, "cos_b2c/round0": 0.124149, "rms_a/round0": 0.663271, "rms_b/round0": 1.257792, "rms_c/round0": 0.825037, "cos_a2b/round1": 0.357241, "cos_c2b/round1": 0.36084, "cos_b2a/round1": 0.166164, "cos_b2c/round1": 0.174588, "rms_a/round1": 0.928084, "rms_b/round1": 1.740281, "rms_c/round1": 1.210965, "cos_a2b/round2": 0.132048, "cos_c2b/round2": 0.607822, "cos_b2a/round2": 0.18497, "cos_b2c/round2": 0.04489, "rms_a/round2": 1.32892, "rms_b/round2": 3.569461, "rms_c/round2": 1.584808}, "alpha_mean": 0.4895626418914958, "alpha_hist": [0.03288, 0.10593, 0.13037, 0.13205, 0.13196, 0.11835, 0.0993, 0.10574, 0.11041, 0.033], "alpha_by_type": {"special": {"share": 0.00114, "mean": 0.81136, "std": 0.06175}, "whitespace": {"share": 0.03067, "mean": 0.70968, "std": 0.15585}, "digit": {"share": 0.00861, "mean": 0.56939, "std": 0.24343}, "word_start": {"share": 0.59138, "mean": 0.50234, "std": 0.22589}, "word_cont": {"share": 0.25429, "mean": 0.33502, "std": 0.20506}, "punct": {"share": 0.11391, "mean": 0.69969, "std": 0.197}}, "alpha_p05": 0.11398, "alpha_p50": 0.48133, "alpha_p95": 0.87769}
{"step": 1510, "total_steps": 72479, "tokens": 791674880, "loss": 5.881555616855621, "l_mix": 3.621366709470749, "l_a": 3.755167245864868, "l_c": 3.773531526327133, "z_loss": 15.7915358543396, "lr": 0.0019845714285714284, "wd": 0.1, "grad_norm": 0.4556715190410614, "tokens_per_second": 1461720.578848522, "eta_seconds": 25455.06686463356, "data_wait_seconds": 1.817941665649414e-05, "mix": {"fineweb_hq": 0.50927, "dclm": 0.20795, "cosmopedia_v2": 0.20066, "finemath": 0.08212}}
{"step": 1520, "total_steps": 72479, "tokens": 796917760, "loss": 5.970408320426941, "l_mix": 3.675388753414154, "l_a": 3.803871840238571, "l_c": 3.840834856033325, "z_loss": 16.074219703674316, "lr": 0.001997714285714286, "wd": 0.1, "grad_norm": 0.5657132863998413, "tokens_per_second": 1439684.4430692988, "eta_seconds": 25841.04618973732, "data_wait_seconds": 2.4437904357910156e-05, "mix": {"fineweb_hq": 0.50921, "dclm": 0.20789, "cosmopedia_v2": 0.20066, "finemath": 0.08224}}
{"step": 1530, "total_steps": 72479, "tokens": 802160640, "loss": 4.782869637012482, "l_mix": 2.9209280908107758, "l_a": 3.132534444332123, "l_c": 3.0691356658935547, "z_loss": 14.403419017791748, "lr": 0.002010857142857143, "wd": 0.1, "grad_norm": 0.3976997137069702, "tokens_per_second": 1434362.1901376673, "eta_seconds": 25933.27512936592, "data_wait_seconds": 2.250075340270996e-05, "mix": {"fineweb_hq": 0.50915, "dclm": 0.20784, "cosmopedia_v2": 0.20065, "finemath": 0.08235}}
{"step": 1540, "total_steps": 72479, "tokens": 807403520, "loss": 5.640060544013977, "l_mix": 3.466654270887375, "l_a": 3.6222980618476868, "l_c": 3.616943836212158, "z_loss": 16.335779190063477, "lr": 0.0020239999999999998, "wd": 0.1, "grad_norm": 0.24540096521377563, "tokens_per_second": 1430890.388026821, "eta_seconds": 25992.5335603714, "data_wait_seconds": 1.6421079635620117e-05, "mix": {"fineweb_hq": 0.50974, "dclm": 0.20779, "cosmopedia_v2": 0.2, "finemath": 0.08247}}
{"step": 1550, "total_steps": 72479, "tokens": 812646400, "loss": 5.5671274065971375, "l_mix": 3.423914760351181, "l_a": 3.570128947496414, "l_c": 3.5684399604797363, "z_loss": 16.419503211975098, "lr": 0.002037142857142857, "wd": 0.1, "grad_norm": 0.2624792754650116, "tokens_per_second": 1398101.9555558325, "eta_seconds": 26598.36316244602, "data_wait_seconds": 1.7911195755004883e-05, "mix": {"fineweb_hq": 0.50968, "dclm": 0.20774, "cosmopedia_v2": 0.2, "finemath": 0.08258}}
{"step": 1560, "total_steps": 72479, "tokens": 817889280, "loss": 5.575970411300659, "l_mix": 3.430813729763031, "l_a": 3.5828343629837036, "l_c": 3.56203493475914, "z_loss": 16.956971406936646, "lr": 0.002050285714285714, "wd": 0.1, "grad_norm": 0.3612804710865021, "tokens_per_second": 1334992.163789361, "eta_seconds": 27851.834400629996, "data_wait_seconds": 1.722574234008789e-05, "mix": {"fineweb_hq": 0.50962, "dclm": 0.20769, "cosmopedia_v2": 0.20064, "finemath": 0.08205}}
{"step": 1570, "total_steps": 72479, "tokens": 823132160, "loss": 4.733025014400482, "l_mix": 2.8923244178295135, "l_a": 3.0868577361106873, "l_c": 3.0439680218696594, "z_loss": 14.528416156768799, "lr": 0.002063428571428571, "wd": 0.1, "grad_norm": 0.4019538164138794, "tokens_per_second": 1467899.6017701922, "eta_seconds": 25326.485372138024, "data_wait_seconds": 1.6003847122192383e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20764, "cosmopedia_v2": 0.20064, "finemath": 0.08217}}
{"step": 1580, "total_steps": 72479, "tokens": 828375040, "loss": 4.772237539291382, "l_mix": 2.916307896375656, "l_a": 3.105850338935852, "l_c": 3.075670152902603, "z_loss": 14.734213471412659, "lr": 0.0020765714285714285, "wd": 0.1, "grad_norm": 0.39850929379463196, "tokens_per_second": 1421854.5038281363, "eta_seconds": 26142.966676211356, "data_wait_seconds": 1.6868114471435547e-05, "mix": {"fineweb_hq": 0.50949, "dclm": 0.20759, "cosmopedia_v2": 0.20063, "finemath": 0.08228}}
{"step": 1590, "total_steps": 72479, "tokens": 833617920, "loss": 4.711366891860962, "l_mix": 2.879717618227005, "l_a": 3.074548065662384, "l_c": 3.026206910610199, "z_loss": 14.228463768959045, "lr": 0.002089714285714286, "wd": 0.1, "grad_norm": 0.38938435912132263, "tokens_per_second": 1465610.8973862615, "eta_seconds": 25358.88079044819, "data_wait_seconds": 2.5719404220581055e-05, "mix": {"fineweb_hq": 0.50943, "dclm": 0.20755, "cosmopedia_v2": 0.20063, "finemath": 0.08239}}
{"step": 1600, "total_steps": 72479, "tokens": 838860800, "loss": 4.673660933971405, "l_mix": 2.857440263032913, "l_a": 3.0321598052978516, "l_c": 3.016664743423462, "z_loss": 15.73154866695404, "lr": 0.002102857142857143, "wd": 0.1, "grad_norm": 0.5418248176574707, "tokens_per_second": 1462120.1232689202, "eta_seconds": 25415.8386582613, "data_wait_seconds": 1.9997358322143555e-05, "mix": {"fineweb_hq": 0.50938, "dclm": 0.2075, "cosmopedia_v2": 0.20062, "finemath": 0.0825}, "gates": {"gate_a2b/L1_absmean": 0.02870532, "gate_a2b/L1_absmax": 0.1716931, "gate_a2b/L1_rms": 0.04242295, "gate_a2b/L3_absmean": 0.15302132, "gate_a2b/L3_absmax": 0.33354926, "gate_a2b/L3_rms": 0.16197698, "gate_a2b/L5_absmean": 0.23301868, "gate_a2b/L5_absmax": 0.46361277, "gate_a2b/L5_rms": 0.24522315, "gate_c2b/L1_absmean": 0.0138596, "gate_c2b/L1_absmax": 0.17164557, "gate_c2b/L1_rms": 0.02464141, "gate_c2b/L3_absmean": 0.19679423, "gate_c2b/L3_absmax": 0.41344142, "gate_c2b/L3_rms": 0.20579673, "gate_c2b/L5_absmean": 0.35404009, "gate_c2b/L5_absmax": 0.55379826, "gate_c2b/L5_rms": 0.3638345, "gate_b2a/L7_absmean": 0.02567026, "gate_b2a/L7_absmax": 0.10502835, "gate_b2a/L7_rms": 0.03384129, "gate_b2a/L11_absmean": 0.03939056, "gate_b2a/L11_absmax": 0.10450707, "gate_b2a/L11_rms": 0.0444229, "gate_b2a/L14_absmean": 0.11495855, "gate_b2a/L14_absmax": 0.15792631, "gate_b2a/L14_rms": 0.11607356, "gate_b2c/L3_absmean": 0.02218124, "gate_b2c/L3_absmax": 0.16975667, "gate_b2c/L3_rms": 0.032389, "gate_b2c/L5_absmean": 0.0395554, "gate_b2c/L5_absmax": 0.09389278, "gate_b2c/L5_rms": 0.04313362, "gate_b2c/L6_absmean": 0.03934776, "gate_b2c/L6_absmax": 0.14972562, "gate_b2c/L6_rms": 0.0435687}, "streams": {"cos_a2b/round0": 0.279434, "cos_c2b/round0": 0.305464, "cos_b2a/round0": 0.136407, "cos_b2c/round0": 0.122608, "rms_a/round0": 0.74949, "rms_b/round0": 1.445485, "rms_c/round0": 0.890015, "cos_a2b/round1": 0.362023, "cos_c2b/round1": 0.349859, "cos_b2a/round1": 0.173101, "cos_b2c/round1": 0.178205, "rms_a/round1": 1.072145, "rms_b/round1": 2.022761, "rms_c/round1": 1.317624, "cos_a2b/round2": 0.171079, "cos_c2b/round2": 0.560373, "cos_b2a/round2": 0.174342, "cos_b2c/round2": 0.056098, "rms_a/round2": 1.560428, "rms_b/round2": 3.80011, "rms_c/round2": 1.760457}, "alpha_mean": 0.480704414025773, "alpha_hist": [0.04585, 0.1104, 0.12605, 0.1241, 0.12204, 0.12276, 0.12024, 0.1089, 0.0912, 0.02846], "alpha_by_type": {"special": {"share": 0.0011, "mean": 0.59104, "std": 0.12831}, "whitespace": {"share": 0.03214, "mean": 0.55397, "std": 0.18018}, "digit": {"share": 0.01009, "mean": 0.41937, "std": 0.25087}, "word_start": {"share": 0.57823, "mean": 0.51774, "std": 0.22219}, "word_cont": {"share": 0.25761, "mean": 0.31501, "std": 0.2073}, "punct": {"share": 0.12083, "mean": 0.64134, "std": 0.23463}}, "alpha_p05": 0.11717, "alpha_p50": 0.46247, "alpha_p95": 0.84822}
{"step": 1610, "total_steps": 72479, "tokens": 844103680, "loss": 4.76736456155777, "l_mix": 2.910944551229477, "l_a": 3.106129229068756, "l_c": 3.0775056183338165, "z_loss": 13.294909715652466, "lr": 0.002116, "wd": 0.1, "grad_norm": 0.6477193236351013, "tokens_per_second": 1435402.202462025, "eta_seconds": 25885.264916181564, "data_wait_seconds": 1.96993350982666e-05, "mix": {"fineweb_hq": 0.50932, "dclm": 0.20745, "cosmopedia_v2": 0.20062, "finemath": 0.08261}}
{"step": 1620, "total_steps": 72479, "tokens": 849346560, "loss": 5.62733119726181, "l_mix": 3.460707426071167, "l_a": 3.6129441261291504, "l_c": 3.6038776338100433, "z_loss": 15.773223161697388, "lr": 0.0021291428571428572, "wd": 0.1, "grad_norm": 0.383698970079422, "tokens_per_second": 1395830.7158676027, "eta_seconds": 26615.350249624255, "data_wait_seconds": 1.6480684280395508e-05, "mix": {"fineweb_hq": 0.50988, "dclm": 0.20741, "cosmopedia_v2": 0.20062, "finemath": 0.0821}}
{"step": 1630, "total_steps": 72479, "tokens": 854589440, "loss": 5.691203832626343, "l_mix": 3.5030608475208282, "l_a": 3.6418424546718597, "l_c": 3.6469235122203827, "z_loss": 15.130305647850037, "lr": 0.002142285714285714, "wd": 0.1, "grad_norm": 0.4173033833503723, "tokens_per_second": 1441325.8796481367, "eta_seconds": 25771.604490351678, "data_wait_seconds": 2.1398067474365234e-05, "mix": {"fineweb_hq": 0.50982, "dclm": 0.20736, "cosmopedia_v2": 0.20061, "finemath": 0.08221}}
{"step": 1640, "total_steps": 72479, "tokens": 859832320, "loss": 5.5292248129844666, "l_mix": 3.401210218667984, "l_a": 3.5458240807056427, "l_c": 3.5422285199165344, "z_loss": 15.986873626708984, "lr": 0.002155428571428571, "wd": 0.1, "grad_norm": 0.37714821100234985, "tokens_per_second": 1341179.4994263607, "eta_seconds": 27692.07078387737, "data_wait_seconds": 1.940131187438965e-05, "mix": {"fineweb_hq": 0.50976, "dclm": 0.20732, "cosmopedia_v2": 0.20061, "finemath": 0.08232}}
{"step": 1650, "total_steps": 72479, "tokens": 865075200, "loss": 5.603922188282013, "l_mix": 3.446375995874405, "l_a": 3.5941411554813385, "l_c": 3.5920983254909515, "z_loss": 16.74113440513611, "lr": 0.0021685714285714286, "wd": 0.1, "grad_norm": 0.3248688280582428, "tokens_per_second": 1440863.6910527374, "eta_seconds": 25772.593884205817, "data_wait_seconds": 1.8268823623657227e-05, "mix": {"fineweb_hq": 0.5097, "dclm": 0.20727, "cosmopedia_v2": 0.20061, "finemath": 0.08242}}
{"step": 1660, "total_steps": 72479, "tokens": 870318080, "loss": 5.676292598247528, "l_mix": 3.490763336420059, "l_a": 3.6495871543884277, "l_c": 3.630276769399643, "z_loss": 15.700549483299255, "lr": 0.002181714285714286, "wd": 0.1, "grad_norm": 0.442329078912735, "tokens_per_second": 1456891.2849618741, "eta_seconds": 25485.465013933182, "data_wait_seconds": 2.536177635192871e-05, "mix": {"fineweb_hq": 0.50964, "dclm": 0.20723, "cosmopedia_v2": 0.2006, "finemath": 0.08253}}
{"step": 1670, "total_steps": 72479, "tokens": 875560960, "loss": 5.603530704975128, "l_mix": 3.44915172457695, "l_a": 3.6017171442508698, "l_c": 3.5740634500980377, "z_loss": 16.44762110710144, "lr": 0.002194857142857143, "wd": 0.1, "grad_norm": 0.392281174659729, "tokens_per_second": 1455354.7391659946, "eta_seconds": 25508.76978163719, "data_wait_seconds": 2.9146671295166016e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20778, "cosmopedia_v2": 0.2006, "finemath": 0.08204}}
{"step": 1680, "total_steps": 72479, "tokens": 880803840, "loss": 4.652345418930054, "l_mix": 2.8455366492271423, "l_a": 3.0196653604507446, "l_c": 2.9977984726428986, "z_loss": 15.696058988571167, "lr": 0.002208, "wd": 0.1, "grad_norm": 0.46833544969558716, "tokens_per_second": 1228362.9042526658, "eta_seconds": 30218.32227551937, "data_wait_seconds": 4.4345855712890625e-05, "mix": {"fineweb_hq": 0.50952, "dclm": 0.20774, "cosmopedia_v2": 0.2006, "finemath": 0.08214}}
{"step": 1690, "total_steps": 72479, "tokens": 886046720, "loss": 4.701807260513306, "l_mix": 2.875477910041809, "l_a": 3.0469761192798615, "l_c": 3.035790294408798, "z_loss": 14.992761254310608, "lr": 0.0022211428571428573, "wd": 0.1, "grad_norm": 0.4084111750125885, "tokens_per_second": 1382358.2251060128, "eta_seconds": 26848.195032191274, "data_wait_seconds": 2.130866050720215e-05, "mix": {"fineweb_hq": 0.50947, "dclm": 0.20769, "cosmopedia_v2": 0.20059, "finemath": 0.08225}}
{"step": 1700, "total_steps": 72479, "tokens": 891289600, "loss": 4.671900570392609, "l_mix": 2.8562495410442352, "l_a": 3.0321476757526398, "l_c": 3.015009820461273, "z_loss": 15.039068222045898, "lr": 0.0022342857142857143, "wd": 0.1, "grad_norm": 0.39999592304229736, "tokens_per_second": 1422306.5075959677, "eta_seconds": 26090.424359178545, "data_wait_seconds": 1.6450881958007812e-05, "mix": {"fineweb_hq": 0.50941, "dclm": 0.20765, "cosmopedia_v2": 0.20059, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.03112685, "gate_a2b/L1_absmax": 0.20911442, "gate_a2b/L1_rms": 0.0488964, "gate_a2b/L3_absmean": 0.17239289, "gate_a2b/L3_absmax": 0.37709635, "gate_a2b/L3_rms": 0.18274437, "gate_a2b/L5_absmean": 0.25651264, "gate_a2b/L5_absmax": 0.47863871, "gate_a2b/L5_rms": 0.26858208, "gate_c2b/L1_absmean": 0.01270319, "gate_c2b/L1_absmax": 0.21034406, "gate_c2b/L1_rms": 0.02758644, "gate_c2b/L3_absmean": 0.21003591, "gate_c2b/L3_absmax": 0.44919467, "gate_c2b/L3_rms": 0.21983883, "gate_c2b/L5_absmean": 0.37160403, "gate_c2b/L5_absmax": 0.59003019, "gate_c2b/L5_rms": 0.38212201, "gate_b2a/L7_absmean": 0.02653724, "gate_b2a/L7_absmax": 0.11827943, "gate_b2a/L7_rms": 0.03534952, "gate_b2a/L11_absmean": 0.04151864, "gate_b2a/L11_absmax": 0.10866952, "gate_b2a/L11_rms": 0.04662156, "gate_b2a/L14_absmean": 0.12602606, "gate_b2a/L14_absmax": 0.16584206, "gate_b2a/L14_rms": 0.12711668, "gate_b2c/L3_absmean": 0.0216324, "gate_b2c/L3_absmax": 0.2003864, "gate_b2c/L3_rms": 0.03278087, "gate_b2c/L5_absmean": 0.04190246, "gate_b2c/L5_absmax": 0.10612939, "gate_b2c/L5_rms": 0.0452636, "gate_b2c/L6_absmean": 0.04581951, "gate_b2c/L6_absmax": 0.17118505, "gate_b2c/L6_rms": 0.0495202}, "streams": {"cos_a2b/round0": 0.290912, "cos_c2b/round0": 0.315048, "cos_b2a/round0": 0.135174, "cos_b2c/round0": 0.124277, "rms_a/round0": 0.847972, "rms_b/round0": 1.581647, "rms_c/round0": 0.9547, "cos_a2b/round1": 0.374952, "cos_c2b/round1": 0.353926, "cos_b2a/round1": 0.182966, "cos_b2c/round1": 0.195902, "rms_a/round1": 1.236134, "rms_b/round1": 2.262055, "rms_c/round1": 1.445955, "cos_a2b/round2": 0.195436, "cos_c2b/round2": 0.560801, "cos_b2a/round2": 0.16078, "cos_b2c/round2": 0.066897, "rms_a/round2": 1.818583, "rms_b/round2": 4.181604, "rms_c/round2": 1.948716}, "alpha_mean": 0.538284224890532, "alpha_hist": [0.0146, 0.07468, 0.10744, 0.12384, 0.13251, 0.12698, 0.11168, 0.12558, 0.13699, 0.0457], "alpha_by_type": {"special": {"share": 0.00119, "mean": 0.81271, "std": 0.07239}, "whitespace": {"share": 0.03276, "mean": 0.77602, "std": 0.13086}, "digit": {"share": 0.00914, "mean": 0.57887, "std": 0.24373}, "word_start": {"share": 0.57446, "mean": 0.54668, "std": 0.21614}, "word_cont": {"share": 0.25934, "mean": 0.38592, "std": 0.20502}, "punct": {"share": 0.12311, "mean": 0.75112, "std": 0.17495}}, "alpha_p05": 0.13823, "alpha_p50": 0.52567, "alpha_p95": 0.88876}
{"step": 1710, "total_steps": 72479, "tokens": 896532480, "loss": 4.68269670009613, "l_mix": 2.8618715405464172, "l_a": 3.0474750995635986, "l_c": 3.0172199308872223, "z_loss": 14.16512644290924, "lr": 0.0022474285714285712, "wd": 0.1, "grad_norm": 0.41673603653907776, "tokens_per_second": 1430335.4986211294, "eta_seconds": 25940.30387120247, "data_wait_seconds": 2.6971101760864258e-05, "mix": {"fineweb_hq": 0.50936, "dclm": 0.2076, "cosmopedia_v2": 0.20058, "finemath": 0.08246}}
{"step": 1720, "total_steps": 72479, "tokens": 901775360, "loss": 4.65797758102417, "l_mix": 2.846878170967102, "l_a": 3.021965205669403, "l_c": 3.010215640068054, "z_loss": 14.45012354850769, "lr": 0.0022605714285714286, "wd": 0.1, "grad_norm": 0.33336496353149414, "tokens_per_second": 1484607.1399410053, "eta_seconds": 24988.49264154434, "data_wait_seconds": 2.9295682907104492e-05, "mix": {"fineweb_hq": 0.5093, "dclm": 0.20756, "cosmopedia_v2": 0.20058, "finemath": 0.08256}}
{"step": 1730, "total_steps": 72479, "tokens": 907018240, "loss": 5.542114615440369, "l_mix": 3.4103066623210907, "l_a": 3.547895610332489, "l_c": 3.552777647972107, "z_loss": 16.058656811714172, "lr": 0.0022737142857142856, "wd": 0.1, "grad_norm": 0.32529178261756897, "tokens_per_second": 1457717.592080176, "eta_seconds": 25445.84212574959, "data_wait_seconds": 1.5139579772949219e-05, "mix": {"fineweb_hq": 0.50983, "dclm": 0.20751, "cosmopedia_v2": 0.20058, "finemath": 0.08208}}
{"step": 1740, "total_steps": 72479, "tokens": 912261120, "loss": 5.670702219009399, "l_mix": 3.491836726665497, "l_a": 3.6334586441516876, "l_c": 3.624117851257324, "z_loss": 15.923676252365112, "lr": 0.0022868571428571426, "wd": 0.1, "grad_norm": 0.38739702105522156, "tokens_per_second": 1459157.8509108128, "eta_seconds": 25417.132772064208, "data_wait_seconds": 1.677870750427246e-05, "mix": {"fineweb_hq": 0.50977, "dclm": 0.20747, "cosmopedia_v2": 0.20057, "finemath": 0.08218}}
{"step": 1750, "total_steps": 72479, "tokens": 917504000, "loss": 5.556439995765686, "l_mix": 3.4219774305820465, "l_a": 3.552911549806595, "l_c": 3.5565345883369446, "z_loss": 16.28438949584961, "lr": 0.0023000000000000004, "wd": 0.1, "grad_norm": 0.46838366985321045, "tokens_per_second": 1471315.180983235, "eta_seconds": 25203.550151109695, "data_wait_seconds": 1.84476375579834e-05, "mix": {"fineweb_hq": 0.50971, "dclm": 0.20743, "cosmopedia_v2": 0.20057, "finemath": 0.08229}}
{"step": 1760, "total_steps": 72479, "tokens": 922746880, "loss": 5.4551092982292175, "l_mix": 3.3565680384635925, "l_a": 3.5003224313259125, "l_c": 3.4890723824501038, "z_loss": 17.226924896240234, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.338798850774765, "tokens_per_second": 1452942.7961869212, "eta_seconds": 25518.63925359249, "data_wait_seconds": 1.615285873413086e-05, "mix": {"fineweb_hq": 0.50966, "dclm": 0.20739, "cosmopedia_v2": 0.20057, "finemath": 0.08239}}
{"step": 1770, "total_steps": 72479, "tokens": 927989760, "loss": 5.687047064304352, "l_mix": 3.4985172152519226, "l_a": 3.6331247985363007, "l_c": 3.6567125618457794, "z_loss": 15.785502552986145, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.48259708285331726, "tokens_per_second": 1317486.7863595847, "eta_seconds": 28138.33169016838, "data_wait_seconds": 2.8848648071289062e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20734, "cosmopedia_v2": 0.20056, "finemath": 0.08249}}
{"step": 1780, "total_steps": 72479, "tokens": 933232640, "loss": 5.594672858715057, "l_mix": 3.4426729679107666, "l_a": 3.5826640725135803, "l_c": 3.5849227607250214, "z_loss": 17.236823320388794, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.4048011004924774, "tokens_per_second": 1423754.207558378, "eta_seconds": 26034.435659766194, "data_wait_seconds": 2.1129846572875977e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.2073, "cosmopedia_v2": 0.20056, "finemath": 0.08258}}
{"step": 1790, "total_steps": 72479, "tokens": 938475520, "loss": 5.630514740943909, "l_mix": 3.467810094356537, "l_a": 3.5896652340888977, "l_c": 3.6138612627983093, "z_loss": 16.466612577438354, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.32825779914855957, "tokens_per_second": 1466201.3202819903, "eta_seconds": 25277.152543330194, "data_wait_seconds": 2.7954578399658203e-05, "mix": {"fineweb_hq": 0.5095, "dclm": 0.20782, "cosmopedia_v2": 0.20056, "finemath": 0.08212}}
{"step": 1800, "total_steps": 72479, "tokens": 943718400, "loss": 5.724473416805267, "l_mix": 3.5282114148139954, "l_a": 3.64393749833107, "l_c": 3.6712052822113037, "z_loss": 17.191205978393555, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2617133557796478, "tokens_per_second": 1440318.7764113778, "eta_seconds": 25727.7431627512, "data_wait_seconds": 2.1398067474365234e-05, "mix": {"fineweb_hq": 0.50944, "dclm": 0.20778, "cosmopedia_v2": 0.20056, "finemath": 0.08222}, "gates": {"gate_a2b/L1_absmean": 0.03443722, "gate_a2b/L1_absmax": 0.24162708, "gate_a2b/L1_rms": 0.0552687, "gate_a2b/L3_absmean": 0.18949033, "gate_a2b/L3_absmax": 0.41144222, "gate_a2b/L3_rms": 0.20085855, "gate_a2b/L5_absmean": 0.27595684, "gate_a2b/L5_absmax": 0.49951023, "gate_a2b/L5_rms": 0.28812799, "gate_c2b/L1_absmean": 0.01259748, "gate_c2b/L1_absmax": 0.26104724, "gate_c2b/L1_rms": 0.03039769, "gate_c2b/L3_absmean": 0.22078137, "gate_c2b/L3_absmax": 0.48294991, "gate_c2b/L3_rms": 0.23144342, "gate_c2b/L5_absmean": 0.38902172, "gate_c2b/L5_absmax": 0.61497521, "gate_c2b/L5_rms": 0.40017813, "gate_b2a/L7_absmean": 0.02738145, "gate_b2a/L7_absmax": 0.12948534, "gate_b2a/L7_rms": 0.03661643, "gate_b2a/L11_absmean": 0.04571337, "gate_b2a/L11_absmax": 0.11212261, "gate_b2a/L11_rms": 0.05071228, "gate_b2a/L14_absmean": 0.1370198, "gate_b2a/L14_absmax": 0.17627956, "gate_b2a/L14_rms": 0.13809453, "gate_b2c/L3_absmean": 0.02155493, "gate_b2c/L3_absmax": 0.23062235, "gate_b2c/L3_rms": 0.03342082, "gate_b2c/L5_absmean": 0.04437248, "gate_b2c/L5_absmax": 0.12825927, "gate_b2c/L5_rms": 0.04760208, "gate_b2c/L6_absmean": 0.05316012, "gate_b2c/L6_absmax": 0.18854074, "gate_b2c/L6_rms": 0.05630262}, "streams": {"cos_a2b/round0": 0.319451, "cos_c2b/round0": 0.328548, "cos_b2a/round0": 0.1501, "cos_b2c/round0": 0.143067, "rms_a/round0": 0.9441, "rms_b/round0": 1.727202, "rms_c/round0": 1.007377, "cos_a2b/round1": 0.384207, "cos_c2b/round1": 0.37387, "cos_b2a/round1": 0.19314, "cos_b2c/round1": 0.202331, "rms_a/round1": 1.392915, "rms_b/round1": 2.533729, "rms_c/round1": 1.52995, "cos_a2b/round2": 0.204763, "cos_c2b/round2": 0.557544, "cos_b2a/round2": 0.170537, "cos_b2c/round2": 0.088354, "rms_a/round2": 2.102854, "rms_b/round2": 4.961899, "rms_c/round2": 2.121017}, "alpha_mean": 0.4923240121386101, "alpha_hist": [0.01056, 0.07019, 0.12268, 0.14417, 0.15881, 0.16872, 0.15123, 0.10577, 0.06177, 0.0061], "alpha_by_type": {"special": {"share": 0.00035, "mean": 0.63443, "std": 0.11514}, "whitespace": {"share": 0.03763, "mean": 0.50828, "std": 0.13357}, "digit": {"share": 0.03679, "mean": 0.52025, "std": 0.2017}, "word_start": {"share": 0.54521, "mean": 0.53792, "std": 0.17666}, "word_cont": {"share": 0.27995, "mean": 0.34441, "std": 0.17096}, "punct": {"share": 0.10007, "mean": 0.64093, "std": 0.18329}}, "alpha_p05": 0.19286, "alpha_p50": 0.4968, "alpha_p95": 0.80837}
{"step": 1810, "total_steps": 72479, "tokens": 948961280, "loss": 4.600106477737427, "l_mix": 2.8129283487796783, "l_a": 2.9815207421779633, "l_c": 2.9708486199378967, "z_loss": 14.672969222068787, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.37738898396492004, "tokens_per_second": 1388768.3211288478, "eta_seconds": 26678.970212888715, "data_wait_seconds": 2.1249055862426758e-05, "mix": {"fineweb_hq": 0.50939, "dclm": 0.20773, "cosmopedia_v2": 0.20055, "finemath": 0.08232}}
{"step": 1820, "total_steps": 72479, "tokens": 954204160, "loss": 4.583521604537964, "l_mix": 2.803867667913437, "l_a": 2.9666723906993866, "l_c": 2.9607405066490173, "z_loss": 14.299499988555908, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.3699977993965149, "tokens_per_second": 1408364.6597022843, "eta_seconds": 26304.02966787815, "data_wait_seconds": 1.806020736694336e-05, "mix": {"fineweb_hq": 0.50934, "dclm": 0.20769, "cosmopedia_v2": 0.20055, "finemath": 0.08242}}
{"step": 1830, "total_steps": 72479, "tokens": 959447040, "loss": 4.583469271659851, "l_mix": 2.8054513335227966, "l_a": 2.971978634595871, "l_c": 2.949857175350189, "z_loss": 14.671529293060303, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.46945813298225403, "tokens_per_second": 1296868.8735842812, "eta_seconds": 28561.424879932405, "data_wait_seconds": 2.1278858184814453e-05, "mix": {"fineweb_hq": 0.50929, "dclm": 0.20765, "cosmopedia_v2": 0.20055, "finemath": 0.08251}}
{"step": 1840, "total_steps": 72479, "tokens": 964689920, "loss": 5.502140820026398, "l_mix": 3.388327330350876, "l_a": 3.520661771297455, "l_c": 3.519958734512329, "z_loss": 16.27230715751648, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.3358713388442993, "tokens_per_second": 1375977.1543604154, "eta_seconds": 26915.548644566537, "data_wait_seconds": 2.0444393157958984e-05, "mix": {"fineweb_hq": 0.50978, "dclm": 0.20761, "cosmopedia_v2": 0.20054, "finemath": 0.08207}}
{"step": 1850, "total_steps": 72479, "tokens": 969932800, "loss": 5.566343605518341, "l_mix": 3.429606705904007, "l_a": 3.559303432703018, "l_c": 3.5576427280902863, "z_loss": 16.529460668563843, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.32508090138435364, "tokens_per_second": 1445105.121612623, "eta_seconds": 25624.389947962758, "data_wait_seconds": 1.5437602996826172e-05, "mix": {"fineweb_hq": 0.50973, "dclm": 0.20757, "cosmopedia_v2": 0.20054, "finemath": 0.08216}}
{"step": 1860, "total_steps": 72479, "tokens": 975175680, "loss": 5.444896042346954, "l_mix": 3.352449059486389, "l_a": 3.4831026792526245, "l_c": 3.4863735139369965, "z_loss": 16.041075587272644, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.46789008378982544, "tokens_per_second": 1466578.8655295454, "eta_seconds": 25245.621045160296, "data_wait_seconds": 1.558661460876465e-05, "mix": {"fineweb_hq": 0.50968, "dclm": 0.20753, "cosmopedia_v2": 0.20054, "finemath": 0.08226}}
{"step": 1870, "total_steps": 72479, "tokens": 980418560, "loss": 5.4909772872924805, "l_mix": 3.3825858533382416, "l_a": 3.5096529722213745, "l_c": 3.512459695339203, "z_loss": 17.57507872581482, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.384528785943985, "tokens_per_second": 1460281.2712055633, "eta_seconds": 25350.90473456383, "data_wait_seconds": 1.570582389831543e-05, "mix": {"fineweb_hq": 0.50963, "dclm": 0.20749, "cosmopedia_v2": 0.20053, "finemath": 0.08235}}
{"step": 1880, "total_steps": 72479, "tokens": 985661440, "loss": 5.484338879585266, "l_mix": 3.377007156610489, "l_a": 3.5057373046875, "l_c": 3.5134312510490417, "z_loss": 15.810742974281311, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.37996208667755127, "tokens_per_second": 1444775.6640277067, "eta_seconds": 25619.346611094476, "data_wait_seconds": 1.683831214904785e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20745, "cosmopedia_v2": 0.20053, "finemath": 0.08245}}
{"step": 1890, "total_steps": 72479, "tokens": 990904320, "loss": 5.592708885669708, "l_mix": 3.4451593458652496, "l_a": 3.569009393453598, "l_c": 3.5838108360767365, "z_loss": 17.033183574676514, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.5029730200767517, "tokens_per_second": 1438482.1874431712, "eta_seconds": 25727.788605976104, "data_wait_seconds": 2.0444393157958984e-05, "mix": {"fineweb_hq": 0.50952, "dclm": 0.20741, "cosmopedia_v2": 0.20053, "finemath": 0.08254}}
{"step": 1900, "total_steps": 72479, "tokens": 996147200, "loss": 5.6262558698654175, "l_mix": 3.470389634370804, "l_a": 3.5864563584327698, "l_c": 3.5941942036151886, "z_loss": 16.7097270488739, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2684158682823181, "tokens_per_second": 1429331.3338627242, "eta_seconds": 25888.834782624243, "data_wait_seconds": 1.6629695892333984e-05, "mix": {"fineweb_hq": 0.50947, "dclm": 0.20789, "cosmopedia_v2": 0.20053, "finemath": 0.08211}, "gates": {"gate_a2b/L1_absmean": 0.0380941, "gate_a2b/L1_absmax": 0.28824022, "gate_a2b/L1_rms": 0.06268974, "gate_a2b/L3_absmean": 0.20378821, "gate_a2b/L3_absmax": 0.41543627, "gate_a2b/L3_rms": 0.21563132, "gate_a2b/L5_absmean": 0.29519197, "gate_a2b/L5_absmax": 0.51560563, "gate_a2b/L5_rms": 0.30737245, "gate_c2b/L1_absmean": 0.01303697, "gate_c2b/L1_absmax": 0.29190284, "gate_c2b/L1_rms": 0.03294436, "gate_c2b/L3_absmean": 0.23072334, "gate_c2b/L3_absmax": 0.50692385, "gate_c2b/L3_rms": 0.24187323, "gate_c2b/L5_absmean": 0.40437222, "gate_c2b/L5_absmax": 0.63662654, "gate_c2b/L5_rms": 0.41591555, "gate_b2a/L7_absmean": 0.02874636, "gate_b2a/L7_absmax": 0.13892484, "gate_b2a/L7_rms": 0.0382163, "gate_b2a/L11_absmean": 0.0492493, "gate_b2a/L11_absmax": 0.11465254, "gate_b2a/L11_rms": 0.05437575, "gate_b2a/L14_absmean": 0.14847898, "gate_b2a/L14_absmax": 0.19292282, "gate_b2a/L14_rms": 0.14951989, "gate_b2c/L3_absmean": 0.02158929, "gate_b2c/L3_absmax": 0.25047722, "gate_b2c/L3_rms": 0.03427586, "gate_b2c/L5_absmean": 0.04725697, "gate_b2c/L5_absmax": 0.14619659, "gate_b2c/L5_rms": 0.05034556, "gate_b2c/L6_absmean": 0.06127615, "gate_b2c/L6_absmax": 0.20059307, "gate_b2c/L6_rms": 0.06402747}, "streams": {"cos_a2b/round0": 0.325979, "cos_c2b/round0": 0.316583, "cos_b2a/round0": 0.152828, "cos_b2c/round0": 0.140948, "rms_a/round0": 1.00571, "rms_b/round0": 1.853287, "rms_c/round0": 1.038064, "cos_a2b/round1": 0.395399, "cos_c2b/round1": 0.368873, "cos_b2a/round1": 0.199841, "cos_b2c/round1": 0.217308, "rms_a/round1": 1.508224, "rms_b/round1": 2.706252, "rms_c/round1": 1.611042, "cos_a2b/round2": 0.242403, "cos_c2b/round2": 0.53622, "cos_b2a/round2": 0.167864, "cos_b2c/round2": 0.110362, "rms_a/round2": 2.296102, "rms_b/round2": 4.791889, "rms_c/round2": 2.210539}, "alpha_mean": 0.56305630068357, "alpha_hist": [0.00606, 0.03011, 0.06198, 0.09186, 0.1375, 0.20303, 0.23393, 0.15758, 0.07114, 0.00682], "alpha_by_type": {"special": {"share": 0.00067, "mean": 0.67033, "std": 0.12362}, "whitespace": {"share": 0.05252, "mean": 0.61375, "std": 0.12186}, "digit": {"share": 0.03476, "mean": 0.60661, "std": 0.1665}, "word_start": {"share": 0.56239, "mean": 0.59632, "std": 0.15169}, "word_cont": {"share": 0.23981, "mean": 0.42106, "std": 0.18101}, "punct": {"share": 0.10985, "mean": 0.6641, "std": 0.15752}}, "alpha_p05": 0.22816, "alpha_p50": 0.6051, "alpha_p95": 0.83575}
{"step": 1910, "total_steps": 72479, "tokens": 1001390080, "loss": 5.610076367855072, "l_mix": 3.459757536649704, "l_a": 3.572082281112671, "l_c": 3.5898395478725433, "z_loss": 17.421445846557617, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2754519581794739, "tokens_per_second": 1149543.07612383, "eta_seconds": 32185.37925238609, "data_wait_seconds": 1.8477439880371094e-05, "mix": {"fineweb_hq": 0.50942, "dclm": 0.20785, "cosmopedia_v2": 0.20052, "finemath": 0.0822}}
{"step": 1920, "total_steps": 72479, "tokens": 1006632960, "loss": 5.68332439661026, "l_mix": 3.504735618829727, "l_a": 3.6143110394477844, "l_c": 3.6420326232910156, "z_loss": 16.85582709312439, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.3300733268260956, "tokens_per_second": 1361802.985513703, "eta_seconds": 27164.89638040066, "data_wait_seconds": 1.811981201171875e-05, "mix": {"fineweb_hq": 0.50938, "dclm": 0.20781, "cosmopedia_v2": 0.20052, "finemath": 0.08229}}
{"step": 1930, "total_steps": 72479, "tokens": 1011875840, "loss": 5.627560794353485, "l_mix": 3.4709306359291077, "l_a": 3.585011273622513, "l_c": 3.5982885360717773, "z_loss": 16.40136444568634, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.3529491722583771, "tokens_per_second": 1115856.5268021184, "eta_seconds": 33147.62536542416, "data_wait_seconds": 4.926323890686035e-05, "mix": {"fineweb_hq": 0.50933, "dclm": 0.20777, "cosmopedia_v2": 0.20052, "finemath": 0.08238}}
{"step": 1940, "total_steps": 72479, "tokens": 1017118720, "loss": 4.512788891792297, "l_mix": 2.7621361911296844, "l_a": 2.922786206007004, "l_c": 2.9076587855815887, "z_loss": 15.192557215690613, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.5050147175788879, "tokens_per_second": 1278734.7414378014, "eta_seconds": 28921.36268262863, "data_wait_seconds": 2.6106834411621094e-05, "mix": {"fineweb_hq": 0.50928, "dclm": 0.20773, "cosmopedia_v2": 0.20052, "finemath": 0.08247}}
{"step": 1950, "total_steps": 72479, "tokens": 1022361600, "loss": 4.498347878456116, "l_mix": 2.7528026700019836, "l_a": 2.9194235503673553, "l_c": 2.8941898941993713, "z_loss": 14.611059784889221, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.34560173749923706, "tokens_per_second": 1359711.7859189515, "eta_seconds": 27195.107621288298, "data_wait_seconds": 2.5779008865356445e-05, "mix": {"fineweb_hq": 0.50923, "dclm": 0.20769, "cosmopedia_v2": 0.20051, "finemath": 0.08256}}
{"step": 1960, "total_steps": 72479, "tokens": 1027604480, "loss": 5.105934202671051, "l_mix": 3.1418972611427307, "l_a": 3.270713448524475, "l_c": 3.2704287469387054, "z_loss": 16.94226908683777, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.35864442586898804, "tokens_per_second": 1219287.0742254644, "eta_seconds": 30322.85525989532, "data_wait_seconds": 2.7745962142944336e-05, "mix": {"fineweb_hq": 0.50969, "dclm": 0.20765, "cosmopedia_v2": 0.20051, "finemath": 0.08214}}
{"step": 1970, "total_steps": 72479, "tokens": 1032847360, "loss": 5.217660963535309, "l_mix": 3.2112796902656555, "l_a": 3.3542214930057526, "l_c": 3.328307420015335, "z_loss": 16.223255038261414, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.42813271284103394, "tokens_per_second": 1415726.537543557, "eta_seconds": 26111.697147488594, "data_wait_seconds": 1.7315149307250977e-05, "mix": {"fineweb_hq": 0.50964, "dclm": 0.20761, "cosmopedia_v2": 0.20051, "finemath": 0.08223}}
{"step": 1980, "total_steps": 72479, "tokens": 1038090240, "loss": 5.021475374698639, "l_mix": 3.0889495611190796, "l_a": 3.229501634836197, "l_c": 3.2068951427936554, "z_loss": 16.067182302474976, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.3435131907463074, "tokens_per_second": 1406182.350924026, "eta_seconds": 26285.196715569495, "data_wait_seconds": 1.677870750427246e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20758, "cosmopedia_v2": 0.20051, "finemath": 0.08232}}
{"step": 1990, "total_steps": 72479, "tokens": 1043333120, "loss": 5.059096336364746, "l_mix": 3.115629553794861, "l_a": 3.2491171956062317, "l_c": 3.2238443195819855, "z_loss": 15.782577753067017, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.5003516674041748, "tokens_per_second": 1296089.3734361187, "eta_seconds": 28513.879975748063, "data_wait_seconds": 1.6480684280395508e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20754, "cosmopedia_v2": 0.2005, "finemath": 0.08241}}
{"step": 2000, "total_steps": 72479, "tokens": 1048576000, "loss": 4.973346948623657, "l_mix": 3.058404326438904, "l_a": 3.199208289384842, "l_c": 3.178209066390991, "z_loss": 17.173276662826538, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.38523462414741516, "tokens_per_second": 1315519.9353912459, "eta_seconds": 28088.73735616207, "data_wait_seconds": 2.5004148483276367e-05, "mix": {"fineweb_hq": 0.5095, "dclm": 0.2075, "cosmopedia_v2": 0.2005, "finemath": 0.0825}, "gates": {"gate_a2b/L1_absmean": 0.04244411, "gate_a2b/L1_absmax": 0.33365342, "gate_a2b/L1_rms": 0.07055268, "gate_a2b/L3_absmean": 0.21619001, "gate_a2b/L3_absmax": 0.44272658, "gate_a2b/L3_rms": 0.22852901, "gate_a2b/L5_absmean": 0.31260312, "gate_a2b/L5_absmax": 0.52585131, "gate_a2b/L5_rms": 0.3250232, "gate_c2b/L1_absmean": 0.01317473, "gate_c2b/L1_absmax": 0.33058175, "gate_c2b/L1_rms": 0.03568581, "gate_c2b/L3_absmean": 0.23887502, "gate_c2b/L3_absmax": 0.52375329, "gate_c2b/L3_rms": 0.25052658, "gate_c2b/L5_absmean": 0.41834766, "gate_c2b/L5_absmax": 0.65519458, "gate_c2b/L5_rms": 0.43032801, "gate_b2a/L7_absmean": 0.02993659, "gate_b2a/L7_absmax": 0.14737007, "gate_b2a/L7_rms": 0.03955466, "gate_b2a/L11_absmean": 0.05251355, "gate_b2a/L11_absmax": 0.11803918, "gate_b2a/L11_rms": 0.05737824, "gate_b2a/L14_absmean": 0.15940446, "gate_b2a/L14_absmax": 0.20667265, "gate_b2a/L14_rms": 0.16048335, "gate_b2c/L3_absmean": 0.02200797, "gate_b2c/L3_absmax": 0.26236165, "gate_b2c/L3_rms": 0.03530058, "gate_b2c/L5_absmean": 0.05003598, "gate_b2c/L5_absmax": 0.15757279, "gate_b2c/L5_rms": 0.05294765, "gate_b2c/L6_absmean": 0.06788489, "gate_b2c/L6_absmax": 0.21005619, "gate_b2c/L6_rms": 0.07044592}, "streams": {"cos_a2b/round0": 0.317939, "cos_c2b/round0": 0.309517, "cos_b2a/round0": 0.148202, "cos_b2c/round0": 0.138342, "rms_a/round0": 1.063867, "rms_b/round0": 1.971725, "rms_c/round0": 1.074839, "cos_a2b/round1": 0.395033, "cos_c2b/round1": 0.346286, "cos_b2a/round1": 0.194149, "cos_b2c/round1": 0.20387, "rms_a/round1": 1.627348, "rms_b/round1": 2.923736, "rms_c/round1": 1.68512, "cos_a2b/round2": 0.250391, "cos_c2b/round2": 0.526174, "cos_b2a/round2": 0.13349, "cos_b2c/round2": 0.093956, "rms_a/round2": 2.482985, "rms_b/round2": 5.261709, "rms_c/round2": 2.349388}, "alpha_mean": 0.49805619594629036, "alpha_hist": [0.01123, 0.06133, 0.11424, 0.13779, 0.16733, 0.17767, 0.15695, 0.1144, 0.05414, 0.00493], "alpha_by_type": {"special": {"share": 0.00102, "mean": 0.58983, "std": 0.11482}, "whitespace": {"share": 0.02974, "mean": 0.58114, "std": 0.1169}, "digit": {"share": 0.01804, "mean": 0.52583, "std": 0.16951}, "word_start": {"share": 0.6082, "mean": 0.52017, "std": 0.18022}, "word_cont": {"share": 0.24338, "mean": 0.369, "std": 0.18062}, "punct": {"share": 0.09962, "mean": 0.64758, "std": 0.17236}}, "alpha_p05": 0.16, "alpha_p50": 0.5197, "alpha_p95": 0.81874}
{"step": 2010, "total_steps": 72479, "tokens": 1053818880, "loss": 4.1465219259262085, "l_mix": 2.547149509191513, "l_a": 2.706806570291519, "l_c": 2.618368446826935, "z_loss": 18.200464725494385, "lr": 0.0023, "wd": 0.1, "grad_norm": 2.6351640224456787, "tokens_per_second": 1330060.3683472206, "eta_seconds": 27777.724944853784, "data_wait_seconds": 2.6673078536987305e-05, "mix": {"fineweb_hq": 0.50945, "dclm": 0.20746, "cosmopedia_v2": 0.2005, "finemath": 0.08259}}
{"step": 2020, "total_steps": 72479, "tokens": 1059061760, "loss": 5.629587769508362, "l_mix": 3.473379135131836, "l_a": 3.586716443300247, "l_c": 3.5949330627918243, "z_loss": 17.137208461761475, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.25685831904411316, "tokens_per_second": 1296226.432714122, "eta_seconds": 28498.73082332611, "data_wait_seconds": 2.1189451217651367e-05, "mix": {"fineweb_hq": 0.50941, "dclm": 0.20792, "cosmopedia_v2": 0.2005, "finemath": 0.08218}}
{"step": 2030, "total_steps": 72479, "tokens": 1064304640, "loss": 5.678263425827026, "l_mix": 3.5057730078697205, "l_a": 3.6136117577552795, "l_c": 3.6223147809505463, "z_loss": 17.124151468276978, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2946218252182007, "tokens_per_second": 1452450.6794854377, "eta_seconds": 25429.82411291599, "data_wait_seconds": 1.537799835205078e-05, "mix": {"fineweb_hq": 0.50936, "dclm": 0.20788, "cosmopedia_v2": 0.20049, "finemath": 0.08227}}
{"step": 2040, "total_steps": 72479, "tokens": 1069547520, "loss": 5.66764098405838, "l_mix": 3.4991476833820343, "l_a": 3.600278615951538, "l_c": 3.6224358081817627, "z_loss": 16.788862943649292, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.3364436626434326, "tokens_per_second": 1343510.0478715855, "eta_seconds": 27487.939141583443, "data_wait_seconds": 2.1338462829589844e-05, "mix": {"fineweb_hq": 0.50931, "dclm": 0.20784, "cosmopedia_v2": 0.20049, "finemath": 0.08235}}
{"step": 2050, "total_steps": 72479, "tokens": 1074790400, "loss": 5.802862763404846, "l_mix": 3.583852082490921, "l_a": 3.6951574087142944, "l_c": 3.695880174636841, "z_loss": 16.992111682891846, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.3386134207248688, "tokens_per_second": 1433798.6221803888, "eta_seconds": 25753.323361301424, "data_wait_seconds": 2.7894973754882812e-05, "mix": {"fineweb_hq": 0.50927, "dclm": 0.2078, "cosmopedia_v2": 0.20049, "finemath": 0.08244}}
{"step": 2060, "total_steps": 72479, "tokens": 1080033280, "loss": 5.6385297775268555, "l_mix": 3.4774464070796967, "l_a": 3.593233436346054, "l_c": 3.6046937108039856, "z_loss": 17.052690505981445, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.3184490203857422, "tokens_per_second": 1430687.629446584, "eta_seconds": 25805.65870013237, "data_wait_seconds": 2.8133392333984375e-05, "mix": {"fineweb_hq": 0.50922, "dclm": 0.20777, "cosmopedia_v2": 0.20049, "finemath": 0.08252}}
{"step": 2070, "total_steps": 72479, "tokens": 1085276160, "loss": 4.706439852714539, "l_mix": 2.8913763761520386, "l_a": 3.028911054134369, "l_c": 3.015940636396408, "z_loss": 16.07848298549652, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.4543702006340027, "tokens_per_second": 1335781.7682186193, "eta_seconds": 27635.19810667038, "data_wait_seconds": 3.451108932495117e-05, "mix": {"fineweb_hq": 0.50966, "dclm": 0.20773, "cosmopedia_v2": 0.20048, "finemath": 0.08213}}
{"step": 2080, "total_steps": 72479, "tokens": 1090519040, "loss": 4.729339420795441, "l_mix": 2.9053634107112885, "l_a": 3.0524154007434845, "l_c": 3.0221785604953766, "z_loss": 15.977158784866333, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.305302232503891, "tokens_per_second": 1374968.3104431771, "eta_seconds": 26843.782966971397, "data_wait_seconds": 1.5944242477416992e-05, "mix": {"fineweb_hq": 0.50962, "dclm": 0.20769, "cosmopedia_v2": 0.20048, "finemath": 0.08221}}
{"step": 2090, "total_steps": 72479, "tokens": 1095761920, "loss": 4.895808100700378, "l_mix": 3.0118676126003265, "l_a": 3.1514020562171936, "l_c": 3.1226128339767456, "z_loss": 17.359028339385986, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.40692073106765747, "tokens_per_second": 1417400.5545477516, "eta_seconds": 26036.47071647644, "data_wait_seconds": 1.6421079635620117e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20048, "finemath": 0.0823}}
{"step": 2100, "total_steps": 72479, "tokens": 1101004800, "loss": 4.822641134262085, "l_mix": 2.9661462008953094, "l_a": 3.1083708703517914, "l_c": 3.0747106671333313, "z_loss": 15.704185843467712, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.3988726735115051, "tokens_per_second": 1439733.4573834892, "eta_seconds": 25628.955806207658, "data_wait_seconds": 2.4974346160888672e-05, "mix": {"fineweb_hq": 0.50952, "dclm": 0.20762, "cosmopedia_v2": 0.20048, "finemath": 0.08238}, "gates": {"gate_a2b/L1_absmean": 0.04910718, "gate_a2b/L1_absmax": 0.37392437, "gate_a2b/L1_rms": 0.07975682, "gate_a2b/L3_absmean": 0.22654144, "gate_a2b/L3_absmax": 0.45571855, "gate_a2b/L3_rms": 0.23916373, "gate_a2b/L5_absmean": 0.32509598, "gate_a2b/L5_absmax": 0.53760689, "gate_a2b/L5_rms": 0.33777726, "gate_c2b/L1_absmean": 0.01373624, "gate_c2b/L1_absmax": 0.36468503, "gate_c2b/L1_rms": 0.03984533, "gate_c2b/L3_absmean": 0.24483351, "gate_c2b/L3_absmax": 0.54615736, "gate_c2b/L3_rms": 0.2566787, "gate_c2b/L5_absmean": 0.43034002, "gate_c2b/L5_absmax": 0.67328274, "gate_c2b/L5_rms": 0.44279388, "gate_b2a/L7_absmean": 0.03137448, "gate_b2a/L7_absmax": 0.1541279, "gate_b2a/L7_rms": 0.04171378, "gate_b2a/L11_absmean": 0.05552986, "gate_b2a/L11_absmax": 0.12478487, "gate_b2a/L11_rms": 0.06043676, "gate_b2a/L14_absmean": 0.16815859, "gate_b2a/L14_absmax": 0.22465631, "gate_b2a/L14_rms": 0.16923711, "gate_b2c/L3_absmean": 0.02291609, "gate_b2c/L3_absmax": 0.27895907, "gate_b2c/L3_rms": 0.03663909, "gate_b2c/L5_absmean": 0.05333084, "gate_b2c/L5_absmax": 0.17897972, "gate_b2c/L5_rms": 0.05610967, "gate_b2c/L6_absmean": 0.07404374, "gate_b2c/L6_absmax": 0.21675019, "gate_b2c/L6_rms": 0.07625039}, "streams": {"cos_a2b/round0": 0.314402, "cos_c2b/round0": 0.297204, "cos_b2a/round0": 0.148029, "cos_b2c/round0": 0.135046, "rms_a/round0": 1.140299, "rms_b/round0": 2.090686, "rms_c/round0": 1.116805, "cos_a2b/round1": 0.390414, "cos_c2b/round1": 0.331345, "cos_b2a/round1": 0.1971, "cos_b2c/round1": 0.217172, "rms_a/round1": 1.752332, "rms_b/round1": 3.10956, "rms_c/round1": 1.761596, "cos_a2b/round2": 0.275424, "cos_c2b/round2": 0.491163, "cos_b2a/round2": 0.121372, "cos_b2c/round2": 0.116353, "rms_a/round2": 2.69528, "rms_b/round2": 5.616132, "rms_c/round2": 2.47031}, "alpha_mean": 0.5361598287561407, "alpha_hist": [0.01283, 0.04848, 0.09351, 0.1236, 0.14355, 0.16652, 0.16203, 0.13712, 0.09502, 0.01735], "alpha_by_type": {"special": {"share": 0.00087, "mean": 0.65144, "std": 0.09071}, "whitespace": {"share": 0.02332, "mean": 0.6274, "std": 0.11922}, "digit": {"share": 0.01361, "mean": 0.51463, "std": 0.17311}, "word_start": {"share": 0.6124, "mean": 0.57142, "std": 0.1852}, "word_cont": {"share": 0.26065, "mean": 0.38799, "std": 0.18947}, "punct": {"share": 0.08916, "mean": 0.70542, "std": 0.17495}}, "alpha_p05": 0.19193, "alpha_p50": 0.55266, "alpha_p95": 0.84847}
{"step": 2110, "total_steps": 72479, "tokens": 1106247680, "loss": 4.710659384727478, "l_mix": 2.8969951272010803, "l_a": 3.033079355955124, "l_c": 3.007103681564331, "z_loss": 16.092856407165527, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.36592400074005127, "tokens_per_second": 1368967.2266561184, "eta_seconds": 26949.96750369072, "data_wait_seconds": 2.4557113647460938e-05, "mix": {"fineweb_hq": 0.50948, "dclm": 0.20758, "cosmopedia_v2": 0.20047, "finemath": 0.08246}}
{"step": 2120, "total_steps": 72479, "tokens": 1111490560, "loss": 3.9958013892173767, "l_mix": 2.4601239562034607, "l_a": 2.561804562807083, "l_c": 2.5505323708057404, "z_loss": 19.762498140335083, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.40602725744247437, "tokens_per_second": 1364817.5573834856, "eta_seconds": 27028.066273355485, "data_wait_seconds": 3.5703182220458984e-05, "mix": {"fineweb_hq": 0.50943, "dclm": 0.20755, "cosmopedia_v2": 0.20047, "finemath": 0.08255}}
{"step": 2130, "total_steps": 72479, "tokens": 1116733440, "loss": 4.144042432308197, "l_mix": 2.5523498952388763, "l_a": 2.6564250886440277, "l_c": 2.642388552427292, "z_loss": 20.48344111442566, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.581527054309845, "tokens_per_second": 1349615.2382481454, "eta_seconds": 27328.63075840473, "data_wait_seconds": 2.828240394592285e-05, "mix": {"fineweb_hq": 0.50939, "dclm": 0.20751, "cosmopedia_v2": 0.20047, "finemath": 0.08263}}
{"step": 2140, "total_steps": 72479, "tokens": 1121976320, "loss": 4.721434533596039, "l_mix": 2.903700292110443, "l_a": 3.0377945601940155, "l_c": 3.0157600343227386, "z_loss": 16.67808485031128, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.23797142505645752, "tokens_per_second": 1429183.816995695, "eta_seconds": 25803.464322400094, "data_wait_seconds": 1.8358230590820312e-05, "mix": {"fineweb_hq": 0.50981, "dclm": 0.20748, "cosmopedia_v2": 0.20047, "finemath": 0.08224}}
{"step": 2150, "total_steps": 72479, "tokens": 1127219200, "loss": 4.707368791103363, "l_mix": 2.8963599801063538, "l_a": 3.0280668437480927, "l_c": 3.0032396018505096, "z_loss": 16.167267322540283, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.29560932517051697, "tokens_per_second": 1417735.5589152188, "eta_seconds": 26008.130021238325, "data_wait_seconds": 2.22623348236084e-05, "mix": {"fineweb_hq": 0.50977, "dclm": 0.20744, "cosmopedia_v2": 0.20047, "finemath": 0.08233}}
{"step": 2160, "total_steps": 72479, "tokens": 1132462080, "loss": 4.744737863540649, "l_mix": 2.9161168038845062, "l_a": 3.0526364743709564, "l_c": 3.037242740392685, "z_loss": 16.573478937149048, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.3346899449825287, "tokens_per_second": 1468215.4777732287, "eta_seconds": 25110.35228147507, "data_wait_seconds": 2.053380012512207e-05, "mix": {"fineweb_hq": 0.50972, "dclm": 0.20741, "cosmopedia_v2": 0.20046, "finemath": 0.08241}}
{"step": 2170, "total_steps": 72479, "tokens": 1137704960, "loss": 4.705306828022003, "l_mix": 2.893194228410721, "l_a": 3.0310228765010834, "l_c": 3.0036535263061523, "z_loss": 17.096809148788452, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.28299930691719055, "tokens_per_second": 1399763.9303376877, "eta_seconds": 26334.558415937423, "data_wait_seconds": 2.1964311599731445e-05, "mix": {"fineweb_hq": 0.50968, "dclm": 0.20737, "cosmopedia_v2": 0.20046, "finemath": 0.08249}}
{"step": 2180, "total_steps": 72479, "tokens": 1142947840, "loss": 4.599419713020325, "l_mix": 2.8279568552970886, "l_a": 2.962572991847992, "l_c": 2.9367562234401703, "z_loss": 16.641555786132812, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2904132306575775, "tokens_per_second": 1440721.335502451, "eta_seconds": 25582.26993920803, "data_wait_seconds": 1.9758939743041992e-05, "mix": {"fineweb_hq": 0.50963, "dclm": 0.2078, "cosmopedia_v2": 0.20046, "finemath": 0.08211}}
{"step": 2190, "total_steps": 72479, "tokens": 1148190720, "loss": 4.740397393703461, "l_mix": 2.916202276945114, "l_a": 3.0472367107868195, "l_c": 3.0278534591197968, "z_loss": 16.680331230163574, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.3028625547885895, "tokens_per_second": 1314066.020005622, "eta_seconds": 28044.00895462036, "data_wait_seconds": 2.5928020477294922e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20776, "cosmopedia_v2": 0.20046, "finemath": 0.08219}}
{"step": 2200, "total_steps": 72479, "tokens": 1153433600, "loss": 4.700819551944733, "l_mix": 2.890225738286972, "l_a": 3.0287086069583893, "l_c": 3.001135289669037, "z_loss": 16.405462980270386, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.37056785821914673, "tokens_per_second": 1234046.7227662825, "eta_seconds": 29858.21822807789, "data_wait_seconds": 2.2917985916137695e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20773, "cosmopedia_v2": 0.20045, "finemath": 0.08227}, "gates": {"gate_a2b/L1_absmean": 0.0528506, "gate_a2b/L1_absmax": 0.40682667, "gate_a2b/L1_rms": 0.08664211, "gate_a2b/L3_absmean": 0.23396246, "gate_a2b/L3_absmax": 0.46203548, "gate_a2b/L3_rms": 0.24653861, "gate_a2b/L5_absmean": 0.33719489, "gate_a2b/L5_absmax": 0.54632598, "gate_a2b/L5_rms": 0.35005894, "gate_c2b/L1_absmean": 0.01391905, "gate_c2b/L1_absmax": 0.38695136, "gate_c2b/L1_rms": 0.04187851, "gate_c2b/L3_absmean": 0.25169978, "gate_c2b/L3_absmax": 0.5532884, "gate_c2b/L3_rms": 0.26361078, "gate_c2b/L5_absmean": 0.44189906, "gate_c2b/L5_absmax": 0.69101787, "gate_c2b/L5_rms": 0.45467269, "gate_b2a/L7_absmean": 0.03299359, "gate_b2a/L7_absmax": 0.16339503, "gate_b2a/L7_rms": 0.04346631, "gate_b2a/L11_absmean": 0.06030551, "gate_b2a/L11_absmax": 0.13170694, "gate_b2a/L11_rms": 0.06483899, "gate_b2a/L14_absmean": 0.17697755, "gate_b2a/L14_absmax": 0.2435877, "gate_b2a/L14_rms": 0.17807482, "gate_b2c/L3_absmean": 0.02344322, "gate_b2c/L3_absmax": 0.28976113, "gate_b2c/L3_rms": 0.03766401, "gate_b2c/L5_absmean": 0.05544774, "gate_b2c/L5_absmax": 0.19151296, "gate_b2c/L5_rms": 0.05810585, "gate_b2c/L6_absmean": 0.07981336, "gate_b2c/L6_absmax": 0.22723167, "gate_b2c/L6_rms": 0.08180743}, "streams": {"cos_a2b/round0": 0.325749, "cos_c2b/round0": 0.294242, "cos_b2a/round0": 0.150713, "cos_b2c/round0": 0.137869, "rms_a/round0": 1.172613, "rms_b/round0": 2.13519, "rms_c/round0": 1.136583, "cos_a2b/round1": 0.392751, "cos_c2b/round1": 0.317156, "cos_b2a/round1": 0.19723, "cos_b2c/round1": 0.217862, "rms_a/round1": 1.819958, "rms_b/round1": 3.26139, "rms_c/round1": 1.803848, "cos_a2b/round2": 0.296535, "cos_c2b/round2": 0.476981, "cos_b2a/round2": 0.111757, "cos_b2c/round2": 0.110805, "rms_a/round2": 2.842232, "rms_b/round2": 5.588136, "rms_c/round2": 2.532915}, "alpha_mean": 0.5117661522228474, "alpha_hist": [0.01292, 0.05746, 0.11435, 0.14423, 0.15601, 0.14938, 0.14098, 0.12769, 0.08408, 0.01291], "alpha_by_type": {"special": {"share": 0.00105, "mean": 0.7113, "std": 0.10971}, "whitespace": {"share": 0.02393, "mean": 0.67439, "std": 0.1307}, "digit": {"share": 0.0209, "mean": 0.7007, "std": 0.15696}, "word_start": {"share": 0.59557, "mean": 0.52958, "std": 0.18836}, "word_cont": {"share": 0.25883, "mean": 0.37435, "std": 0.18592}, "punct": {"share": 0.09972, "mean": 0.68134, "std": 0.18503}}, "alpha_p05": 0.1538, "alpha_p50": 0.5219, "alpha_p95": 0.85824}
{"step": 2210, "total_steps": 72479, "tokens": 1158676480, "loss": 4.6880329847335815, "l_mix": 2.8835044503211975, "l_a": 3.0226535201072693, "l_c": 2.986837714910507, "z_loss": 16.81059432029724, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.41825971007347107, "tokens_per_second": 1330202.6148470214, "eta_seconds": 27695.92621514797, "data_wait_seconds": 2.1576881408691406e-05, "mix": {"fineweb_hq": 0.5095, "dclm": 0.20769, "cosmopedia_v2": 0.20045, "finemath": 0.08235}}
{"step": 2220, "total_steps": 72479, "tokens": 1163919360, "loss": 4.635028123855591, "l_mix": 2.8498655557632446, "l_a": 2.9896090030670166, "l_c": 2.9555834531784058, "z_loss": 16.047555685043335, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2808935344219208, "tokens_per_second": 1467241.6286223186, "eta_seconds": 25105.578981280327, "data_wait_seconds": 3.573298454284668e-05, "mix": {"fineweb_hq": 0.50946, "dclm": 0.20766, "cosmopedia_v2": 0.20045, "finemath": 0.08243}}
{"step": 2230, "total_steps": 72479, "tokens": 1169162240, "loss": 4.156103700399399, "l_mix": 2.5602395236492157, "l_a": 2.6448715925216675, "l_c": 2.668275773525238, "z_loss": 19.199142694473267, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.5139042735099792, "tokens_per_second": 1437115.3065029336, "eta_seconds": 25628.220328140258, "data_wait_seconds": 1.6063451766967773e-05, "mix": {"fineweb_hq": 0.50942, "dclm": 0.20762, "cosmopedia_v2": 0.20045, "finemath": 0.08251}}
{"step": 2240, "total_steps": 72479, "tokens": 1174405120, "loss": 4.084444612264633, "l_mix": 2.5152573585510254, "l_a": 2.618768870830536, "l_c": 2.605492115020752, "z_loss": 19.08907699584961, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.4049450159072876, "tokens_per_second": 1445835.4957420956, "eta_seconds": 25470.02403831482, "data_wait_seconds": 2.5600194931030273e-05, "mix": {"fineweb_hq": 0.50938, "dclm": 0.20759, "cosmopedia_v2": 0.20045, "finemath": 0.08259}}
{"step": 2250, "total_steps": 72479, "tokens": 1179648000, "loss": 4.63978636264801, "l_mix": 2.8538338243961334, "l_a": 2.9912229478359222, "l_c": 2.9563792049884796, "z_loss": 16.71903657913208, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2792530953884125, "tokens_per_second": 1446527.0237215424, "eta_seconds": 25454.223355793954, "data_wait_seconds": 2.530217170715332e-05, "mix": {"fineweb_hq": 0.50978, "dclm": 0.20756, "cosmopedia_v2": 0.20044, "finemath": 0.08222}}
{"step": 2260, "total_steps": 72479, "tokens": 1184890880, "loss": 4.638400614261627, "l_mix": 2.8547890186309814, "l_a": 2.978949695825577, "l_c": 2.9610415399074554, "z_loss": 16.14068078994751, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.35670778155326843, "tokens_per_second": 1419307.5127637899, "eta_seconds": 25938.69104540348, "data_wait_seconds": 1.5914440155029297e-05, "mix": {"fineweb_hq": 0.50973, "dclm": 0.20752, "cosmopedia_v2": 0.20044, "finemath": 0.0823}}
{"step": 2270, "total_steps": 72479, "tokens": 1190133760, "loss": 4.682765483856201, "l_mix": 2.880199670791626, "l_a": 3.012058138847351, "l_c": 2.990924686193466, "z_loss": 16.709614515304565, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2460559457540512, "tokens_per_second": 1410028.9867551653, "eta_seconds": 26105.65920116901, "data_wait_seconds": 1.963973045349121e-05, "mix": {"fineweb_hq": 0.50969, "dclm": 0.20749, "cosmopedia_v2": 0.20044, "finemath": 0.08238}}
{"step": 2280, "total_steps": 72479, "tokens": 1195376640, "loss": 4.647280335426331, "l_mix": 2.859428435564041, "l_a": 2.9924805760383606, "l_c": 2.9613450169563293, "z_loss": 17.04148769378662, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.3171254098415375, "tokens_per_second": 1425018.666673579, "eta_seconds": 25827.37628126144, "data_wait_seconds": 2.1517276763916016e-05, "mix": {"fineweb_hq": 0.50965, "dclm": 0.20746, "cosmopedia_v2": 0.20044, "finemath": 0.08246}}
{"step": 2290, "total_steps": 72479, "tokens": 1200619520, "loss": 4.5816091895103455, "l_mix": 2.8181035220623016, "l_a": 2.951197475194931, "l_c": 2.9214121401309967, "z_loss": 17.226440906524658, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.36701682209968567, "tokens_per_second": 1447591.3332603073, "eta_seconds": 25421.021517944333, "data_wait_seconds": 2.390146255493164e-05, "mix": {"fineweb_hq": 0.50961, "dclm": 0.20742, "cosmopedia_v2": 0.20044, "finemath": 0.08253}}
{"step": 2300, "total_steps": 72479, "tokens": 1205862400, "loss": 5.767224609851837, "l_mix": 3.563264101743698, "l_a": 3.6626115441322327, "l_c": 3.678186058998108, "z_loss": 17.21025276184082, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.36557644605636597, "tokens_per_second": 1417903.577613377, "eta_seconds": 25949.58369026184, "data_wait_seconds": 4.309415817260742e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20783, "cosmopedia_v2": 0.20043, "finemath": 0.08217}, "gates": {"gate_a2b/L1_absmean": 0.05635227, "gate_a2b/L1_absmax": 0.42840394, "gate_a2b/L1_rms": 0.09260914, "gate_a2b/L3_absmean": 0.24014059, "gate_a2b/L3_absmax": 0.47026584, "gate_a2b/L3_rms": 0.2528967, "gate_a2b/L5_absmean": 0.34681451, "gate_a2b/L5_absmax": 0.55202574, "gate_a2b/L5_rms": 0.35983893, "gate_c2b/L1_absmean": 0.01391844, "gate_c2b/L1_absmax": 0.41049349, "gate_c2b/L1_rms": 0.04345671, "gate_c2b/L3_absmean": 0.25527802, "gate_c2b/L3_absmax": 0.56501532, "gate_c2b/L3_rms": 0.26727453, "gate_c2b/L5_absmean": 0.45188388, "gate_c2b/L5_absmax": 0.69761789, "gate_c2b/L5_rms": 0.46496451, "gate_b2a/L7_absmean": 0.03445926, "gate_b2a/L7_absmax": 0.17047486, "gate_b2a/L7_rms": 0.04493363, "gate_b2a/L11_absmean": 0.06319848, "gate_b2a/L11_absmax": 0.13524242, "gate_b2a/L11_rms": 0.06764991, "gate_b2a/L14_absmean": 0.18461579, "gate_b2a/L14_absmax": 0.25989413, "gate_b2a/L14_rms": 0.1856432, "gate_b2c/L3_absmean": 0.02398475, "gate_b2c/L3_absmax": 0.29808116, "gate_b2c/L3_rms": 0.03862439, "gate_b2c/L5_absmean": 0.05784024, "gate_b2c/L5_absmax": 0.20746048, "gate_b2c/L5_rms": 0.06052282, "gate_b2c/L6_absmean": 0.08509214, "gate_b2c/L6_absmax": 0.23655201, "gate_b2c/L6_rms": 0.08692213}, "streams": {"cos_a2b/round0": 0.335689, "cos_c2b/round0": 0.292737, "cos_b2a/round0": 0.153464, "cos_b2c/round0": 0.14137, "rms_a/round0": 1.198829, "rms_b/round0": 2.185497, "rms_c/round0": 1.142837, "cos_a2b/round1": 0.405959, "cos_c2b/round1": 0.351074, "cos_b2a/round1": 0.204649, "cos_b2c/round1": 0.225861, "rms_a/round1": 1.882198, "rms_b/round1": 3.314849, "rms_c/round1": 1.83899, "cos_a2b/round2": 0.331086, "cos_c2b/round2": 0.46323, "cos_b2a/round2": 0.124642, "cos_b2c/round2": 0.146287, "rms_a/round2": 2.991586, "rms_b/round2": 5.585664, "rms_c/round2": 2.573462}, "alpha_mean": 0.563509508056768, "alpha_hist": [0.00395, 0.02194, 0.05736, 0.10229, 0.14821, 0.20589, 0.22884, 0.16052, 0.06728, 0.00372], "alpha_by_type": {"special": {"share": 0.00066, "mean": 0.6789, "std": 0.11413}, "whitespace": {"share": 0.04414, "mean": 0.61153, "std": 0.13944}, "digit": {"share": 0.01834, "mean": 0.58593, "std": 0.15734}, "word_start": {"share": 0.58528, "mean": 0.59005, "std": 0.15117}, "word_cont": {"share": 0.23901, "mean": 0.44177, "std": 0.1681}, "punct": {"share": 0.11256, "mean": 0.66086, "std": 0.16029}}, "alpha_p05": 0.26058, "alpha_p50": 0.57153, "alpha_p95": 0.82276}
{"step": 2310, "total_steps": 72479, "tokens": 1211105280, "loss": 5.6121296882629395, "l_mix": 3.4679121673107147, "l_a": 3.5720938444137573, "l_c": 3.569572329521179, "z_loss": 17.175447463989258, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2845194339752197, "tokens_per_second": 1331665.2200894137, "eta_seconds": 27626.136146688463, "data_wait_seconds": 1.683831214904785e-05, "mix": {"fineweb_hq": 0.50952, "dclm": 0.20779, "cosmopedia_v2": 0.20043, "finemath": 0.08225}}
{"step": 2320, "total_steps": 72479, "tokens": 1216348160, "loss": 5.667418539524078, "l_mix": 3.5011276602745056, "l_a": 3.6055166721343994, "l_c": 3.6097503900527954, "z_loss": 17.106778383255005, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.33672937750816345, "tokens_per_second": 1372026.9043906394, "eta_seconds": 26809.62135238647, "data_wait_seconds": 1.71661376953125e-05, "mix": {"fineweb_hq": 0.50948, "dclm": 0.20776, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 2330, "total_steps": 72479, "tokens": 1221591040, "loss": 5.617470920085907, "l_mix": 3.4704307913780212, "l_a": 3.5649569034576416, "l_c": 3.586161881685257, "z_loss": 17.04333519935608, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.3148016035556793, "tokens_per_second": 1402330.621776236, "eta_seconds": 26226.539120578767, "data_wait_seconds": 2.5570392608642578e-05, "mix": {"fineweb_hq": 0.50944, "dclm": 0.20773, "cosmopedia_v2": 0.20043, "finemath": 0.0824}}
{"step": 2340, "total_steps": 72479, "tokens": 1226833920, "loss": 5.667783617973328, "l_mix": 3.5040578842163086, "l_a": 3.60172501206398, "l_c": 3.6048357784748077, "z_loss": 17.573943376541138, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.3269079327583313, "tokens_per_second": 1421325.6151786405, "eta_seconds": 25872.351584529875, "data_wait_seconds": 3.057718276977539e-05, "mix": {"fineweb_hq": 0.5094, "dclm": 0.20769, "cosmopedia_v2": 0.20043, "finemath": 0.08248}}
{"step": 2350, "total_steps": 72479, "tokens": 1232076800, "loss": 3.810428112745285, "l_mix": 2.3498746752738953, "l_a": 2.4352546632289886, "l_c": 2.4267131090164185, "z_loss": 19.630233764648438, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.5020597577095032, "tokens_per_second": 1443316.370914864, "eta_seconds": 25474.520966386794, "data_wait_seconds": 2.6732683181762695e-05, "mix": {"fineweb_hq": 0.50936, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08255}}
{"step": 2360, "total_steps": 72479, "tokens": 1237319680, "loss": 4.656264841556549, "l_mix": 2.864670693874359, "l_a": 3.0020668506622314, "l_c": 2.964536875486374, "z_loss": 16.129056334495544, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.28447896242141724, "tokens_per_second": 1437641.0694419711, "eta_seconds": 25571.438555431363, "data_wait_seconds": 1.8477439880371094e-05, "mix": {"fineweb_hq": 0.50975, "dclm": 0.20763, "cosmopedia_v2": 0.20042, "finemath": 0.0822}}
{"step": 2370, "total_steps": 72479, "tokens": 1242562560, "loss": 4.59905070066452, "l_mix": 2.8302465975284576, "l_a": 2.9568188786506653, "l_c": 2.9336162209510803, "z_loss": 16.734991312026978, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.332559734582901, "tokens_per_second": 1434348.5305825442, "eta_seconds": 25626.48241224289, "data_wait_seconds": 1.659989356994629e-05, "mix": {"fineweb_hq": 0.5097, "dclm": 0.20759, "cosmopedia_v2": 0.20042, "finemath": 0.08228}}
{"step": 2380, "total_steps": 72479, "tokens": 1247805440, "loss": 4.65423721075058, "l_mix": 2.8640938997268677, "l_a": 2.9924414455890656, "l_c": 2.9691631197929382, "z_loss": 16.61852240562439, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.24744170904159546, "tokens_per_second": 1478680.3019084928, "eta_seconds": 24854.638602113722, "data_wait_seconds": 1.823902130126953e-05, "mix": {"fineweb_hq": 0.50966, "dclm": 0.20756, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 2390, "total_steps": 72479, "tokens": 1253048320, "loss": 4.572071015834808, "l_mix": 2.8135093450546265, "l_a": 2.941047251224518, "l_c": 2.9151906073093414, "z_loss": 16.903151035308838, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.27546706795692444, "tokens_per_second": 1431997.542871886, "eta_seconds": 25661.23232188225, "data_wait_seconds": 1.4960765838623047e-05, "mix": {"fineweb_hq": 0.50962, "dclm": 0.20753, "cosmopedia_v2": 0.20042, "finemath": 0.08243}}
{"step": 2400, "total_steps": 72479, "tokens": 1258291200, "loss": 4.623627483844757, "l_mix": 2.8446884155273438, "l_a": 2.9742259979248047, "l_c": 2.9501045644283295, "z_loss": 16.397727966308594, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.3643510639667511, "tokens_per_second": 1418404.5791162744, "eta_seconds": 25903.454693365096, "data_wait_seconds": 2.1129846572875977e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.2075, "cosmopedia_v2": 0.20042, "finemath": 0.0825}, "gates": {"gate_a2b/L1_absmean": 0.05991738, "gate_a2b/L1_absmax": 0.45729843, "gate_a2b/L1_rms": 0.0979458, "gate_a2b/L3_absmean": 0.24568538, "gate_a2b/L3_absmax": 0.47377774, "gate_a2b/L3_rms": 0.25845981, "gate_a2b/L5_absmean": 0.35454455, "gate_a2b/L5_absmax": 0.56612635, "gate_a2b/L5_rms": 0.36771733, "gate_c2b/L1_absmean": 0.01453769, "gate_c2b/L1_absmax": 0.43574107, "gate_c2b/L1_rms": 0.04542368, "gate_c2b/L3_absmean": 0.25847459, "gate_c2b/L3_absmax": 0.57216763, "gate_c2b/L3_rms": 0.27033693, "gate_c2b/L5_absmean": 0.45949197, "gate_c2b/L5_absmax": 0.71045822, "gate_c2b/L5_rms": 0.47296077, "gate_b2a/L7_absmean": 0.0353192, "gate_b2a/L7_absmax": 0.17865776, "gate_b2a/L7_rms": 0.04587425, "gate_b2a/L11_absmean": 0.06576995, "gate_b2a/L11_absmax": 0.13860318, "gate_b2a/L11_rms": 0.07013101, "gate_b2a/L14_absmean": 0.19075428, "gate_b2a/L14_absmax": 0.26987579, "gate_b2a/L14_rms": 0.19178534, "gate_b2c/L3_absmean": 0.02461023, "gate_b2c/L3_absmax": 0.305599, "gate_b2c/L3_rms": 0.0394493, "gate_b2c/L5_absmean": 0.05972983, "gate_b2c/L5_absmax": 0.22031373, "gate_b2c/L5_rms": 0.0624834, "gate_b2c/L6_absmean": 0.08884734, "gate_b2c/L6_absmax": 0.24389617, "gate_b2c/L6_rms": 0.0907103}, "streams": {"cos_a2b/round0": 0.328431, "cos_c2b/round0": 0.284539, "cos_b2a/round0": 0.153866, "cos_b2c/round0": 0.13388, "rms_a/round0": 1.240883, "rms_b/round0": 2.2706, "rms_c/round0": 1.1745, "cos_a2b/round1": 0.390701, "cos_c2b/round1": 0.314411, "cos_b2a/round1": 0.191349, "cos_b2c/round1": 0.214128, "rms_a/round1": 1.977083, "rms_b/round1": 3.502031, "rms_c/round1": 1.887615, "cos_a2b/round2": 0.315009, "cos_c2b/round2": 0.47116, "cos_b2a/round2": 0.091896, "cos_b2c/round2": 0.117435, "rms_a/round2": 3.150561, "rms_b/round2": 6.176569, "rms_c/round2": 2.719605}, "alpha_mean": 0.5082902121994834, "alpha_hist": [0.01401, 0.06436, 0.12361, 0.14107, 0.14503, 0.14244, 0.14146, 0.12941, 0.08734, 0.01126], "alpha_by_type": {"special": {"share": 0.00084, "mean": 0.65579, "std": 0.09186}, "whitespace": {"share": 0.02377, "mean": 0.6552, "std": 0.1205}, "digit": {"share": 0.01509, "mean": 0.50116, "std": 0.21425}, "word_start": {"share": 0.61436, "mean": 0.53066, "std": 0.1995}, "word_cont": {"share": 0.24522, "mean": 0.36989, "std": 0.18027}, "punct": {"share": 0.10071, "mean": 0.67397, "std": 0.19563}}, "alpha_p05": 0.1747, "alpha_p50": 0.51379, "alpha_p95": 0.84104}
{"step": 2410, "total_steps": 72479, "tokens": 1263534080, "loss": 5.546211659908295, "l_mix": 3.4281980991363525, "l_a": 3.524568885564804, "l_c": 3.5299288630485535, "z_loss": 16.641395807266235, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2922145426273346, "tokens_per_second": 1425092.0843757058, "eta_seconds": 25778.219018101692, "data_wait_seconds": 1.6123056411743164e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20788, "cosmopedia_v2": 0.20041, "finemath": 0.08216}}
{"step": 2420, "total_steps": 72479, "tokens": 1268776960, "loss": 5.559323310852051, "l_mix": 3.43680340051651, "l_a": 3.532117873430252, "l_c": 3.5373426377773285, "z_loss": 16.81695246696472, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2790813446044922, "tokens_per_second": 1430721.9770637979, "eta_seconds": 25673.11719596386, "data_wait_seconds": 3.2514333724975586e-05, "mix": {"fineweb_hq": 0.5095, "dclm": 0.20785, "cosmopedia_v2": 0.20041, "finemath": 0.08223}}
{"step": 2430, "total_steps": 72479, "tokens": 1274019840, "loss": 5.70593935251236, "l_mix": 3.525200366973877, "l_a": 3.627245754003525, "l_c": 3.6363291144371033, "z_loss": 16.664295434951782, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2936001420021057, "tokens_per_second": 1410522.5379438738, "eta_seconds": 26037.053024005887, "data_wait_seconds": 2.5510787963867188e-05, "mix": {"fineweb_hq": 0.50947, "dclm": 0.20782, "cosmopedia_v2": 0.20041, "finemath": 0.0823}}
{"step": 2440, "total_steps": 72479, "tokens": 1279262720, "loss": 5.645860731601715, "l_mix": 3.48820224404335, "l_a": 3.592505842447281, "l_c": 3.59407976269722, "z_loss": 16.825567960739136, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2833835184574127, "tokens_per_second": 1393542.5578276024, "eta_seconds": 26350.54597058296, "data_wait_seconds": 1.895427703857422e-05, "mix": {"fineweb_hq": 0.50943, "dclm": 0.20779, "cosmopedia_v2": 0.20041, "finemath": 0.08238}}
{"step": 2450, "total_steps": 72479, "tokens": 1284505600, "loss": 5.488640904426575, "l_mix": 3.39019638299942, "l_a": 3.4906692802906036, "l_c": 3.498547464609146, "z_loss": 16.793192148208618, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.3367616832256317, "tokens_per_second": 1421046.120279458, "eta_seconds": 25836.85626246929, "data_wait_seconds": 3.090500831604004e-05, "mix": {"fineweb_hq": 0.50939, "dclm": 0.20776, "cosmopedia_v2": 0.20041, "finemath": 0.08245}}
{"step": 2460, "total_steps": 72479, "tokens": 1289748480, "loss": 3.86846125125885, "l_mix": 2.383086070418358, "l_a": 2.4785002321004868, "l_c": 2.4662449955940247, "z_loss": 19.51638436317444, "lr": 0.0023, "wd": 0.1, "grad_norm": 1.0467431545257568, "tokens_per_second": 1410946.5423429948, "eta_seconds": 26018.081033062932, "data_wait_seconds": 1.722574234008789e-05, "mix": {"fineweb_hq": 0.50935, "dclm": 0.20772, "cosmopedia_v2": 0.20041, "finemath": 0.08252}}
{"step": 2470, "total_steps": 72479, "tokens": 1294991360, "loss": 4.658819258213043, "l_mix": 2.867401897907257, "l_a": 2.9991262555122375, "l_c": 2.966567486524582, "z_loss": 17.091325521469116, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.25720956921577454, "tokens_per_second": 1384241.1791923116, "eta_seconds": 26516.245249557498, "data_wait_seconds": 1.952052116394043e-05, "mix": {"fineweb_hq": 0.50972, "dclm": 0.20769, "cosmopedia_v2": 0.2004, "finemath": 0.08219}}
{"step": 2480, "total_steps": 72479, "tokens": 1300234240, "loss": 4.552320718765259, "l_mix": 2.8014542758464813, "l_a": 2.933699280023575, "l_c": 2.8970176577568054, "z_loss": 16.511658191680908, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2928479015827179, "tokens_per_second": 1428865.1990416113, "eta_seconds": 25684.463262605666, "data_wait_seconds": 2.1696090698242188e-05, "mix": {"fineweb_hq": 0.50968, "dclm": 0.20766, "cosmopedia_v2": 0.2004, "finemath": 0.08226}}
{"step": 2490, "total_steps": 72479, "tokens": 1305477120, "loss": 4.616571366786957, "l_mix": 2.841470330953598, "l_a": 2.9714220464229584, "l_c": 2.9400185644626617, "z_loss": 16.687687397003174, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2886890769004822, "tokens_per_second": 1344066.5532373693, "eta_seconds": 27301.023705720898, "data_wait_seconds": 1.755356788635254e-05, "mix": {"fineweb_hq": 0.50964, "dclm": 0.20763, "cosmopedia_v2": 0.2004, "finemath": 0.08233}}
{"step": 2500, "total_steps": 72479, "tokens": 1310720000, "loss": 4.5862279534339905, "l_mix": 2.8237110674381256, "l_a": 2.9543877243995667, "l_c": 2.9151352643966675, "z_loss": 16.599857568740845, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.35182151198387146, "tokens_per_second": 1373693.8446435533, "eta_seconds": 26708.389278340343, "data_wait_seconds": 2.2798776626586914e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.2076, "cosmopedia_v2": 0.2004, "finemath": 0.0824}, "gates": {"gate_a2b/L1_absmean": 0.06435408, "gate_a2b/L1_absmax": 0.48427922, "gate_a2b/L1_rms": 0.10335653, "gate_a2b/L3_absmean": 0.24852701, "gate_a2b/L3_absmax": 0.4730728, "gate_a2b/L3_rms": 0.26125601, "gate_a2b/L5_absmean": 0.3617931, "gate_a2b/L5_absmax": 0.57373196, "gate_a2b/L5_rms": 0.37498435, "gate_c2b/L1_absmean": 0.015098, "gate_c2b/L1_absmax": 0.46335083, "gate_c2b/L1_rms": 0.04760996, "gate_c2b/L3_absmean": 0.26071164, "gate_c2b/L3_absmax": 0.57269633, "gate_c2b/L3_rms": 0.27275538, "gate_c2b/L5_absmean": 0.46679011, "gate_c2b/L5_absmax": 0.72264004, "gate_c2b/L5_rms": 0.48064905, "gate_b2a/L7_absmean": 0.03613361, "gate_b2a/L7_absmax": 0.18733747, "gate_b2a/L7_rms": 0.04692521, "gate_b2a/L11_absmean": 0.06907403, "gate_b2a/L11_absmax": 0.14474393, "gate_b2a/L11_rms": 0.07311056, "gate_b2a/L14_absmean": 0.19691637, "gate_b2a/L14_absmax": 0.28166482, "gate_b2a/L14_rms": 0.19795157, "gate_b2c/L3_absmean": 0.02460577, "gate_b2c/L3_absmax": 0.31107479, "gate_b2c/L3_rms": 0.03979965, "gate_b2c/L5_absmean": 0.06065933, "gate_b2c/L5_absmax": 0.22946814, "gate_b2c/L5_rms": 0.0633645, "gate_b2c/L6_absmean": 0.09162281, "gate_b2c/L6_absmax": 0.24749462, "gate_b2c/L6_rms": 0.0932585}, "streams": {"cos_a2b/round0": 0.32927, "cos_c2b/round0": 0.278562, "cos_b2a/round0": 0.154645, "cos_b2c/round0": 0.1386, "rms_a/round0": 1.287127, "rms_b/round0": 2.359298, "rms_c/round0": 1.201295, "cos_a2b/round1": 0.400121, "cos_c2b/round1": 0.312061, "cos_b2a/round1": 0.195535, "cos_b2c/round1": 0.222233, "rms_a/round1": 2.056309, "rms_b/round1": 3.686988, "rms_c/round1": 1.944377, "cos_a2b/round2": 0.341242, "cos_c2b/round2": 0.463379, "cos_b2a/round2": 0.104257, "cos_b2c/round2": 0.139693, "rms_a/round2": 3.307179, "rms_b/round2": 6.420895, "rms_c/round2": 2.811836}, "alpha_mean": 0.5130451569509233, "alpha_hist": [0.01315, 0.05067, 0.10234, 0.14482, 0.16367, 0.15898, 0.15739, 0.13162, 0.07133, 0.00601], "alpha_by_type": {"special": {"share": 0.00079, "mean": 0.67881, "std": 0.0937}, "whitespace": {"share": 0.02621, "mean": 0.62467, "std": 0.13075}, "digit": {"share": 0.02216, "mean": 0.52282, "std": 0.16462}, "word_start": {"share": 0.6095, "mean": 0.53733, "std": 0.18575}, "word_cont": {"share": 0.24617, "mean": 0.38012, "std": 0.17536}, "punct": {"share": 0.09517, "mean": 0.66695, "std": 0.17992}}, "alpha_p05": 0.16776, "alpha_p50": 0.50685, "alpha_p95": 0.82922}
{"step": 2510, "total_steps": 72479, "tokens": 1315962880, "loss": 4.704711019992828, "l_mix": 2.895788699388504, "l_a": 3.019769459962845, "l_c": 3.0044564604759216, "z_loss": 16.54498267173767, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.4256422221660614, "tokens_per_second": 1426347.6348967468, "eta_seconds": 25718.770217370988, "data_wait_seconds": 2.6732683181762695e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20757, "cosmopedia_v2": 0.2004, "finemath": 0.08247}}
{"step": 2520, "total_steps": 72479, "tokens": 1321205760, "loss": 3.796196699142456, "l_mix": 2.337347701191902, "l_a": 2.4295609891414642, "l_c": 2.426859363913536, "z_loss": 19.22885012626648, "lr": 0.0023, "wd": 0.1, "grad_norm": 1.0617798566818237, "tokens_per_second": 1441489.3315198335, "eta_seconds": 25444.9779058218, "data_wait_seconds": 1.9669532775878906e-05, "mix": {"fineweb_hq": 0.50952, "dclm": 0.20754, "cosmopedia_v2": 0.2004, "finemath": 0.08254}}
{"step": 2530, "total_steps": 72479, "tokens": 1326448640, "loss": 5.497790336608887, "l_mix": 3.397385835647583, "l_a": 3.4940289855003357, "l_c": 3.5016313791275024, "z_loss": 17.06352686882019, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2781347632408142, "tokens_per_second": 1383581.7927293074, "eta_seconds": 26506.146224761007, "data_wait_seconds": 2.9712915420532227e-05, "mix": {"fineweb_hq": 0.50949, "dclm": 0.20791, "cosmopedia_v2": 0.2004, "finemath": 0.08221}}
{"step": 2540, "total_steps": 72479, "tokens": 1331691520, "loss": 5.572350978851318, "l_mix": 3.4449569284915924, "l_a": 3.5427035987377167, "l_c": 3.543024957180023, "z_loss": 16.753882884979248, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2389383316040039, "tokens_per_second": 1446120.643559626, "eta_seconds": 25356.237458682062, "data_wait_seconds": 1.5228986740112305e-05, "mix": {"fineweb_hq": 0.50945, "dclm": 0.20787, "cosmopedia_v2": 0.20039, "finemath": 0.08228}}
{"step": 2550, "total_steps": 72479, "tokens": 1336934400, "loss": 5.555274784564972, "l_mix": 3.433639705181122, "l_a": 3.534158080816269, "l_c": 3.5322408378124237, "z_loss": 17.15300941467285, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.24041016399860382, "tokens_per_second": 1311878.131778823, "eta_seconds": 27946.90654861927, "data_wait_seconds": 1.6868114471435547e-05, "mix": {"fineweb_hq": 0.50941, "dclm": 0.20784, "cosmopedia_v2": 0.20039, "finemath": 0.08235}}
{"step": 2560, "total_steps": 72479, "tokens": 1342177280, "loss": 5.727406978607178, "l_mix": 3.540097177028656, "l_a": 3.643816590309143, "l_c": 3.6415760219097137, "z_loss": 16.919147729873657, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.30231085419654846, "tokens_per_second": 1466679.5184086342, "eta_seconds": 24993.662359023092, "data_wait_seconds": 1.633167266845703e-05, "mix": {"fineweb_hq": 0.50938, "dclm": 0.20781, "cosmopedia_v2": 0.20039, "finemath": 0.08242}}
{"step": 2570, "total_steps": 72479, "tokens": 1347420160, "loss": 5.593443989753723, "l_mix": 3.4573274552822113, "l_a": 3.5535594820976257, "l_c": 3.561182200908661, "z_loss": 16.939923524856567, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2796071171760559, "tokens_per_second": 1420822.548410511, "eta_seconds": 25796.6414124012, "data_wait_seconds": 2.0325183868408203e-05, "mix": {"fineweb_hq": 0.50934, "dclm": 0.20778, "cosmopedia_v2": 0.20039, "finemath": 0.08249}}
{"step": 2580, "total_steps": 72479, "tokens": 1352663040, "loss": 4.496428072452545, "l_mix": 2.7685626745224, "l_a": 2.8952411115169525, "l_c": 2.8585260212421417, "z_loss": 17.352195739746094, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.329554945230484, "tokens_per_second": 1391471.0595126953, "eta_seconds": 26337.024159765246, "data_wait_seconds": 1.8835067749023438e-05, "mix": {"fineweb_hq": 0.50969, "dclm": 0.20775, "cosmopedia_v2": 0.20039, "finemath": 0.08217}}
{"step": 2590, "total_steps": 72479, "tokens": 1357905920, "loss": 4.414956450462341, "l_mix": 2.7181261479854584, "l_a": 2.8464765548706055, "l_c": 2.8039999902248383, "z_loss": 16.87294888496399, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.39715123176574707, "tokens_per_second": 1489504.2912727012, "eta_seconds": 24600.106388878823, "data_wait_seconds": 1.519918441772461e-05, "mix": {"fineweb_hq": 0.50965, "dclm": 0.20772, "cosmopedia_v2": 0.20039, "finemath": 0.08224}}
{"step": 2600, "total_steps": 72479, "tokens": 1363148800, "loss": 4.580801963806152, "l_mix": 2.8204130232334137, "l_a": 2.948806196451187, "l_c": 2.9133930802345276, "z_loss": 17.290042638778687, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.26965388655662537, "tokens_per_second": 1437893.4707785407, "eta_seconds": 25479.440512490273, "data_wait_seconds": 1.6033649444580078e-05, "mix": {"fineweb_hq": 0.50962, "dclm": 0.20769, "cosmopedia_v2": 0.20038, "finemath": 0.08231}, "gates": {"gate_a2b/L1_absmean": 0.06591325, "gate_a2b/L1_absmax": 0.51142269, "gate_a2b/L1_rms": 0.10626353, "gate_a2b/L3_absmean": 0.2507416, "gate_a2b/L3_absmax": 0.47539237, "gate_a2b/L3_rms": 0.26315263, "gate_a2b/L5_absmean": 0.36788526, "gate_a2b/L5_absmax": 0.58360004, "gate_a2b/L5_rms": 0.38117516, "gate_c2b/L1_absmean": 0.01471696, "gate_c2b/L1_absmax": 0.49407229, "gate_c2b/L1_rms": 0.04987093, "gate_c2b/L3_absmean": 0.26376072, "gate_c2b/L3_absmax": 0.58043993, "gate_c2b/L3_rms": 0.27576292, "gate_c2b/L5_absmean": 0.47336051, "gate_c2b/L5_absmax": 0.72815645, "gate_c2b/L5_rms": 0.487212, "gate_b2a/L7_absmean": 0.03744419, "gate_b2a/L7_absmax": 0.19215724, "gate_b2a/L7_rms": 0.04809964, "gate_b2a/L11_absmean": 0.07227064, "gate_b2a/L11_absmax": 0.15109161, "gate_b2a/L11_rms": 0.07613489, "gate_b2a/L14_absmean": 0.2027442, "gate_b2a/L14_absmax": 0.29457742, "gate_b2a/L14_rms": 0.20382455, "gate_b2c/L3_absmean": 0.0249271, "gate_b2c/L3_absmax": 0.31661227, "gate_b2c/L3_rms": 0.04017904, "gate_b2c/L5_absmean": 0.0618104, "gate_b2c/L5_absmax": 0.24017788, "gate_b2c/L5_rms": 0.06447277, "gate_b2c/L6_absmean": 0.09403853, "gate_b2c/L6_absmax": 0.25675353, "gate_b2c/L6_rms": 0.09560364}, "streams": {"cos_a2b/round0": 0.3291, "cos_c2b/round0": 0.277104, "cos_b2a/round0": 0.154765, "cos_b2c/round0": 0.137073, "rms_a/round0": 1.316483, "rms_b/round0": 2.412913, "rms_c/round0": 1.2243, "cos_a2b/round1": 0.39467, "cos_c2b/round1": 0.309373, "cos_b2a/round1": 0.197358, "cos_b2c/round1": 0.216026, "rms_a/round1": 2.113457, "rms_b/round1": 3.77058, "rms_c/round1": 1.992124, "cos_a2b/round2": 0.342422, "cos_c2b/round2": 0.465488, "cos_b2a/round2": 0.093009, "cos_b2c/round2": 0.133877, "rms_a/round2": 3.421283, "rms_b/round2": 6.614093, "rms_c/round2": 2.879483}, "alpha_mean": 0.49669553089705687, "alpha_hist": [0.01178, 0.05083, 0.10962, 0.15321, 0.17249, 0.18018, 0.15973, 0.11282, 0.0459, 0.00345], "alpha_by_type": {"special": {"share": 0.0009, "mean": 0.64081, "std": 0.10342}, "whitespace": {"share": 0.02776, "mean": 0.61085, "std": 0.11833}, "digit": {"share": 0.01859, "mean": 0.67325, "std": 0.14659}, "word_start": {"share": 0.5847, "mean": 0.50928, "std": 0.17669}, "word_cont": {"share": 0.25505, "mean": 0.39908, "std": 0.17645}, "punct": {"share": 0.11301, "mean": 0.59367, "std": 0.1942}}, "alpha_p05": 0.18832, "alpha_p50": 0.4926, "alpha_p95": 0.79311}
{"step": 2610, "total_steps": 72479, "tokens": 1368391680, "loss": 4.442593514919281, "l_mix": 2.734528362751007, "l_a": 2.8593307435512543, "l_c": 2.828595668077469, "z_loss": 16.87243628501892, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2502986192703247, "tokens_per_second": 1435356.6680339072, "eta_seconds": 25520.819380855562, "data_wait_seconds": 1.9937753677368164e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20038, "finemath": 0.08238}}
{"step": 2620, "total_steps": 72479, "tokens": 1373634560, "loss": 4.5924636125564575, "l_mix": 2.827498197555542, "l_a": 2.955115109682083, "l_c": 2.922558218240738, "z_loss": 16.633841514587402, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.32281917333602905, "tokens_per_second": 1444638.133833698, "eta_seconds": 25353.224820947646, "data_wait_seconds": 1.946091651916504e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20763, "cosmopedia_v2": 0.20038, "finemath": 0.08244}}
{"step": 2630, "total_steps": 72479, "tokens": 1378877440, "loss": 4.05973294377327, "l_mix": 2.5033033192157745, "l_a": 2.5851987302303314, "l_c": 2.5966121554374695, "z_loss": 18.86344289779663, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.9327920079231262, "tokens_per_second": 1427915.7601934671, "eta_seconds": 25646.465661978724, "data_wait_seconds": 2.059340476989746e-05, "mix": {"fineweb_hq": 0.50951, "dclm": 0.2076, "cosmopedia_v2": 0.20038, "finemath": 0.08251}}
{"step": 2640, "total_steps": 72479, "tokens": 1384120320, "loss": 3.893650323152542, "l_mix": 2.397730737924576, "l_a": 2.5146733820438385, "l_c": 2.465060591697693, "z_loss": 19.993542432785034, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.941082239151001, "tokens_per_second": 1422589.075389631, "eta_seconds": 25738.809797883034, "data_wait_seconds": 2.2023916244506836e-05, "mix": {"fineweb_hq": 0.50947, "dclm": 0.20758, "cosmopedia_v2": 0.20038, "finemath": 0.08258}}
{"step": 2650, "total_steps": 72479, "tokens": 1389363200, "loss": 4.461478888988495, "l_mix": 2.743541896343231, "l_a": 2.878806710243225, "l_c": 2.8420700132846832, "z_loss": 16.74013376235962, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.3140951097011566, "tokens_per_second": 1431979.6388532147, "eta_seconds": 25566.359855031966, "data_wait_seconds": 1.576542854309082e-05, "mix": {"fineweb_hq": 0.50981, "dclm": 0.20755, "cosmopedia_v2": 0.20038, "finemath": 0.08226}}
{"step": 2660, "total_steps": 72479, "tokens": 1394606080, "loss": 4.625954866409302, "l_mix": 2.8489815294742584, "l_a": 2.975825399160385, "l_c": 2.9417252242565155, "z_loss": 17.081342458724976, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2693294286727905, "tokens_per_second": 1458765.5362372699, "eta_seconds": 25093.315521025655, "data_wait_seconds": 1.901388168334961e-05, "mix": {"fineweb_hq": 0.50977, "dclm": 0.20752, "cosmopedia_v2": 0.20038, "finemath": 0.08233}}
{"step": 2670, "total_steps": 72479, "tokens": 1399848960, "loss": 4.5268471240997314, "l_mix": 2.7867060601711273, "l_a": 2.9118548929691315, "l_c": 2.8831210136413574, "z_loss": 16.482366800308228, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2643594741821289, "tokens_per_second": 1453882.5629371107, "eta_seconds": 25173.9871740818, "data_wait_seconds": 1.963973045349121e-05, "mix": {"fineweb_hq": 0.50974, "dclm": 0.20749, "cosmopedia_v2": 0.20037, "finemath": 0.0824}}
{"step": 2680, "total_steps": 72479, "tokens": 1405091840, "loss": 4.6672433614730835, "l_mix": 2.8758081793785095, "l_a": 2.999347984790802, "l_c": 2.9666034281253815, "z_loss": 16.49665904045105, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.3196388781070709, "tokens_per_second": 1387631.3538216818, "eta_seconds": 26372.118222332003, "data_wait_seconds": 1.7076730728149414e-05, "mix": {"fineweb_hq": 0.5097, "dclm": 0.20746, "cosmopedia_v2": 0.20037, "finemath": 0.08246}}
{"step": 2690, "total_steps": 72479, "tokens": 1410334720, "loss": 4.490809202194214, "l_mix": 2.764244884252548, "l_a": 2.8923275768756866, "l_c": 2.857379138469696, "z_loss": 16.521453857421875, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.25693339109420776, "tokens_per_second": 1422899.4680405213, "eta_seconds": 25714.771882224082, "data_wait_seconds": 1.77919864654541e-05, "mix": {"fineweb_hq": 0.50967, "dclm": 0.20743, "cosmopedia_v2": 0.20037, "finemath": 0.08253}}
{"step": 2700, "total_steps": 72479, "tokens": 1415577600, "loss": 4.430315136909485, "l_mix": 2.7277083694934845, "l_a": 2.8520488142967224, "l_c": 2.817736327648163, "z_loss": 16.711704969406128, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2502738833427429, "tokens_per_second": 1381678.5800691547, "eta_seconds": 26478.149751853944, "data_wait_seconds": 1.683831214904785e-05, "mix": {"fineweb_hq": 0.50963, "dclm": 0.20778, "cosmopedia_v2": 0.20037, "finemath": 0.08222}, "gates": {"gate_a2b/L1_absmean": 0.06959222, "gate_a2b/L1_absmax": 0.53179157, "gate_a2b/L1_rms": 0.11141229, "gate_a2b/L3_absmean": 0.25281224, "gate_a2b/L3_absmax": 0.4731774, "gate_a2b/L3_rms": 0.26520163, "gate_a2b/L5_absmean": 0.37227806, "gate_a2b/L5_absmax": 0.58319205, "gate_a2b/L5_rms": 0.38549176, "gate_c2b/L1_absmean": 0.01536324, "gate_c2b/L1_absmax": 0.50685048, "gate_c2b/L1_rms": 0.05108527, "gate_c2b/L3_absmean": 0.26551992, "gate_c2b/L3_absmax": 0.58844304, "gate_c2b/L3_rms": 0.27719784, "gate_c2b/L5_absmean": 0.47898695, "gate_c2b/L5_absmax": 0.73037094, "gate_c2b/L5_rms": 0.49315512, "gate_b2a/L7_absmean": 0.03841899, "gate_b2a/L7_absmax": 0.20135991, "gate_b2a/L7_rms": 0.04907658, "gate_b2a/L11_absmean": 0.07486825, "gate_b2a/L11_absmax": 0.1551795, "gate_b2a/L11_rms": 0.078539, "gate_b2a/L14_absmean": 0.20653167, "gate_b2a/L14_absmax": 0.3036679, "gate_b2a/L14_rms": 0.2075642, "gate_b2c/L3_absmean": 0.02496823, "gate_b2c/L3_absmax": 0.32174039, "gate_b2c/L3_rms": 0.04065631, "gate_b2c/L5_absmean": 0.06260022, "gate_b2c/L5_absmax": 0.2534903, "gate_b2c/L5_rms": 0.0653801, "gate_b2c/L6_absmean": 0.0964001, "gate_b2c/L6_absmax": 0.26566637, "gate_b2c/L6_rms": 0.09791247}, "streams": {"cos_a2b/round0": 0.332602, "cos_c2b/round0": 0.263747, "cos_b2a/round0": 0.159644, "cos_b2c/round0": 0.134328, "rms_a/round0": 1.345096, "rms_b/round0": 2.454113, "rms_c/round0": 1.241438, "cos_a2b/round1": 0.393428, "cos_c2b/round1": 0.297048, "cos_b2a/round1": 0.195614, "cos_b2c/round1": 0.220941, "rms_a/round1": 2.181135, "rms_b/round1": 3.887884, "rms_c/round1": 2.033896, "cos_a2b/round2": 0.348763, "cos_c2b/round2": 0.457264, "cos_b2a/round2": 0.083159, "cos_b2c/round2": 0.128401, "rms_a/round2": 3.559299, "rms_b/round2": 6.912583, "rms_c/round2": 2.988279}, "alpha_mean": 0.4911472879029475, "alpha_hist": [0.01428, 0.0661, 0.12665, 0.15344, 0.15503, 0.15533, 0.14317, 0.11826, 0.06252, 0.00522], "alpha_by_type": {"special": {"share": 0.0011, "mean": 0.64326, "std": 0.0891}, "whitespace": {"share": 0.02234, "mean": 0.62157, "std": 0.12108}, "digit": {"share": 0.01616, "mean": 0.57688, "std": 0.17041}, "word_start": {"share": 0.60925, "mean": 0.51296, "std": 0.18719}, "word_cont": {"share": 0.25496, "mean": 0.35424, "std": 0.17452}, "punct": {"share": 0.09619, "mean": 0.66941, "std": 0.17292}}, "alpha_p05": 0.15457, "alpha_p50": 0.46782, "alpha_p95": 0.80594}
{"step": 2710, "total_steps": 72479, "tokens": 1420820480, "loss": 4.583334028720856, "l_mix": 2.823854297399521, "l_a": 2.945069134235382, "l_c": 2.914342701435089, "z_loss": 16.5615451335907, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21350957453250885, "tokens_per_second": 1371990.5235401823, "eta_seconds": 26661.298926186562, "data_wait_seconds": 1.5556812286376953e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20775, "cosmopedia_v2": 0.20037, "finemath": 0.08229}}
{"step": 2720, "total_steps": 72479, "tokens": 1426063360, "loss": 4.446316659450531, "l_mix": 2.738215535879135, "l_a": 2.858394831418991, "l_c": 2.829685777425766, "z_loss": 16.7673602104187, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.30907782912254333, "tokens_per_second": 1487789.9040632609, "eta_seconds": 24582.64200618267, "data_wait_seconds": 2.0444393157958984e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20772, "cosmopedia_v2": 0.20037, "finemath": 0.08235}}
{"step": 2730, "total_steps": 72479, "tokens": 1431306240, "loss": 4.353112101554871, "l_mix": 2.678156226873398, "l_a": 2.805430084466934, "l_c": 2.772281289100647, "z_loss": 16.423709630966187, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.30324578285217285, "tokens_per_second": 1354133.0583008772, "eta_seconds": 27005.14804496765, "data_wait_seconds": 1.6242265701293945e-05, "mix": {"fineweb_hq": 0.50952, "dclm": 0.20769, "cosmopedia_v2": 0.20037, "finemath": 0.08242}}
{"step": 2740, "total_steps": 72479, "tokens": 1436549120, "loss": 3.493220806121826, "l_mix": 2.154197543859482, "l_a": 2.2369906306266785, "l_c": 2.2195246815681458, "z_loss": 20.686419010162354, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.5118781924247742, "tokens_per_second": 1428374.595461831, "eta_seconds": 25597.851535701753, "data_wait_seconds": 2.917647361755371e-05, "mix": {"fineweb_hq": 0.50949, "dclm": 0.20766, "cosmopedia_v2": 0.20036, "finemath": 0.08248}}
{"step": 2750, "total_steps": 72479, "tokens": 1441792000, "loss": 3.980578273534775, "l_mix": 2.4560849964618683, "l_a": 2.5417443215847015, "l_c": 2.5333937406539917, "z_loss": 19.517320156097412, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.5391130447387695, "tokens_per_second": 1469122.8945062666, "eta_seconds": 24884.288502144816, "data_wait_seconds": 1.5735626220703125e-05, "mix": {"fineweb_hq": 0.50945, "dclm": 0.20764, "cosmopedia_v2": 0.20036, "finemath": 0.08255}}
{"step": 2760, "total_steps": 72479, "tokens": 1447034880, "loss": 4.429271697998047, "l_mix": 2.725927412509918, "l_a": 2.8569826185703278, "l_c": 2.8150604963302612, "z_loss": 17.312649488449097, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2331305891275406, "tokens_per_second": 1370133.0451486153, "eta_seconds": 26678.310695028304, "data_wait_seconds": 2.828240394592285e-05, "mix": {"fineweb_hq": 0.50978, "dclm": 0.20761, "cosmopedia_v2": 0.20036, "finemath": 0.08225}}
{"step": 2770, "total_steps": 72479, "tokens": 1452277760, "loss": 4.525601446628571, "l_mix": 2.788898378610611, "l_a": 2.9100639820098877, "l_c": 2.8734282851219177, "z_loss": 16.55302119255066, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.25552549958229065, "tokens_per_second": 1355249.2646522373, "eta_seconds": 26967.4318557024, "data_wait_seconds": 1.5497207641601562e-05, "mix": {"fineweb_hq": 0.50975, "dclm": 0.20758, "cosmopedia_v2": 0.20036, "finemath": 0.08231}}
{"step": 2780, "total_steps": 72479, "tokens": 1457520640, "loss": 4.524087727069855, "l_mix": 2.785662442445755, "l_a": 2.912101089954376, "l_c": 2.87702077627182, "z_loss": 16.887515544891357, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2735183835029602, "tokens_per_second": 1301254.0936269343, "eta_seconds": 28082.408724761008, "data_wait_seconds": 2.568960189819336e-05, "mix": {"fineweb_hq": 0.50971, "dclm": 0.20755, "cosmopedia_v2": 0.20036, "finemath": 0.08237}}
{"step": 2790, "total_steps": 72479, "tokens": 1462763520, "loss": 4.434198439121246, "l_mix": 2.7309573590755463, "l_a": 2.856227219104767, "l_c": 2.8156650066375732, "z_loss": 16.73189377784729, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.32397332787513733, "tokens_per_second": 1342665.3729541819, "eta_seconds": 27212.369640254972, "data_wait_seconds": 2.321600914001465e-05, "mix": {"fineweb_hq": 0.50968, "dclm": 0.20753, "cosmopedia_v2": 0.20036, "finemath": 0.08244}}
{"step": 2800, "total_steps": 72479, "tokens": 1468006400, "loss": 4.424098074436188, "l_mix": 2.723790407180786, "l_a": 2.8502045571804047, "l_c": 2.811922550201416, "z_loss": 16.694617986679077, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.33435893058776855, "tokens_per_second": 1407170.6299220056, "eta_seconds": 25961.218046474456, "data_wait_seconds": 1.7642974853515625e-05, "mix": {"fineweb_hq": 0.50964, "dclm": 0.2075, "cosmopedia_v2": 0.20036, "finemath": 0.0825}, "gates": {"gate_a2b/L1_absmean": 0.07258614, "gate_a2b/L1_absmax": 0.54430592, "gate_a2b/L1_rms": 0.11482493, "gate_a2b/L3_absmean": 0.25663841, "gate_a2b/L3_absmax": 0.47538388, "gate_a2b/L3_rms": 0.26907066, "gate_a2b/L5_absmean": 0.37695691, "gate_a2b/L5_absmax": 0.58826143, "gate_a2b/L5_rms": 0.39013913, "gate_c2b/L1_absmean": 0.01612028, "gate_c2b/L1_absmax": 0.51799828, "gate_c2b/L1_rms": 0.05280871, "gate_c2b/L3_absmean": 0.26645279, "gate_c2b/L3_absmax": 0.58623874, "gate_c2b/L3_rms": 0.27813041, "gate_c2b/L5_absmean": 0.48421082, "gate_c2b/L5_absmax": 0.73215544, "gate_c2b/L5_rms": 0.49880365, "gate_b2a/L7_absmean": 0.03940712, "gate_b2a/L7_absmax": 0.20416939, "gate_b2a/L7_rms": 0.04991486, "gate_b2a/L11_absmean": 0.07778719, "gate_b2a/L11_absmax": 0.15832908, "gate_b2a/L11_rms": 0.08123762, "gate_b2a/L14_absmean": 0.2113868, "gate_b2a/L14_absmax": 0.31163722, "gate_b2a/L14_rms": 0.21239758, "gate_b2c/L3_absmean": 0.02497711, "gate_b2c/L3_absmax": 0.32535309, "gate_b2c/L3_rms": 0.04102061, "gate_b2c/L5_absmean": 0.06481525, "gate_b2c/L5_absmax": 0.2587055, "gate_b2c/L5_rms": 0.06757182, "gate_b2c/L6_absmean": 0.09969728, "gate_b2c/L6_absmax": 0.26699939, "gate_b2c/L6_rms": 0.10111914}, "streams": {"cos_a2b/round0": 0.328294, "cos_c2b/round0": 0.261408, "cos_b2a/round0": 0.156599, "cos_b2c/round0": 0.131535, "rms_a/round0": 1.365685, "rms_b/round0": 2.487494, "rms_c/round0": 1.252113, "cos_a2b/round1": 0.391624, "cos_c2b/round1": 0.2875, "cos_b2a/round1": 0.184624, "cos_b2c/round1": 0.204351, "rms_a/round1": 2.224341, "rms_b/round1": 3.982636, "rms_c/round1": 2.043101, "cos_a2b/round2": 0.354553, "cos_c2b/round2": 0.451525, "cos_b2a/round2": 0.077921, "cos_b2c/round2": 0.118892, "rms_a/round2": 3.63834, "rms_b/round2": 7.222624, "rms_c/round2": 3.01491}, "alpha_mean": 0.5237004133562095, "alpha_hist": [0.00977, 0.04839, 0.10649, 0.14674, 0.15419, 0.15352, 0.14311, 0.12793, 0.09122, 0.01865], "alpha_by_type": {"special": {"share": 0.00096, "mean": 0.69852, "std": 0.09507}, "whitespace": {"share": 0.02345, "mean": 0.65385, "std": 0.13224}, "digit": {"share": 0.01598, "mean": 0.56064, "std": 0.16187}, "word_start": {"share": 0.60152, "mean": 0.54547, "std": 0.19378}, "word_cont": {"share": 0.26381, "mean": 0.39871, "std": 0.18722}, "punct": {"share": 0.09428, "mean": 0.69413, "std": 0.18004}}, "alpha_p05": 0.18654, "alpha_p50": 0.53753, "alpha_p95": 0.85244}
{"step": 2810, "total_steps": 72479, "tokens": 1473249280, "loss": 5.4219611287117, "l_mix": 3.352444887161255, "l_a": 3.4495637118816376, "l_c": 3.4432685375213623, "z_loss": 16.664957284927368, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20305566489696503, "tokens_per_second": 1391163.7525090817, "eta_seconds": 26256.16186888218, "data_wait_seconds": 1.868605613708496e-05, "mix": {"fineweb_hq": 0.50961, "dclm": 0.20783, "cosmopedia_v2": 0.20036, "finemath": 0.08221}}
{"step": 2820, "total_steps": 72479, "tokens": 1478492160, "loss": 5.513869285583496, "l_mix": 3.4092216193675995, "l_a": 3.505181699991226, "l_c": 3.504687398672104, "z_loss": 16.86665177345276, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2574259340763092, "tokens_per_second": 1430604.7923855938, "eta_seconds": 25528.628162288667, "data_wait_seconds": 1.5676021575927734e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.2078, "cosmopedia_v2": 0.20035, "finemath": 0.08227}}
{"step": 2830, "total_steps": 72479, "tokens": 1483735040, "loss": 5.517089009284973, "l_mix": 3.411821961402893, "l_a": 3.5063067078590393, "l_c": 3.5056701600551605, "z_loss": 16.7401602268219, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2808968722820282, "tokens_per_second": 1355745.7429337201, "eta_seconds": 26934.35336406231, "data_wait_seconds": 2.4646520614624023e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20777, "cosmopedia_v2": 0.20035, "finemath": 0.08233}}
{"step": 2840, "total_steps": 72479, "tokens": 1488977920, "loss": 5.445261716842651, "l_mix": 3.367079973220825, "l_a": 3.45940962433815, "l_c": 3.462270051240921, "z_loss": 16.77736520767212, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.23250772058963776, "tokens_per_second": 1327285.3393993876, "eta_seconds": 27507.944937086108, "data_wait_seconds": 2.1517276763916016e-05, "mix": {"fineweb_hq": 0.50951, "dclm": 0.20775, "cosmopedia_v2": 0.20035, "finemath": 0.08239}}
{"step": 2850, "total_steps": 72479, "tokens": 1494220800, "loss": 5.286056995391846, "l_mix": 3.267245799303055, "l_a": 3.361861437559128, "l_c": 3.362043708562851, "z_loss": 16.396783709526062, "lr": 0.0023, "wd": 0.1, "grad_norm": 1.3621892929077148, "tokens_per_second": 1390584.2792359823, "eta_seconds": 26252.022043609617, "data_wait_seconds": 1.5854835510253906e-05, "mix": {"fineweb_hq": 0.50947, "dclm": 0.20772, "cosmopedia_v2": 0.20035, "finemath": 0.08246}}
{"step": 2860, "total_steps": 72479, "tokens": 1499463680, "loss": 3.7612246572971344, "l_mix": 2.320985347032547, "l_a": 2.39956995844841, "l_c": 2.394774705171585, "z_loss": 19.358817100524902, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.3868440091609955, "tokens_per_second": 1475664.5792521473, "eta_seconds": 24734.893542337417, "data_wait_seconds": 1.564621925354004e-05, "mix": {"fineweb_hq": 0.50944, "dclm": 0.20769, "cosmopedia_v2": 0.20035, "finemath": 0.08252}}
{"step": 2870, "total_steps": 72479, "tokens": 1504706560, "loss": 5.0625070333480835, "l_mix": 3.1263259649276733, "l_a": 3.2324013113975525, "l_c": 3.215961217880249, "z_loss": 16.723408699035645, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20357148349285126, "tokens_per_second": 1398430.8331926863, "eta_seconds": 26097.224493169782, "data_wait_seconds": 2.2858381271362305e-05, "mix": {"fineweb_hq": 0.50976, "dclm": 0.20767, "cosmopedia_v2": 0.20035, "finemath": 0.08223}}
{"step": 2880, "total_steps": 72479, "tokens": 1509949440, "loss": 5.113283336162567, "l_mix": 3.15749192237854, "l_a": 3.2639825344085693, "l_c": 3.2497077882289886, "z_loss": 16.843175172805786, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2546347677707672, "tokens_per_second": 1434391.1941653315, "eta_seconds": 25439.308788585662, "data_wait_seconds": 2.244114875793457e-05, "mix": {"fineweb_hq": 0.50972, "dclm": 0.20764, "cosmopedia_v2": 0.20035, "finemath": 0.08229}}
{"step": 2890, "total_steps": 72479, "tokens": 1515192320, "loss": 5.085122764110565, "l_mix": 3.1396536231040955, "l_a": 3.250643312931061, "l_c": 3.2287698686122894, "z_loss": 16.451712131500244, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.24292917549610138, "tokens_per_second": 1388944.9836764373, "eta_seconds": 26267.90698032379, "data_wait_seconds": 1.5169382095336914e-05, "mix": {"fineweb_hq": 0.50969, "dclm": 0.20761, "cosmopedia_v2": 0.20035, "finemath": 0.08235}}
{"step": 2900, "total_steps": 72479, "tokens": 1520435200, "loss": 5.0344120264053345, "l_mix": 3.109256863594055, "l_a": 3.2135225534439087, "l_c": 3.198111653327942, "z_loss": 16.647177696228027, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.24809852242469788, "tokens_per_second": 1398860.9458197432, "eta_seconds": 26077.956398034097, "data_wait_seconds": 2.5153160095214844e-05, "mix": {"fineweb_hq": 0.50966, "dclm": 0.20759, "cosmopedia_v2": 0.20034, "finemath": 0.08241}, "gates": {"gate_a2b/L1_absmean": 0.07492574, "gate_a2b/L1_absmax": 0.56555516, "gate_a2b/L1_rms": 0.11828066, "gate_a2b/L3_absmean": 0.25550634, "gate_a2b/L3_absmax": 0.46571061, "gate_a2b/L3_rms": 0.26757994, "gate_a2b/L5_absmean": 0.3792344, "gate_a2b/L5_absmax": 0.59351087, "gate_a2b/L5_rms": 0.39234248, "gate_c2b/L1_absmean": 0.01620478, "gate_c2b/L1_absmax": 0.52284008, "gate_c2b/L1_rms": 0.05363826, "gate_c2b/L3_absmean": 0.26733884, "gate_c2b/L3_absmax": 0.58732551, "gate_c2b/L3_rms": 0.27898404, "gate_c2b/L5_absmean": 0.49014217, "gate_c2b/L5_absmax": 0.73047435, "gate_c2b/L5_rms": 0.50483471, "gate_b2a/L7_absmean": 0.04039579, "gate_b2a/L7_absmax": 0.21174516, "gate_b2a/L7_rms": 0.05105071, "gate_b2a/L11_absmean": 0.0800795, "gate_b2a/L11_absmax": 0.16449162, "gate_b2a/L11_rms": 0.08334727, "gate_b2a/L14_absmean": 0.21526769, "gate_b2a/L14_absmax": 0.322256, "gate_b2a/L14_rms": 0.21626587, "gate_b2c/L3_absmean": 0.02515966, "gate_b2c/L3_absmax": 0.33161822, "gate_b2c/L3_rms": 0.04113022, "gate_b2c/L5_absmean": 0.0633898, "gate_b2c/L5_absmax": 0.26743403, "gate_b2c/L5_rms": 0.06627478, "gate_b2c/L6_absmean": 0.10032491, "gate_b2c/L6_absmax": 0.27067688, "gate_b2c/L6_rms": 0.1017473}, "streams": {"cos_a2b/round0": 0.359648, "cos_c2b/round0": 0.282084, "cos_b2a/round0": 0.170001, "cos_b2c/round0": 0.145545, "rms_a/round0": 1.407495, "rms_b/round0": 2.585248, "rms_c/round0": 1.261506, "cos_a2b/round1": 0.405444, "cos_c2b/round1": 0.306188, "cos_b2a/round1": 0.203033, "cos_b2c/round1": 0.216154, "rms_a/round1": 2.284367, "rms_b/round1": 4.056589, "rms_c/round1": 2.105277, "cos_a2b/round2": 0.364806, "cos_c2b/round2": 0.456002, "cos_b2a/round2": 0.101336, "cos_b2c/round2": 0.139557, "rms_a/round2": 3.789998, "rms_b/round2": 7.247336, "rms_c/round2": 3.091197}, "alpha_mean": 0.4987354892580669, "alpha_hist": [0.00676, 0.03497, 0.09634, 0.16153, 0.19969, 0.20312, 0.1566, 0.10728, 0.03276, 0.00093], "alpha_by_type": {"special": {"share": 0.00075, "mean": 0.62299, "std": 0.09999}, "whitespace": {"share": 0.04814, "mean": 0.58427, "std": 0.09904}, "digit": {"share": 0.05482, "mean": 0.55325, "std": 0.16336}, "word_start": {"share": 0.54337, "mean": 0.51653, "std": 0.15763}, "word_cont": {"share": 0.24365, "mean": 0.3816, "std": 0.16076}, "punct": {"share": 0.10927, "mean": 0.60554, "std": 0.16309}}, "alpha_p05": 0.20151, "alpha_p50": 0.47281, "alpha_p95": 0.77628}
{"step": 2910, "total_steps": 72479, "tokens": 1525678080, "loss": 5.177207350730896, "l_mix": 3.199295073747635, "l_a": 3.305973529815674, "l_c": 3.2815797924995422, "z_loss": 16.461219310760498, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.3048383593559265, "tokens_per_second": 1418500.7409165793, "eta_seconds": 25713.199027609822, "data_wait_seconds": 1.728534698486328e-05, "mix": {"fineweb_hq": 0.50962, "dclm": 0.20756, "cosmopedia_v2": 0.20034, "finemath": 0.08247}}
{"step": 2920, "total_steps": 72479, "tokens": 1530920960, "loss": 3.403494507074356, "l_mix": 2.0997557938098907, "l_a": 2.171478718519211, "l_c": 2.1681129038333893, "z_loss": 18.61053729057312, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.4227580726146698, "tokens_per_second": 1393901.1009749915, "eta_seconds": 26163.225616574287, "data_wait_seconds": 1.7374753952026367e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20753, "cosmopedia_v2": 0.20034, "finemath": 0.08253}}
{"step": 2930, "total_steps": 72479, "tokens": 1536163840, "loss": 5.425346732139587, "l_mix": 3.354238510131836, "l_a": 3.4546606838703156, "l_c": 3.443562299013138, "z_loss": 16.41210675239563, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.3536548912525177, "tokens_per_second": 1472456.322145663, "eta_seconds": 24763.862644743916, "data_wait_seconds": 2.0056962966918945e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20785, "cosmopedia_v2": 0.20034, "finemath": 0.08225}}
{"step": 2940, "total_steps": 72479, "tokens": 1541406720, "loss": 5.4874107837677, "l_mix": 3.394672930240631, "l_a": 3.4865134954452515, "l_c": 3.48381644487381, "z_loss": 16.386855244636536, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20979470014572144, "tokens_per_second": 1380407.0742584611, "eta_seconds": 26411.3853890419, "data_wait_seconds": 2.2858381271362305e-05, "mix": {"fineweb_hq": 0.50952, "dclm": 0.20782, "cosmopedia_v2": 0.20034, "finemath": 0.08231}}
{"step": 2950, "total_steps": 72479, "tokens": 1546649600, "loss": 5.539082109928131, "l_mix": 3.4270367324352264, "l_a": 3.5202331840991974, "l_c": 3.514378398656845, "z_loss": 16.61787223815918, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2561380863189697, "tokens_per_second": 1445773.517832859, "eta_seconds": 25213.645085048676, "data_wait_seconds": 1.615285873413086e-05, "mix": {"fineweb_hq": 0.50949, "dclm": 0.2078, "cosmopedia_v2": 0.20034, "finemath": 0.08237}}
{"step": 2960, "total_steps": 72479, "tokens": 1551892480, "loss": 5.351715624332428, "l_mix": 3.3088542819023132, "l_a": 3.4050393402576447, "l_c": 3.398938924074173, "z_loss": 16.678702116012573, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.26513054966926575, "tokens_per_second": 1396088.0588638845, "eta_seconds": 26107.219555807114, "data_wait_seconds": 2.3305416107177734e-05, "mix": {"fineweb_hq": 0.50946, "dclm": 0.20777, "cosmopedia_v2": 0.20034, "finemath": 0.08243}}
{"step": 2970, "total_steps": 72479, "tokens": 1557135360, "loss": 3.598728120326996, "l_mix": 2.219395786523819, "l_a": 2.3096534609794617, "l_c": 2.28161084651947, "z_loss": 19.530346155166626, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.270797461271286, "tokens_per_second": 1404962.6638168183, "eta_seconds": 25938.578675818444, "data_wait_seconds": 2.613663673400879e-05, "mix": {"fineweb_hq": 0.50943, "dclm": 0.20774, "cosmopedia_v2": 0.20034, "finemath": 0.08249}}
{"step": 2980, "total_steps": 72479, "tokens": 1562378240, "loss": 5.163679540157318, "l_mix": 3.1896926164627075, "l_a": 3.2918102741241455, "l_c": 3.2826773524284363, "z_loss": 16.405482292175293, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.24585384130477905, "tokens_per_second": 1272554.6235268845, "eta_seconds": 28633.341970825197, "data_wait_seconds": 2.0205974578857422e-05, "mix": {"fineweb_hq": 0.50973, "dclm": 0.20772, "cosmopedia_v2": 0.20034, "finemath": 0.08221}}
{"step": 2990, "total_steps": 72479, "tokens": 1567621120, "loss": 5.267079055309296, "l_mix": 3.2558424174785614, "l_a": 3.3526215851306915, "l_c": 3.346094310283661, "z_loss": 16.21766424179077, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.23688463866710663, "tokens_per_second": 1443936.0845083056, "eta_seconds": 25231.206022810933, "data_wait_seconds": 2.467632293701172e-05, "mix": {"fineweb_hq": 0.5097, "dclm": 0.20769, "cosmopedia_v2": 0.20033, "finemath": 0.08227}}
{"step": 3000, "total_steps": 72479, "tokens": 1572864000, "loss": 5.236153960227966, "l_mix": 3.2341074347496033, "l_a": 3.341888517141342, "l_c": 3.3260952830314636, "z_loss": 16.512389659881592, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21204644441604614, "tokens_per_second": 1428225.5135481572, "eta_seconds": 25505.079979634287, "data_wait_seconds": 1.7762184143066406e-05, "mix": {"fineweb_hq": 0.50967, "dclm": 0.20767, "cosmopedia_v2": 0.20033, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.07551225, "gate_a2b/L1_absmax": 0.56178254, "gate_a2b/L1_rms": 0.11944244, "gate_a2b/L3_absmean": 0.25631618, "gate_a2b/L3_absmax": 0.46812683, "gate_a2b/L3_rms": 0.26845774, "gate_a2b/L5_absmean": 0.38235641, "gate_a2b/L5_absmax": 0.59510946, "gate_a2b/L5_rms": 0.39545837, "gate_c2b/L1_absmean": 0.01534319, "gate_c2b/L1_absmax": 0.52166688, "gate_c2b/L1_rms": 0.05325873, "gate_c2b/L3_absmean": 0.26685178, "gate_c2b/L3_absmax": 0.58625835, "gate_c2b/L3_rms": 0.27840203, "gate_c2b/L5_absmean": 0.49509898, "gate_c2b/L5_absmax": 0.73597473, "gate_c2b/L5_rms": 0.51005715, "gate_b2a/L7_absmean": 0.04077877, "gate_b2a/L7_absmax": 0.21620579, "gate_b2a/L7_rms": 0.05173884, "gate_b2a/L11_absmean": 0.08117076, "gate_b2a/L11_absmax": 0.16759706, "gate_b2a/L11_rms": 0.08457989, "gate_b2a/L14_absmean": 0.21784127, "gate_b2a/L14_absmax": 0.33142149, "gate_b2a/L14_rms": 0.21883126, "gate_b2c/L3_absmean": 0.02512252, "gate_b2c/L3_absmax": 0.33599135, "gate_b2c/L3_rms": 0.04144759, "gate_b2c/L5_absmean": 0.06405391, "gate_b2c/L5_absmax": 0.2782875, "gate_b2c/L5_rms": 0.0669355, "gate_b2c/L6_absmean": 0.10191981, "gate_b2c/L6_absmax": 0.27825218, "gate_b2c/L6_rms": 0.10329732}, "streams": {"cos_a2b/round0": 0.343208, "cos_c2b/round0": 0.265474, "cos_b2a/round0": 0.162543, "cos_b2c/round0": 0.131476, "rms_a/round0": 1.428773, "rms_b/round0": 2.574372, "rms_c/round0": 1.279037, "cos_a2b/round1": 0.3954, "cos_c2b/round1": 0.288679, "cos_b2a/round1": 0.194345, "cos_b2c/round1": 0.216848, "rms_a/round1": 2.343197, "rms_b/round1": 4.108167, "rms_c/round1": 2.120099, "cos_a2b/round2": 0.390432, "cos_c2b/round2": 0.425346, "cos_b2a/round2": 0.090784, "cos_b2c/round2": 0.149464, "rms_a/round2": 3.898319, "rms_b/round2": 7.037075, "rms_c/round2": 3.067881}, "alpha_mean": 0.5318589941400376, "alpha_hist": [0.00871, 0.03448, 0.08195, 0.12733, 0.16708, 0.19228, 0.18477, 0.13361, 0.06468, 0.0051], "alpha_by_type": {"special": {"share": 0.00172, "mean": 0.65626, "std": 0.11275}, "whitespace": {"share": 0.02644, "mean": 0.6423, "std": 0.12051}, "digit": {"share": 0.03105, "mean": 0.68105, "std": 0.14278}, "word_start": {"share": 0.56133, "mean": 0.54808, "std": 0.16556}, "word_cont": {"share": 0.26285, "mean": 0.41082, "std": 0.17501}, "punct": {"share": 0.11661, "mean": 0.65999, "std": 0.16223}}, "alpha_p05": 0.22781, "alpha_p50": 0.54468, "alpha_p95": 0.81757}
{"step": 3010, "total_steps": 72479, "tokens": 1578106880, "loss": 5.229079067707062, "l_mix": 3.2309477627277374, "l_a": 3.337803691625595, "l_c": 3.3171659111976624, "z_loss": 16.402854919433594, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.25035780668258667, "tokens_per_second": 1435384.681671402, "eta_seconds": 25374.21747429371, "data_wait_seconds": 1.728534698486328e-05, "mix": {"fineweb_hq": 0.50963, "dclm": 0.20764, "cosmopedia_v2": 0.20033, "finemath": 0.08239}}
{"step": 3020, "total_steps": 72479, "tokens": 1583349760, "loss": 5.16353178024292, "l_mix": 3.1893128752708435, "l_a": 3.2900451123714447, "l_c": 3.2851125299930573, "z_loss": 16.71611452102661, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.3048975169658661, "tokens_per_second": 1478164.9354833874, "eta_seconds": 24636.303647732733, "data_wait_seconds": 1.582503318786621e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20762, "cosmopedia_v2": 0.20033, "finemath": 0.08245}}
{"step": 3030, "total_steps": 72479, "tokens": 1588592640, "loss": 3.595635026693344, "l_mix": 2.2192700803279877, "l_a": 2.3000894337892532, "l_c": 2.2810170650482178, "z_loss": 20.32901358604431, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.30812814831733704, "tokens_per_second": 1437737.131311282, "eta_seconds": 25325.406514883045, "data_wait_seconds": 1.6003847122192383e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20759, "cosmopedia_v2": 0.20033, "finemath": 0.08251}}
{"step": 3040, "total_steps": 72479, "tokens": 1593835520, "loss": 3.7409989833831787, "l_mix": 2.3089261949062347, "l_a": 2.389240950345993, "l_c": 2.377969264984131, "z_loss": 19.097159147262573, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.3701050281524658, "tokens_per_second": 1389111.073417755, "eta_seconds": 26208.15219795704, "data_wait_seconds": 2.3573637008666992e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20757, "cosmopedia_v2": 0.20033, "finemath": 0.08257}}
{"step": 3050, "total_steps": 72479, "tokens": 1599078400, "loss": 5.498292803764343, "l_mix": 3.4017960131168365, "l_a": 3.496167540550232, "l_c": 3.4865271747112274, "z_loss": 16.881388664245605, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19755788147449493, "tokens_per_second": 1383777.1654968031, "eta_seconds": 26305.385331988335, "data_wait_seconds": 2.8878450393676758e-05, "mix": {"fineweb_hq": 0.50951, "dclm": 0.20787, "cosmopedia_v2": 0.20033, "finemath": 0.0823}}
{"step": 3060, "total_steps": 72479, "tokens": 1604321280, "loss": 5.4027992486953735, "l_mix": 3.3408988416194916, "l_a": 3.439279556274414, "l_c": 3.4281128644943237, "z_loss": 16.825090169906616, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.26383668184280396, "tokens_per_second": 1318941.87765435, "eta_seconds": 27594.505329322816, "data_wait_seconds": 2.3126602172851562e-05, "mix": {"fineweb_hq": 0.50948, "dclm": 0.20784, "cosmopedia_v2": 0.20033, "finemath": 0.08235}}
{"step": 3070, "total_steps": 72479, "tokens": 1609564160, "loss": 5.28281044960022, "l_mix": 3.267151892185211, "l_a": 3.360591322183609, "l_c": 3.3528450429439545, "z_loss": 16.27501130104065, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.25878193974494934, "tokens_per_second": 1411772.2038487624, "eta_seconds": 25776.32970304489, "data_wait_seconds": 1.856684684753418e-05, "mix": {"fineweb_hq": 0.50945, "dclm": 0.20782, "cosmopedia_v2": 0.20033, "finemath": 0.08241}}
{"step": 3080, "total_steps": 72479, "tokens": 1614807040, "loss": 5.343330085277557, "l_mix": 3.304865062236786, "l_a": 3.4019645154476166, "l_c": 3.387438803911209, "z_loss": 16.4397873878479, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.32514622807502747, "tokens_per_second": 1439624.4993295572, "eta_seconds": 25273.995357084274, "data_wait_seconds": 1.6301870346069336e-05, "mix": {"fineweb_hq": 0.50942, "dclm": 0.20779, "cosmopedia_v2": 0.20032, "finemath": 0.08247}}
{"step": 3090, "total_steps": 72479, "tokens": 1620049920, "loss": 5.099599599838257, "l_mix": 3.1512480080127716, "l_a": 3.25064355134964, "l_c": 3.238363206386566, "z_loss": 16.49410319328308, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.25889304280281067, "tokens_per_second": 1410510.4143521348, "eta_seconds": 25791.954218721392, "data_wait_seconds": 2.0295381546020508e-05, "mix": {"fineweb_hq": 0.50971, "dclm": 0.20777, "cosmopedia_v2": 0.20032, "finemath": 0.0822}}
{"step": 3100, "total_steps": 72479, "tokens": 1625292800, "loss": 5.043545603752136, "l_mix": 3.1160384118556976, "l_a": 3.222122550010681, "l_c": 3.1973136961460114, "z_loss": 16.763549089431763, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.28395941853523254, "tokens_per_second": 1381692.991159646, "eta_seconds": 26326.092254018782, "data_wait_seconds": 1.7076730728149414e-05, "mix": {"fineweb_hq": 0.50968, "dclm": 0.20774, "cosmopedia_v2": 0.20032, "finemath": 0.08226}, "gates": {"gate_a2b/L1_absmean": 0.0775343, "gate_a2b/L1_absmax": 0.56280959, "gate_a2b/L1_rms": 0.12164844, "gate_a2b/L3_absmean": 0.25696975, "gate_a2b/L3_absmax": 0.46789205, "gate_a2b/L3_rms": 0.26888654, "gate_a2b/L5_absmean": 0.3853817, "gate_a2b/L5_absmax": 0.60667235, "gate_a2b/L5_rms": 0.39837044, "gate_c2b/L1_absmean": 0.01560541, "gate_c2b/L1_absmax": 0.5268681, "gate_c2b/L1_rms": 0.05302669, "gate_c2b/L3_absmean": 0.26647645, "gate_c2b/L3_absmax": 0.59343088, "gate_c2b/L3_rms": 0.27804938, "gate_c2b/L5_absmean": 0.49926224, "gate_c2b/L5_absmax": 0.73825026, "gate_c2b/L5_rms": 0.51441509, "gate_b2a/L7_absmean": 0.04175567, "gate_b2a/L7_absmax": 0.22175056, "gate_b2a/L7_rms": 0.05263678, "gate_b2a/L11_absmean": 0.08337044, "gate_b2a/L11_absmax": 0.17169835, "gate_b2a/L11_rms": 0.08658073, "gate_b2a/L14_absmean": 0.22072051, "gate_b2a/L14_absmax": 0.33401015, "gate_b2a/L14_rms": 0.22174394, "gate_b2c/L3_absmean": 0.02507369, "gate_b2c/L3_absmax": 0.34074745, "gate_b2c/L3_rms": 0.04173116, "gate_b2c/L5_absmean": 0.06470467, "gate_b2c/L5_absmax": 0.29116142, "gate_b2c/L5_rms": 0.06772476, "gate_b2c/L6_absmean": 0.10296264, "gate_b2c/L6_absmax": 0.28158167, "gate_b2c/L6_rms": 0.10430973}, "streams": {"cos_a2b/round0": 0.346678, "cos_c2b/round0": 0.26675, "cos_b2a/round0": 0.167606, "cos_b2c/round0": 0.133762, "rms_a/round0": 1.436481, "rms_b/round0": 2.592609, "rms_c/round0": 1.277577, "cos_a2b/round1": 0.394148, "cos_c2b/round1": 0.288509, "cos_b2a/round1": 0.19317, "cos_b2c/round1": 0.206185, "rms_a/round1": 2.36254, "rms_b/round1": 4.119658, "rms_c/round1": 2.107624, "cos_a2b/round2": 0.395576, "cos_c2b/round2": 0.431768, "cos_b2a/round2": 0.092221, "cos_b2c/round2": 0.143872, "rms_a/round2": 3.966003, "rms_b/round2": 7.226016, "rms_c/round2": 3.087518}, "alpha_mean": 0.5205615046867962, "alpha_hist": [0.00745, 0.02878, 0.08946, 0.14799, 0.17957, 0.19165, 0.16881, 0.12711, 0.05719, 0.002], "alpha_by_type": {"special": {"share": 0.0009, "mean": 0.65667, "std": 0.09593}, "whitespace": {"share": 0.02849, "mean": 0.60419, "std": 0.13098}, "digit": {"share": 0.04512, "mean": 0.60414, "std": 0.12506}, "word_start": {"share": 0.55011, "mean": 0.54128, "std": 0.16625}, "word_cont": {"share": 0.26595, "mean": 0.39749, "std": 0.16061}, "punct": {"share": 0.10944, "mean": 0.65814, "std": 0.154}}, "alpha_p05": 0.22203, "alpha_p50": 0.53103, "alpha_p95": 0.82332}
{"step": 3110, "total_steps": 72479, "tokens": 1630535680, "loss": 5.036263108253479, "l_mix": 3.110821098089218, "l_a": 3.2212026715278625, "l_c": 3.191382586956024, "z_loss": 16.663285493850708, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18470846116542816, "tokens_per_second": 1440780.237835642, "eta_seconds": 25242.804778218273, "data_wait_seconds": 1.6301870346069336e-05, "mix": {"fineweb_hq": 0.50965, "dclm": 0.20772, "cosmopedia_v2": 0.20032, "finemath": 0.08232}}
{"step": 3120, "total_steps": 72479, "tokens": 1635778560, "loss": 5.066303551197052, "l_mix": 3.130163222551346, "l_a": 3.2293474674224854, "l_c": 3.2188779413700104, "z_loss": 16.725993633270264, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19875504076480865, "tokens_per_second": 1437824.9329362127, "eta_seconds": 25291.042434310915, "data_wait_seconds": 1.7017126083374023e-05, "mix": {"fineweb_hq": 0.50962, "dclm": 0.20769, "cosmopedia_v2": 0.20032, "finemath": 0.08237}}
{"step": 3130, "total_steps": 72479, "tokens": 1641021440, "loss": 5.042649209499359, "l_mix": 3.114912450313568, "l_a": 3.218354672193527, "l_c": 3.201980084180832, "z_loss": 16.36362051963806, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.23177072405815125, "tokens_per_second": 1390112.2252800013, "eta_seconds": 26155.33325352669, "data_wait_seconds": 1.9550323486328125e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20032, "finemath": 0.08243}}
{"step": 3140, "total_steps": 72479, "tokens": 1646264320, "loss": 3.946022242307663, "l_mix": 2.434458911418915, "l_a": 2.516162008047104, "l_c": 2.516110450029373, "z_loss": 18.814427137374878, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.38216423988342285, "tokens_per_second": 1422723.3597395932, "eta_seconds": 25552.12535390854, "data_wait_seconds": 1.7404556274414062e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20764, "cosmopedia_v2": 0.20032, "finemath": 0.08248}}
{"step": 3150, "total_steps": 72479, "tokens": 1651507200, "loss": 3.614366978406906, "l_mix": 2.231388568878174, "l_a": 2.32235911488533, "l_c": 2.281253308057785, "z_loss": 18.946642637252808, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.3625046908855438, "tokens_per_second": 1421654.389679687, "eta_seconds": 25567.650630044936, "data_wait_seconds": 3.263354301452637e-05, "mix": {"fineweb_hq": 0.50952, "dclm": 0.20762, "cosmopedia_v2": 0.20032, "finemath": 0.08254}}
{"step": 3160, "total_steps": 72479, "tokens": 1656750080, "loss": 3.985856145620346, "l_mix": 2.44505313038826, "l_a": 2.586972177028656, "l_c": 2.5436865985393524, "z_loss": 16.053110241889954, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.33056581020355225, "tokens_per_second": 1357787.4433416852, "eta_seconds": 26766.42802245617, "data_wait_seconds": 3.933906555175781e-05, "mix": {"fineweb_hq": 0.50949, "dclm": 0.20759, "cosmopedia_v2": 0.20063, "finemath": 0.08228}}
{"step": 3170, "total_steps": 72479, "tokens": 1661992960, "loss": 4.0612649619579315, "l_mix": 2.4913797080516815, "l_a": 2.638541966676712, "l_c": 2.5892412960529327, "z_loss": 15.50291359424591, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.28437650203704834, "tokens_per_second": 1416479.059071061, "eta_seconds": 25653.663398194316, "data_wait_seconds": 3.725290298461914e-05, "mix": {"fineweb_hq": 0.50946, "dclm": 0.20757, "cosmopedia_v2": 0.20063, "finemath": 0.08233}}
{"step": 3180, "total_steps": 72479, "tokens": 1667235840, "loss": 5.053303956985474, "l_mix": 3.122340738773346, "l_a": 3.223123997449875, "l_c": 3.2078587412834167, "z_loss": 16.683712482452393, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16857697069644928, "tokens_per_second": 1403696.1664767324, "eta_seconds": 25883.545869612695, "data_wait_seconds": 2.4318695068359375e-05, "mix": {"fineweb_hq": 0.50975, "dclm": 0.20755, "cosmopedia_v2": 0.20031, "finemath": 0.08239}}
{"step": 3190, "total_steps": 72479, "tokens": 1672478720, "loss": 5.057244122028351, "l_mix": 3.1243897080421448, "l_a": 3.225707858800888, "l_c": 3.2117103338241577, "z_loss": 16.289210081100464, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2681294083595276, "tokens_per_second": 1370911.0193722204, "eta_seconds": 26498.72290663719, "data_wait_seconds": 3.3020973205566406e-05, "mix": {"fineweb_hq": 0.50972, "dclm": 0.20752, "cosmopedia_v2": 0.20031, "finemath": 0.08245}}
{"step": 3200, "total_steps": 72479, "tokens": 1677721600, "loss": 5.134325742721558, "l_mix": 3.17121821641922, "l_a": 3.275763839483261, "l_c": 3.262473076581955, "z_loss": 16.36357307434082, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.3320211172103882, "tokens_per_second": 1122901.612213323, "eta_seconds": 32346.68822000027, "data_wait_seconds": 2.0176172256469727e-05, "mix": {"fineweb_hq": 0.50969, "dclm": 0.2075, "cosmopedia_v2": 0.20031, "finemath": 0.0825}, "gates": {"gate_a2b/L1_absmean": 0.07890802, "gate_a2b/L1_absmax": 0.56992781, "gate_a2b/L1_rms": 0.12344171, "gate_a2b/L3_absmean": 0.25842413, "gate_a2b/L3_absmax": 0.46856049, "gate_a2b/L3_rms": 0.27011055, "gate_a2b/L5_absmean": 0.38904974, "gate_a2b/L5_absmax": 0.6150586, "gate_a2b/L5_rms": 0.40193951, "gate_c2b/L1_absmean": 0.0159601, "gate_c2b/L1_absmax": 0.53510219, "gate_c2b/L1_rms": 0.05374636, "gate_c2b/L3_absmean": 0.26558334, "gate_c2b/L3_absmax": 0.59165561, "gate_c2b/L3_rms": 0.27724692, "gate_c2b/L5_absmean": 0.50347751, "gate_c2b/L5_absmax": 0.74977434, "gate_c2b/L5_rms": 0.51866877, "gate_b2a/L7_absmean": 0.04296959, "gate_b2a/L7_absmax": 0.22654846, "gate_b2a/L7_rms": 0.05357668, "gate_b2a/L11_absmean": 0.08600593, "gate_b2a/L11_absmax": 0.17562516, "gate_b2a/L11_rms": 0.08907392, "gate_b2a/L14_absmean": 0.22436963, "gate_b2a/L14_absmax": 0.34561759, "gate_b2a/L14_rms": 0.22536416, "gate_b2c/L3_absmean": 0.02612417, "gate_b2c/L3_absmax": 0.34608617, "gate_b2c/L3_rms": 0.04291474, "gate_b2c/L5_absmean": 0.06571589, "gate_b2c/L5_absmax": 0.29843509, "gate_b2c/L5_rms": 0.06863226, "gate_b2c/L6_absmean": 0.10442504, "gate_b2c/L6_absmax": 0.29064429, "gate_b2c/L6_rms": 0.10585441}, "streams": {"cos_a2b/round0": 0.349428, "cos_c2b/round0": 0.269088, "cos_b2a/round0": 0.167811, "cos_b2c/round0": 0.129404, "rms_a/round0": 1.441495, "rms_b/round0": 2.570481, "rms_c/round0": 1.286285, "cos_a2b/round1": 0.398861, "cos_c2b/round1": 0.284163, "cos_b2a/round1": 0.186741, "cos_b2c/round1": 0.194504, "rms_a/round1": 2.383311, "rms_b/round1": 4.15171, "rms_c/round1": 2.113419, "cos_a2b/round2": 0.391381, "cos_c2b/round2": 0.439741, "cos_b2a/round2": 0.077072, "cos_b2c/round2": 0.143343, "rms_a/round2": 4.015989, "rms_b/round2": 7.552189, "rms_c/round2": 3.127984}, "alpha_mean": 0.48059776200707205, "alpha_hist": [0.0155, 0.05086, 0.10909, 0.1637, 0.19055, 0.19266, 0.15431, 0.09311, 0.02885, 0.00137], "alpha_by_type": {"special": {"share": 0.00119, "mean": 0.58762, "std": 0.08622}, "whitespace": {"share": 0.03123, "mean": 0.59085, "std": 0.11934}, "digit": {"share": 0.0403, "mean": 0.53905, "std": 0.14172}, "word_start": {"share": 0.56702, "mean": 0.51398, "std": 0.15822}, "word_cont": {"share": 0.25537, "mean": 0.33623, "std": 0.1564}, "punct": {"share": 0.10489, "mean": 0.59512, "std": 0.17144}}, "alpha_p05": 0.19529, "alpha_p50": 0.46539, "alpha_p95": 0.78166}
{"step": 3210, "total_steps": 72479, "tokens": 1682964480, "loss": 5.123336434364319, "l_mix": 3.1645986437797546, "l_a": 3.2687831819057465, "l_c": 3.2548072040081024, "z_loss": 16.605716943740845, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2253260612487793, "tokens_per_second": 1454902.4736401783, "eta_seconds": 24961.745635867122, "data_wait_seconds": 2.1159648895263672e-05, "mix": {"fineweb_hq": 0.50966, "dclm": 0.20779, "cosmopedia_v2": 0.20031, "finemath": 0.08224}}
{"step": 3220, "total_steps": 72479, "tokens": 1688207360, "loss": 4.961039185523987, "l_mix": 3.064754366874695, "l_a": 3.1684407591819763, "l_c": 3.1468688547611237, "z_loss": 16.918393850326538, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19426676630973816, "tokens_per_second": 1358145.0168259356, "eta_seconds": 26736.218991446498, "data_wait_seconds": 1.677870750427246e-05, "mix": {"fineweb_hq": 0.50963, "dclm": 0.20776, "cosmopedia_v2": 0.20031, "finemath": 0.0823}}
{"step": 3230, "total_steps": 72479, "tokens": 1693450240, "loss": 5.14337432384491, "l_mix": 3.179161012172699, "l_a": 3.281331866979599, "l_c": 3.2605748772621155, "z_loss": 16.413548231124878, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21721680462360382, "tokens_per_second": 1387490.4799864974, "eta_seconds": 26166.9685202837, "data_wait_seconds": 1.9162893295288086e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20774, "cosmopedia_v2": 0.20031, "finemath": 0.08235}}
{"step": 3240, "total_steps": 72479, "tokens": 1698693120, "loss": 5.144304513931274, "l_mix": 3.1787107288837433, "l_a": 3.284289240837097, "l_c": 3.2622093856334686, "z_loss": 16.440749406814575, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.3026629090309143, "tokens_per_second": 1414510.7166881424, "eta_seconds": 25663.415910339358, "data_wait_seconds": 2.7745962142944336e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20772, "cosmopedia_v2": 0.20031, "finemath": 0.08241}}
{"step": 3250, "total_steps": 72479, "tokens": 1703936000, "loss": 3.6821437180042267, "l_mix": 2.27294060587883, "l_a": 2.362691253423691, "l_c": 2.3283134400844574, "z_loss": 19.016710996627808, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2783212959766388, "tokens_per_second": 1453701.393027847, "eta_seconds": 24967.943297076225, "data_wait_seconds": 2.7507543563842773e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20769, "cosmopedia_v2": 0.20031, "finemath": 0.08246}}
{"step": 3260, "total_steps": 72479, "tokens": 1709178880, "loss": 3.4778309762477875, "l_mix": 2.145902991294861, "l_a": 2.227915495634079, "l_c": 2.205297827720642, "z_loss": 19.638760805130005, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.3668244481086731, "tokens_per_second": 1403814.0922624872, "eta_seconds": 25851.49363582134, "data_wait_seconds": 3.215670585632324e-05, "mix": {"fineweb_hq": 0.50951, "dclm": 0.20767, "cosmopedia_v2": 0.20031, "finemath": 0.08252}}
{"step": 3270, "total_steps": 72479, "tokens": 1714421760, "loss": 4.052426278591156, "l_mix": 2.486342430114746, "l_a": 2.632437765598297, "l_c": 2.582876056432724, "z_loss": 14.896291613578796, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2847483456134796, "tokens_per_second": 1445348.3709880176, "eta_seconds": 25104.98432097435, "data_wait_seconds": 2.0682811737060547e-05, "mix": {"fineweb_hq": 0.50948, "dclm": 0.20765, "cosmopedia_v2": 0.20061, "finemath": 0.08226}}
{"step": 3280, "total_steps": 72479, "tokens": 1719664640, "loss": 4.002543956041336, "l_mix": 2.4557321667671204, "l_a": 2.59320330619812, "l_c": 2.557822823524475, "z_loss": 15.038553953170776, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2889324426651001, "tokens_per_second": 1460666.3487766003, "eta_seconds": 24838.119494152066, "data_wait_seconds": 2.7120113372802734e-05, "mix": {"fineweb_hq": 0.50945, "dclm": 0.20762, "cosmopedia_v2": 0.20061, "finemath": 0.08232}}
{"step": 3290, "total_steps": 72479, "tokens": 1724907520, "loss": 5.028925657272339, "l_mix": 3.1076786816120148, "l_a": 3.2094152569770813, "l_c": 3.1892620623111725, "z_loss": 16.436371088027954, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18736977875232697, "tokens_per_second": 1384205.193284051, "eta_seconds": 26206.347590661047, "data_wait_seconds": 2.2917985916137695e-05, "mix": {"fineweb_hq": 0.50973, "dclm": 0.2076, "cosmopedia_v2": 0.2003, "finemath": 0.08237}}
{"step": 3300, "total_steps": 72479, "tokens": 1730150400, "loss": 5.070995450019836, "l_mix": 3.132856994867325, "l_a": 3.235609769821167, "l_c": 3.219278246164322, "z_loss": 16.72025203704834, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.24785837531089783, "tokens_per_second": 1416770.451823227, "eta_seconds": 25600.279498577118, "data_wait_seconds": 2.3096799850463867e-05, "mix": {"fineweb_hq": 0.5097, "dclm": 0.20758, "cosmopedia_v2": 0.2003, "finemath": 0.08242}, "gates": {"gate_a2b/L1_absmean": 0.08077838, "gate_a2b/L1_absmax": 0.58047682, "gate_a2b/L1_rms": 0.12595458, "gate_a2b/L3_absmean": 0.2584902, "gate_a2b/L3_absmax": 0.47105107, "gate_a2b/L3_rms": 0.27013612, "gate_a2b/L5_absmean": 0.39187545, "gate_a2b/L5_absmax": 0.61799079, "gate_a2b/L5_rms": 0.4047727, "gate_c2b/L1_absmean": 0.0155613, "gate_c2b/L1_absmax": 0.54620802, "gate_c2b/L1_rms": 0.05380551, "gate_c2b/L3_absmean": 0.26527435, "gate_c2b/L3_absmax": 0.58980381, "gate_c2b/L3_rms": 0.27670881, "gate_c2b/L5_absmean": 0.50711399, "gate_c2b/L5_absmax": 0.75202972, "gate_c2b/L5_rms": 0.52251506, "gate_b2a/L7_absmean": 0.04374007, "gate_b2a/L7_absmax": 0.22838953, "gate_b2a/L7_rms": 0.05414614, "gate_b2a/L11_absmean": 0.08754575, "gate_b2a/L11_absmax": 0.17819798, "gate_b2a/L11_rms": 0.09054711, "gate_b2a/L14_absmean": 0.22717312, "gate_b2a/L14_absmax": 0.35303202, "gate_b2a/L14_rms": 0.22821242, "gate_b2c/L3_absmean": 0.02673141, "gate_b2c/L3_absmax": 0.35007003, "gate_b2c/L3_rms": 0.04359467, "gate_b2c/L5_absmean": 0.06656513, "gate_b2c/L5_absmax": 0.30707535, "gate_b2c/L5_rms": 0.06957448, "gate_b2c/L6_absmean": 0.10634081, "gate_b2c/L6_absmax": 0.29353163, "gate_b2c/L6_rms": 0.10774299}, "streams": {"cos_a2b/round0": 0.353599, "cos_c2b/round0": 0.268165, "cos_b2a/round0": 0.172155, "cos_b2c/round0": 0.135859, "rms_a/round0": 1.440829, "rms_b/round0": 2.587312, "rms_c/round0": 1.282862, "cos_a2b/round1": 0.399915, "cos_c2b/round1": 0.281626, "cos_b2a/round1": 0.186714, "cos_b2c/round1": 0.199784, "rms_a/round1": 2.439465, "rms_b/round1": 4.224378, "rms_c/round1": 2.137852, "cos_a2b/round2": 0.409985, "cos_c2b/round2": 0.428482, "cos_b2a/round2": 0.076281, "cos_b2c/round2": 0.142947, "rms_a/round2": 4.136798, "rms_b/round2": 7.473861, "rms_c/round2": 3.167145}, "alpha_mean": 0.5154180525411931, "alpha_hist": [0.00877, 0.03172, 0.08958, 0.15039, 0.17633, 0.19016, 0.18172, 0.1263, 0.0432, 0.00183], "alpha_by_type": {"special": {"share": 0.0013, "mean": 0.58736, "std": 0.11224}, "whitespace": {"share": 0.03421, "mean": 0.56508, "std": 0.13519}, "digit": {"share": 0.04724, "mean": 0.5985, "std": 0.13512}, "word_start": {"share": 0.5578, "mean": 0.53694, "std": 0.16227}, "word_cont": {"share": 0.25066, "mean": 0.39361, "std": 0.16599}, "punct": {"share": 0.1088, "mean": 0.63319, "std": 0.16838}}, "alpha_p05": 0.22101, "alpha_p50": 0.53267, "alpha_p95": 0.81108}
{"step": 3310, "total_steps": 72479, "tokens": 1735393280, "loss": 5.135747194290161, "l_mix": 3.1752320528030396, "l_a": 3.2723260819911957, "l_c": 3.257340043783188, "z_loss": 16.152639031410217, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.26826930046081543, "tokens_per_second": 1335879.5504712616, "eta_seconds": 27146.516809248926, "data_wait_seconds": 2.3603439331054688e-05, "mix": {"fineweb_hq": 0.50967, "dclm": 0.20755, "cosmopedia_v2": 0.2003, "finemath": 0.08248}}
{"step": 3320, "total_steps": 72479, "tokens": 1740636160, "loss": 4.999541163444519, "l_mix": 3.0894444286823273, "l_a": 3.191489100456238, "l_c": 3.1699309647083282, "z_loss": 16.70654559135437, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.26557812094688416, "tokens_per_second": 1382064.3974561484, "eta_seconds": 26235.560266757013, "data_wait_seconds": 1.7136335372924805e-05, "mix": {"fineweb_hq": 0.50964, "dclm": 0.20753, "cosmopedia_v2": 0.2003, "finemath": 0.08253}}
{"step": 3330, "total_steps": 72479, "tokens": 1745879040, "loss": 5.371253550052643, "l_mix": 3.323518216609955, "l_a": 3.414788991212845, "l_c": 3.4053741097450256, "z_loss": 16.861931085586548, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20270390808582306, "tokens_per_second": 1415625.1924750307, "eta_seconds": 25609.87972290516, "data_wait_seconds": 1.6421079635620117e-05, "mix": {"fineweb_hq": 0.50961, "dclm": 0.20781, "cosmopedia_v2": 0.2003, "finemath": 0.08228}}
{"step": 3340, "total_steps": 72479, "tokens": 1751121920, "loss": 5.3891273736953735, "l_mix": 3.334001988172531, "l_a": 3.4265662729740143, "l_c": 3.4183379411697388, "z_loss": 16.53981590270996, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.24032744765281677, "tokens_per_second": 1376724.2870202505, "eta_seconds": 26329.707679128645, "data_wait_seconds": 3.886222839355469e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20778, "cosmopedia_v2": 0.2003, "finemath": 0.08234}}
{"step": 3350, "total_steps": 72479, "tokens": 1756364800, "loss": 5.40762597322464, "l_mix": 3.3453396558761597, "l_a": 3.4411710798740387, "l_c": 3.427755832672119, "z_loss": 16.082183837890625, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22123365104198456, "tokens_per_second": 1337092.265232278, "eta_seconds": 27106.211062932012, "data_wait_seconds": 2.9832124710083008e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20776, "cosmopedia_v2": 0.2003, "finemath": 0.08239}}
{"step": 3360, "total_steps": 72479, "tokens": 1761607680, "loss": 5.417507171630859, "l_mix": 3.3518413603305817, "l_a": 3.4481467604637146, "l_c": 3.4318882524967194, "z_loss": 16.55282759666443, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.32708799839019775, "tokens_per_second": 1375155.8403120139, "eta_seconds": 26352.11312761307, "data_wait_seconds": 2.473592758178711e-05, "mix": {"fineweb_hq": 0.50952, "dclm": 0.20774, "cosmopedia_v2": 0.2003, "finemath": 0.08244}}
{"step": 3370, "total_steps": 72479, "tokens": 1766850560, "loss": 3.8481910824775696, "l_mix": 2.3789783120155334, "l_a": 2.4645601511001587, "l_c": 2.4264532327651978, "z_loss": 19.08711528778076, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.4684218764305115, "tokens_per_second": 1437185.561340836, "eta_seconds": 25211.093380451202, "data_wait_seconds": 3.415346145629883e-05, "mix": {"fineweb_hq": 0.5095, "dclm": 0.20772, "cosmopedia_v2": 0.2003, "finemath": 0.08249}}
{"step": 3380, "total_steps": 72479, "tokens": 1772093440, "loss": 4.031404376029968, "l_mix": 2.47345769405365, "l_a": 2.6135595738887787, "l_c": 2.5744224190711975, "z_loss": 15.51895022392273, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.29233795404434204, "tokens_per_second": 1389829.4980714503, "eta_seconds": 26066.34595270157, "data_wait_seconds": 1.761317253112793e-05, "mix": {"fineweb_hq": 0.50947, "dclm": 0.20769, "cosmopedia_v2": 0.20059, "finemath": 0.08225}}
{"step": 3390, "total_steps": 72479, "tokens": 1777336320, "loss": 4.001728296279907, "l_mix": 2.455420643091202, "l_a": 2.599926769733429, "l_c": 2.5493043065071106, "z_loss": 15.382909417152405, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2842442989349365, "tokens_per_second": 1427325.9312339942, "eta_seconds": 25377.899216532707, "data_wait_seconds": 1.6748905181884766e-05, "mix": {"fineweb_hq": 0.50944, "dclm": 0.20767, "cosmopedia_v2": 0.20059, "finemath": 0.0823}}
{"step": 3400, "total_steps": 72479, "tokens": 1782579200, "loss": 5.045752644538879, "l_mix": 3.1181998550891876, "l_a": 3.2218292355537415, "l_c": 3.1977496445178986, "z_loss": 16.789498329162598, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20846736431121826, "tokens_per_second": 1390089.817280663, "eta_seconds": 26053.92133786678, "data_wait_seconds": 2.1576881408691406e-05, "mix": {"fineweb_hq": 0.50971, "dclm": 0.20765, "cosmopedia_v2": 0.20029, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.08298708, "gate_a2b/L1_absmax": 0.58552474, "gate_a2b/L1_rms": 0.12821223, "gate_a2b/L3_absmean": 0.25894266, "gate_a2b/L3_absmax": 0.47432089, "gate_a2b/L3_rms": 0.27039528, "gate_a2b/L5_absmean": 0.3946138, "gate_a2b/L5_absmax": 0.6179589, "gate_a2b/L5_rms": 0.40735453, "gate_c2b/L1_absmean": 0.015833, "gate_c2b/L1_absmax": 0.54659337, "gate_c2b/L1_rms": 0.05341028, "gate_c2b/L3_absmean": 0.26512942, "gate_c2b/L3_absmax": 0.59208667, "gate_c2b/L3_rms": 0.27642101, "gate_c2b/L5_absmean": 0.51106256, "gate_c2b/L5_absmax": 0.75505495, "gate_c2b/L5_rms": 0.52658075, "gate_b2a/L7_absmean": 0.044925, "gate_b2a/L7_absmax": 0.23101749, "gate_b2a/L7_rms": 0.05536417, "gate_b2a/L11_absmean": 0.08950345, "gate_b2a/L11_absmax": 0.18205123, "gate_b2a/L11_rms": 0.09239534, "gate_b2a/L14_absmean": 0.23105547, "gate_b2a/L14_absmax": 0.35459954, "gate_b2a/L14_rms": 0.23207551, "gate_b2c/L3_absmean": 0.02703891, "gate_b2c/L3_absmax": 0.35400483, "gate_b2c/L3_rms": 0.04418907, "gate_b2c/L5_absmean": 0.06697067, "gate_b2c/L5_absmax": 0.31301931, "gate_b2c/L5_rms": 0.07001749, "gate_b2c/L6_absmean": 0.10739648, "gate_b2c/L6_absmax": 0.29394868, "gate_b2c/L6_rms": 0.10872205}, "streams": {"cos_a2b/round0": 0.356678, "cos_c2b/round0": 0.267978, "cos_b2a/round0": 0.177067, "cos_b2c/round0": 0.135268, "rms_a/round0": 1.455875, "rms_b/round0": 2.597198, "rms_c/round0": 1.2989, "cos_a2b/round1": 0.394282, "cos_c2b/round1": 0.268519, "cos_b2a/round1": 0.184731, "cos_b2c/round1": 0.204222, "rms_a/round1": 2.435285, "rms_b/round1": 4.256816, "rms_c/round1": 2.148158, "cos_a2b/round2": 0.406526, "cos_c2b/round2": 0.429207, "cos_b2a/round2": 0.079958, "cos_b2c/round2": 0.135939, "rms_a/round2": 4.150537, "rms_b/round2": 7.603865, "rms_c/round2": 3.185961}, "alpha_mean": 0.4948980170907191, "alpha_hist": [0.01097, 0.04182, 0.1077, 0.1599, 0.18765, 0.18463, 0.15755, 0.10783, 0.03996, 0.00198], "alpha_by_type": {"special": {"share": 0.00117, "mean": 0.60215, "std": 0.07553}, "whitespace": {"share": 0.03526, "mean": 0.56813, "std": 0.13286}, "digit": {"share": 0.07524, "mean": 0.55655, "std": 0.11345}, "word_start": {"share": 0.52283, "mean": 0.51495, "std": 0.17241}, "word_cont": {"share": 0.25558, "mean": 0.37428, "std": 0.15856}, "punct": {"share": 0.10991, "mean": 0.61313, "std": 0.18403}}, "alpha_p05": 0.19468, "alpha_p50": 0.49615, "alpha_p95": 0.79724}
{"step": 3410, "total_steps": 72479, "tokens": 1787822080, "loss": 5.02752161026001, "l_mix": 3.107597678899765, "l_a": 3.2090131044387817, "l_c": 3.1852777898311615, "z_loss": 16.366387128829956, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18932399153709412, "tokens_per_second": 1422940.0721129861, "eta_seconds": 25448.751203012467, "data_wait_seconds": 2.244114875793457e-05, "mix": {"fineweb_hq": 0.50968, "dclm": 0.20762, "cosmopedia_v2": 0.20029, "finemath": 0.0824}}
{"step": 3420, "total_steps": 72479, "tokens": 1793064960, "loss": 5.047640085220337, "l_mix": 3.1187561452388763, "l_a": 3.225801259279251, "l_c": 3.1982156932353973, "z_loss": 16.787498474121094, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.24062077701091766, "tokens_per_second": 1442309.0392545932, "eta_seconds": 25103.36135084629, "data_wait_seconds": 1.665949821472168e-05, "mix": {"fineweb_hq": 0.50965, "dclm": 0.2076, "cosmopedia_v2": 0.20029, "finemath": 0.08246}}
{"step": 3430, "total_steps": 72479, "tokens": 1798307840, "loss": 4.917809307575226, "l_mix": 3.038528859615326, "l_a": 3.1421011090278625, "l_c": 3.1164687871932983, "z_loss": 17.095086812973022, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.26355811953544617, "tokens_per_second": 1378014.228897336, "eta_seconds": 26270.81880059242, "data_wait_seconds": 2.5391578674316406e-05, "mix": {"fineweb_hq": 0.50962, "dclm": 0.20758, "cosmopedia_v2": 0.20029, "finemath": 0.08251}}
{"step": 3440, "total_steps": 72479, "tokens": 1803550720, "loss": 3.9789413809776306, "l_mix": 2.4413950741291046, "l_a": 2.5842999517917633, "l_c": 2.535603493452072, "z_loss": 15.751459836959839, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2947123944759369, "tokens_per_second": 1317050.3484348918, "eta_seconds": 27482.8667522192, "data_wait_seconds": 1.8358230590820312e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20756, "cosmopedia_v2": 0.20058, "finemath": 0.08227}}
{"step": 3450, "total_steps": 72479, "tokens": 1808793600, "loss": 3.973371833562851, "l_mix": 2.4393962919712067, "l_a": 2.5808989107608795, "l_c": 2.5272912681102753, "z_loss": 15.183699011802673, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.31680184602737427, "tokens_per_second": 1359265.665295094, "eta_seconds": 26625.462024116518, "data_wait_seconds": 2.5391578674316406e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20754, "cosmopedia_v2": 0.20058, "finemath": 0.08232}}
{"step": 3460, "total_steps": 72479, "tokens": 1814036480, "loss": 5.365570485591888, "l_mix": 3.3204010725021362, "l_a": 3.4107960164546967, "l_c": 3.400863081216812, "z_loss": 16.716429233551025, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2337990403175354, "tokens_per_second": 1353040.2508617158, "eta_seconds": 26744.092386722565, "data_wait_seconds": 2.008676528930664e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.2078, "cosmopedia_v2": 0.20029, "finemath": 0.08237}}
{"step": 3470, "total_steps": 72479, "tokens": 1819279360, "loss": 5.323483049869537, "l_mix": 3.293859839439392, "l_a": 3.3923934996128082, "l_c": 3.3675685226917267, "z_loss": 16.345624685287476, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21068038046360016, "tokens_per_second": 1442533.9362864622, "eta_seconds": 25081.27516579628, "data_wait_seconds": 1.8477439880371094e-05, "mix": {"fineweb_hq": 0.50951, "dclm": 0.20778, "cosmopedia_v2": 0.20029, "finemath": 0.08242}}
{"step": 3480, "total_steps": 72479, "tokens": 1824522240, "loss": 5.510275363922119, "l_mix": 3.4109100997447968, "l_a": 3.5031414330005646, "l_c": 3.4891763627529144, "z_loss": 16.69808292388916, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2356969565153122, "tokens_per_second": 1316936.6900622577, "eta_seconds": 27469.31419329643, "data_wait_seconds": 1.8924474716186523e-05, "mix": {"fineweb_hq": 0.50948, "dclm": 0.20776, "cosmopedia_v2": 0.20029, "finemath": 0.08247}}
{"step": 3490, "total_steps": 72479, "tokens": 1829765120, "loss": 3.972127765417099, "l_mix": 2.437888503074646, "l_a": 2.5803342461586, "l_c": 2.5288434624671936, "z_loss": 14.85873818397522, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.3317473530769348, "tokens_per_second": 1365108.5032833025, "eta_seconds": 26496.139131069187, "data_wait_seconds": 1.990795135498047e-05, "mix": {"fineweb_hq": 0.50946, "dclm": 0.20774, "cosmopedia_v2": 0.20057, "finemath": 0.08223}}
{"step": 3500, "total_steps": 72479, "tokens": 1835008000, "loss": 3.9908256828784943, "l_mix": 2.448019653558731, "l_a": 2.5916928350925446, "l_c": 2.54593887925148, "z_loss": 15.164953827857971, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.3009239137172699, "tokens_per_second": 1274368.4933727745, "eta_seconds": 28378.65353708267, "data_wait_seconds": 1.9103288650512695e-05, "mix": {"fineweb_hq": 0.50943, "dclm": 0.20771, "cosmopedia_v2": 0.20057, "finemath": 0.08229}, "gates": {"gate_a2b/L1_absmean": 0.0843215, "gate_a2b/L1_absmax": 0.5862152, "gate_a2b/L1_rms": 0.12984605, "gate_a2b/L3_absmean": 0.25962439, "gate_a2b/L3_absmax": 0.47164544, "gate_a2b/L3_rms": 0.27083722, "gate_a2b/L5_absmean": 0.39766169, "gate_a2b/L5_absmax": 0.61848527, "gate_a2b/L5_rms": 0.41036779, "gate_c2b/L1_absmean": 0.01524055, "gate_c2b/L1_absmax": 0.54643917, "gate_c2b/L1_rms": 0.05323414, "gate_c2b/L3_absmean": 0.26427481, "gate_c2b/L3_absmax": 0.59154892, "gate_c2b/L3_rms": 0.27547106, "gate_c2b/L5_absmean": 0.51446724, "gate_c2b/L5_absmax": 0.76364237, "gate_c2b/L5_rms": 0.53008342, "gate_b2a/L7_absmean": 0.04557645, "gate_b2a/L7_absmax": 0.2350692, "gate_b2a/L7_rms": 0.05602342, "gate_b2a/L11_absmean": 0.09121285, "gate_b2a/L11_absmax": 0.18134485, "gate_b2a/L11_rms": 0.09399822, "gate_b2a/L14_absmean": 0.23357883, "gate_b2a/L14_absmax": 0.35831583, "gate_b2a/L14_rms": 0.23456617, "gate_b2c/L3_absmean": 0.02747681, "gate_b2c/L3_absmax": 0.35614452, "gate_b2c/L3_rms": 0.04466239, "gate_b2c/L5_absmean": 0.06803513, "gate_b2c/L5_absmax": 0.32334879, "gate_b2c/L5_rms": 0.07113162, "gate_b2c/L6_absmean": 0.1094337, "gate_b2c/L6_absmax": 0.2963064, "gate_b2c/L6_rms": 0.11073404}, "streams": {"cos_a2b/round0": 0.320319, "cos_c2b/round0": 0.22959, "cos_b2a/round0": 0.168748, "cos_b2c/round0": 0.094858, "rms_a/round0": 1.460144, "rms_b/round0": 2.609268, "rms_c/round0": 1.314688, "cos_a2b/round1": 0.377267, "cos_c2b/round1": 0.266251, "cos_b2a/round1": 0.183387, "cos_b2c/round1": 0.194957, "rms_a/round1": 2.434862, "rms_b/round1": 4.289436, "rms_c/round1": 2.157562, "cos_a2b/round2": 0.392754, "cos_c2b/round2": 0.422965, "cos_b2a/round2": 0.043817, "cos_b2c/round2": 0.099992, "rms_a/round2": 4.140639, "rms_b/round2": 7.490834, "rms_c/round2": 3.216399}, "alpha_mean": 0.4780005035455517, "alpha_hist": [0.03555, 0.10439, 0.14156, 0.13725, 0.1264, 0.11475, 0.10925, 0.11772, 0.09769, 0.01544], "alpha_by_type": {"special": {"share": 0.00111, "mean": 0.81149, "std": 0.07709}, "whitespace": {"share": 0.03255, "mean": 0.70081, "std": 0.16097}, "digit": {"share": 0.00958, "mean": 0.45379, "std": 0.22095}, "word_start": {"share": 0.57364, "mean": 0.49784, "std": 0.2185}, "word_cont": {"share": 0.26402, "mean": 0.32009, "std": 0.19297}, "punct": {"share": 0.11909, "mean": 0.67044, "std": 0.19701}}, "alpha_p05": 0.10933, "alpha_p50": 0.43939, "alpha_p95": 0.85488}
{"step": 3510, "total_steps": 72479, "tokens": 1840250880, "loss": 5.10204404592514, "l_mix": 3.154441922903061, "l_a": 3.253064811229706, "l_c": 3.233614206314087, "z_loss": 15.983778238296509, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22902970016002655, "tokens_per_second": 1294463.8360132396, "eta_seconds": 27934.051200199126, "data_wait_seconds": 1.9937753677368164e-05, "mix": {"fineweb_hq": 0.50969, "dclm": 0.20769, "cosmopedia_v2": 0.20028, "finemath": 0.08234}}
{"step": 3520, "total_steps": 72479, "tokens": 1845493760, "loss": 5.0091177225112915, "l_mix": 3.096127837896347, "l_a": 3.1967293322086334, "l_c": 3.1744995713233948, "z_loss": 16.210991859436035, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19579114019870758, "tokens_per_second": 1432845.7922716152, "eta_seconds": 25232.566119122504, "data_wait_seconds": 1.8090009689331055e-05, "mix": {"fineweb_hq": 0.50966, "dclm": 0.20767, "cosmopedia_v2": 0.20028, "finemath": 0.08239}}
{"step": 3530, "total_steps": 72479, "tokens": 1850736640, "loss": 5.024641036987305, "l_mix": 3.104680985212326, "l_a": 3.2093628644943237, "l_c": 3.1849217414855957, "z_loss": 16.745376586914062, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20375186204910278, "tokens_per_second": 1442489.0837411487, "eta_seconds": 25060.24740114212, "data_wait_seconds": 2.6464462280273438e-05, "mix": {"fineweb_hq": 0.50963, "dclm": 0.20765, "cosmopedia_v2": 0.20028, "finemath": 0.08244}}
{"step": 3540, "total_steps": 72479, "tokens": 1855979520, "loss": 4.957100033760071, "l_mix": 3.062578320503235, "l_a": 3.1666165590286255, "l_c": 3.142910271883011, "z_loss": 16.636313438415527, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2498045116662979, "tokens_per_second": 1230665.771137986, "eta_seconds": 29369.37979397774, "data_wait_seconds": 2.0444393157958984e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20763, "cosmopedia_v2": 0.20028, "finemath": 0.08249}}
{"step": 3550, "total_steps": 72479, "tokens": 1861222400, "loss": 3.957353800535202, "l_mix": 2.4277788400650024, "l_a": 2.573689430952072, "l_c": 2.5198411345481873, "z_loss": 15.156440019607544, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.329440712928772, "tokens_per_second": 1268133.5736814146, "eta_seconds": 28497.50870256424, "data_wait_seconds": 2.5093555450439453e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20761, "cosmopedia_v2": 0.20056, "finemath": 0.08225}}
{"step": 3560, "total_steps": 72479, "tokens": 1866465280, "loss": 3.971926271915436, "l_mix": 2.4367046654224396, "l_a": 2.5811794996261597, "l_c": 2.5311102271080017, "z_loss": 15.345600247383118, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2979367971420288, "tokens_per_second": 1444805.94506852, "eta_seconds": 25009.174965906146, "data_wait_seconds": 2.2917985916137695e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20758, "cosmopedia_v2": 0.20056, "finemath": 0.0823}}
{"step": 3570, "total_steps": 72479, "tokens": 1871708160, "loss": 3.992758870124817, "l_mix": 2.451033264398575, "l_a": 2.590271234512329, "l_c": 2.543756365776062, "z_loss": 15.17162537574768, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2660013437271118, "tokens_per_second": 1326529.030561844, "eta_seconds": 27235.108286094666, "data_wait_seconds": 2.5928020477294922e-05, "mix": {"fineweb_hq": 0.50952, "dclm": 0.20756, "cosmopedia_v2": 0.20056, "finemath": 0.08235}}
{"step": 3580, "total_steps": 72479, "tokens": 1876951040, "loss": 3.9645688235759735, "l_mix": 2.4330248832702637, "l_a": 2.5753559470176697, "l_c": 2.5245787501335144, "z_loss": 15.634765863418579, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2903347313404083, "tokens_per_second": 1412250.4692854185, "eta_seconds": 25578.266531062127, "data_wait_seconds": 2.9742717742919922e-05, "mix": {"fineweb_hq": 0.5095, "dclm": 0.20754, "cosmopedia_v2": 0.20056, "finemath": 0.0824}}
{"step": 3590, "total_steps": 72479, "tokens": 1882193920, "loss": 3.9804711043834686, "l_mix": 2.4427970349788666, "l_a": 2.5835224092006683, "l_c": 2.5369112491607666, "z_loss": 15.438706159591675, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.25308170914649963, "tokens_per_second": 1478876.8022919323, "eta_seconds": 24422.36971735954, "data_wait_seconds": 2.2709369659423828e-05, "mix": {"fineweb_hq": 0.50947, "dclm": 0.20752, "cosmopedia_v2": 0.20056, "finemath": 0.08245}}
{"step": 3600, "total_steps": 72479, "tokens": 1887436800, "loss": 3.9870656728744507, "l_mix": 2.4467921257019043, "l_a": 2.591999113559723, "l_c": 2.537079930305481, "z_loss": 15.497357726097107, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.3111720085144043, "tokens_per_second": 1414850.2732691027, "eta_seconds": 25523.854950785637, "data_wait_seconds": 1.7404556274414062e-05, "mix": {"fineweb_hq": 0.50944, "dclm": 0.20778, "cosmopedia_v2": 0.20056, "finemath": 0.08222}, "gates": {"gate_a2b/L1_absmean": 0.08663402, "gate_a2b/L1_absmax": 0.58884627, "gate_a2b/L1_rms": 0.13240597, "gate_a2b/L3_absmean": 0.26053616, "gate_a2b/L3_absmax": 0.47929764, "gate_a2b/L3_rms": 0.27160263, "gate_a2b/L5_absmean": 0.39971381, "gate_a2b/L5_absmax": 0.619295, "gate_a2b/L5_rms": 0.41237566, "gate_c2b/L1_absmean": 0.01503226, "gate_c2b/L1_absmax": 0.55469626, "gate_c2b/L1_rms": 0.05326858, "gate_c2b/L3_absmean": 0.26385152, "gate_c2b/L3_absmax": 0.5839985, "gate_c2b/L3_rms": 0.27485323, "gate_c2b/L5_absmean": 0.51755804, "gate_c2b/L5_absmax": 0.76444596, "gate_c2b/L5_rms": 0.53324145, "gate_b2a/L7_absmean": 0.04659304, "gate_b2a/L7_absmax": 0.23874061, "gate_b2a/L7_rms": 0.05676118, "gate_b2a/L11_absmean": 0.09340411, "gate_b2a/L11_absmax": 0.18467705, "gate_b2a/L11_rms": 0.09602153, "gate_b2a/L14_absmean": 0.23633716, "gate_b2a/L14_absmax": 0.36482355, "gate_b2a/L14_rms": 0.23733811, "gate_b2c/L3_absmean": 0.02836453, "gate_b2c/L3_absmax": 0.35867023, "gate_b2c/L3_rms": 0.04558886, "gate_b2c/L5_absmean": 0.06909628, "gate_b2c/L5_absmax": 0.32832143, "gate_b2c/L5_rms": 0.07218434, "gate_b2c/L6_absmean": 0.11029784, "gate_b2c/L6_absmax": 0.30014077, "gate_b2c/L6_rms": 0.11160935}, "streams": {"cos_a2b/round0": 0.326113, "cos_c2b/round0": 0.231284, "cos_b2a/round0": 0.173069, "cos_b2c/round0": 0.097044, "rms_a/round0": 1.460872, "rms_b/round0": 2.609211, "rms_c/round0": 1.320509, "cos_a2b/round1": 0.380629, "cos_c2b/round1": 0.270108, "cos_b2a/round1": 0.188227, "cos_b2c/round1": 0.202074, "rms_a/round1": 2.436855, "rms_b/round1": 4.287495, "rms_c/round1": 2.164609, "cos_a2b/round2": 0.405468, "cos_c2b/round2": 0.408484, "cos_b2a/round2": 0.048947, "cos_b2c/round2": 0.107415, "rms_a/round2": 4.193791, "rms_b/round2": 7.376609, "rms_c/round2": 3.226783}, "alpha_mean": 0.49298646894359877, "alpha_hist": [0.03011, 0.09337, 0.13271, 0.1395, 0.12515, 0.11618, 0.11345, 0.12454, 0.10641, 0.01859], "alpha_by_type": {"special": {"share": 0.00116, "mean": 0.7693, "std": 0.1252}, "whitespace": {"share": 0.03317, "mean": 0.66871, "std": 0.1836}, "digit": {"share": 0.01068, "mean": 0.50223, "std": 0.21955}, "word_start": {"share": 0.57735, "mean": 0.51431, "std": 0.22045}, "word_cont": {"share": 0.26253, "mean": 0.34021, "std": 0.19945}, "punct": {"share": 0.11511, "mean": 0.68016, "std": 0.18771}}, "alpha_p05": 0.12898, "alpha_p50": 0.46405, "alpha_p95": 0.86339}
{"step": 3610, "total_steps": 72479, "tokens": 1892679680, "loss": 3.9701841473579407, "l_mix": 2.4371136128902435, "l_a": 2.577856123447418, "l_c": 2.52731254696846, "z_loss": 15.198497772216797, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2826628088951111, "tokens_per_second": 1407214.5736690476, "eta_seconds": 25658.624454021454, "data_wait_seconds": 2.7805566787719727e-05, "mix": {"fineweb_hq": 0.50942, "dclm": 0.20776, "cosmopedia_v2": 0.20055, "finemath": 0.08227}}
{"step": 3620, "total_steps": 72479, "tokens": 1897922560, "loss": 4.967858135700226, "l_mix": 3.069056957960129, "l_a": 3.17448353767395, "l_c": 3.14912486076355, "z_loss": 17.185306310653687, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.25883570313453674, "tokens_per_second": 1452117.960357797, "eta_seconds": 24861.580379533767, "data_wait_seconds": 2.7298927307128906e-05, "mix": {"fineweb_hq": 0.50967, "dclm": 0.20773, "cosmopedia_v2": 0.20028, "finemath": 0.08232}}
{"step": 3630, "total_steps": 72479, "tokens": 1903165440, "loss": 5.092815637588501, "l_mix": 3.1484751999378204, "l_a": 3.2478802502155304, "l_c": 3.227781057357788, "z_loss": 16.419227838516235, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18497152626514435, "tokens_per_second": 1453088.5054483083, "eta_seconds": 24841.36676923434, "data_wait_seconds": 2.2172927856445312e-05, "mix": {"fineweb_hq": 0.50964, "dclm": 0.20771, "cosmopedia_v2": 0.20028, "finemath": 0.08237}}
{"step": 3640, "total_steps": 72479, "tokens": 1908408320, "loss": 4.9644655585289, "l_mix": 3.0684229731559753, "l_a": 3.1732077896595, "l_c": 3.1414963006973267, "z_loss": 16.313915252685547, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19128429889678955, "tokens_per_second": 1421492.4643932509, "eta_seconds": 25389.836763858795, "data_wait_seconds": 1.728534698486328e-05, "mix": {"fineweb_hq": 0.50962, "dclm": 0.20769, "cosmopedia_v2": 0.20027, "finemath": 0.08242}}
{"step": 3650, "total_steps": 72479, "tokens": 1913651200, "loss": 5.051380634307861, "l_mix": 3.1223001778125763, "l_a": 3.223138988018036, "l_c": 3.2018528878688812, "z_loss": 15.82800304889679, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.26736366748809814, "tokens_per_second": 1435376.6241251817, "eta_seconds": 25140.59247272015, "data_wait_seconds": 1.6033649444580078e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20767, "cosmopedia_v2": 0.20027, "finemath": 0.08247}}
{"step": 3660, "total_steps": 72479, "tokens": 1918894080, "loss": 3.9598962366580963, "l_mix": 2.430370271205902, "l_a": 2.5741328597068787, "l_c": 2.5190964341163635, "z_loss": 15.571585536003113, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2553345561027527, "tokens_per_second": 1439684.820089748, "eta_seconds": 25061.71862654686, "data_wait_seconds": 2.7000904083251953e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20055, "finemath": 0.08224}}
{"step": 3670, "total_steps": 72479, "tokens": 1924136960, "loss": 3.9415594935417175, "l_mix": 2.4180153906345367, "l_a": 2.562495172023773, "l_c": 2.5108003318309784, "z_loss": 15.55485737323761, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.23016192018985748, "tokens_per_second": 1421716.3390732396, "eta_seconds": 25374.775544548032, "data_wait_seconds": 2.0265579223632812e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20763, "cosmopedia_v2": 0.20054, "finemath": 0.08229}}
{"step": 3680, "total_steps": 72479, "tokens": 1929379840, "loss": 3.9077632427215576, "l_mix": 2.3979681730270386, "l_a": 2.5426960587501526, "l_c": 2.484829843044281, "z_loss": 15.372259974479675, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.24176883697509766, "tokens_per_second": 1381281.091405294, "eta_seconds": 26113.794170093537, "data_wait_seconds": 2.6613473892211914e-05, "mix": {"fineweb_hq": 0.50951, "dclm": 0.20761, "cosmopedia_v2": 0.20054, "finemath": 0.08234}}
{"step": 3690, "total_steps": 72479, "tokens": 1934622720, "loss": 4.031489223241806, "l_mix": 2.475774049758911, "l_a": 2.613224357366562, "l_c": 2.567320317029953, "z_loss": 15.51613461971283, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2869931757450104, "tokens_per_second": 1379495.0458848197, "eta_seconds": 26143.803371810915, "data_wait_seconds": 5.710124969482422e-05, "mix": {"fineweb_hq": 0.50949, "dclm": 0.20759, "cosmopedia_v2": 0.20054, "finemath": 0.08238}}
{"step": 3700, "total_steps": 72479, "tokens": 1939865600, "loss": 3.992792993783951, "l_mix": 2.4510334134101868, "l_a": 2.5874609649181366, "l_c": 2.5465281903743744, "z_loss": 15.628312945365906, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.3323324918746948, "tokens_per_second": 1310910.496432203, "eta_seconds": 27507.602120924, "data_wait_seconds": 3.1828880310058594e-05, "mix": {"fineweb_hq": 0.50946, "dclm": 0.20757, "cosmopedia_v2": 0.20054, "finemath": 0.08243}, "gates": {"gate_a2b/L1_absmean": 0.08834776, "gate_a2b/L1_absmax": 0.58879864, "gate_a2b/L1_rms": 0.1339245, "gate_a2b/L3_absmean": 0.26041827, "gate_a2b/L3_absmax": 0.47432962, "gate_a2b/L3_rms": 0.27131996, "gate_a2b/L5_absmean": 0.40186259, "gate_a2b/L5_absmax": 0.61944813, "gate_a2b/L5_rms": 0.41432282, "gate_c2b/L1_absmean": 0.01497833, "gate_c2b/L1_absmax": 0.55619532, "gate_c2b/L1_rms": 0.05313319, "gate_c2b/L3_absmean": 0.26250139, "gate_c2b/L3_absmax": 0.58594888, "gate_c2b/L3_rms": 0.27339295, "gate_c2b/L5_absmean": 0.5197798, "gate_c2b/L5_absmax": 0.77145559, "gate_c2b/L5_rms": 0.53548658, "gate_b2a/L7_absmean": 0.04757637, "gate_b2a/L7_absmax": 0.24140285, "gate_b2a/L7_rms": 0.0578021, "gate_b2a/L11_absmean": 0.09493697, "gate_b2a/L11_absmax": 0.18789814, "gate_b2a/L11_rms": 0.09753212, "gate_b2a/L14_absmean": 0.23891228, "gate_b2a/L14_absmax": 0.37203634, "gate_b2a/L14_rms": 0.23996529, "gate_b2c/L3_absmean": 0.02820032, "gate_b2c/L3_absmax": 0.36179295, "gate_b2c/L3_rms": 0.04600122, "gate_b2c/L5_absmean": 0.06932175, "gate_b2c/L5_absmax": 0.33626148, "gate_b2c/L5_rms": 0.07251766, "gate_b2c/L6_absmean": 0.11106266, "gate_b2c/L6_absmax": 0.303828, "gate_b2c/L6_rms": 0.11239676}, "streams": {"cos_a2b/round0": 0.330212, "cos_c2b/round0": 0.23252, "cos_b2a/round0": 0.169056, "cos_b2c/round0": 0.094353, "rms_a/round0": 1.462085, "rms_b/round0": 2.584952, "rms_c/round0": 1.321951, "cos_a2b/round1": 0.386761, "cos_c2b/round1": 0.262016, "cos_b2a/round1": 0.177888, "cos_b2c/round1": 0.193929, "rms_a/round1": 2.480421, "rms_b/round1": 4.351193, "rms_c/round1": 2.165085, "cos_a2b/round2": 0.402978, "cos_c2b/round2": 0.423509, "cos_b2a/round2": 0.041636, "cos_b2c/round2": 0.09964, "rms_a/round2": 4.260629, "rms_b/round2": 7.786103, "rms_c/round2": 3.288683}, "alpha_mean": 0.48675281121931135, "alpha_hist": [0.03233, 0.09544, 0.1358, 0.13739, 0.12718, 0.11871, 0.1142, 0.12509, 0.0984, 0.01544], "alpha_by_type": {"special": {"share": 0.00104, "mean": 0.77723, "std": 0.09377}, "whitespace": {"share": 0.03763, "mean": 0.6448, "std": 0.17489}, "digit": {"share": 0.01248, "mean": 0.48636, "std": 0.231}, "word_start": {"share": 0.56058, "mean": 0.50855, "std": 0.2154}, "word_cont": {"share": 0.26234, "mean": 0.328, "std": 0.19732}, "punct": {"share": 0.12593, "mean": 0.67087, "std": 0.1912}}, "alpha_p05": 0.12231, "alpha_p50": 0.49033, "alpha_p95": 0.8512}
{"step": 3710, "total_steps": 72479, "tokens": 1945108480, "loss": 3.933313637971878, "l_mix": 2.4139669835567474, "l_a": 2.5549183189868927, "l_c": 2.5044150352478027, "z_loss": 15.465943813323975, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2894705533981323, "tokens_per_second": 1424302.340858566, "eta_seconds": 25313.98035213947, "data_wait_seconds": 1.9162893295288086e-05, "mix": {"fineweb_hq": 0.50943, "dclm": 0.20755, "cosmopedia_v2": 0.20054, "finemath": 0.08248}}
{"step": 3720, "total_steps": 72479, "tokens": 1950351360, "loss": 5.005420625209808, "l_mix": 3.0944624543190002, "l_a": 3.1930136382579803, "l_c": 3.1712575554847717, "z_loss": 16.768038988113403, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.26661717891693115, "tokens_per_second": 1367555.3478137194, "eta_seconds": 26360.555460977554, "data_wait_seconds": 1.8298625946044922e-05, "mix": {"fineweb_hq": 0.50968, "dclm": 0.20753, "cosmopedia_v2": 0.20054, "finemath": 0.08226}}
{"step": 3730, "total_steps": 72479, "tokens": 1955594240, "loss": 5.017277538776398, "l_mix": 3.1004276871681213, "l_a": 3.2082561254501343, "l_c": 3.175759345293045, "z_loss": 16.45048677921295, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.24453219771385193, "tokens_per_second": 1453298.463935944, "eta_seconds": 24801.702201199532, "data_wait_seconds": 1.940131187438965e-05, "mix": {"fineweb_hq": 0.50965, "dclm": 0.20751, "cosmopedia_v2": 0.20054, "finemath": 0.08231}}
{"step": 3740, "total_steps": 72479, "tokens": 1960837120, "loss": 4.938174843788147, "l_mix": 3.0510014593601227, "l_a": 3.1616521179676056, "l_c": 3.1234796941280365, "z_loss": 16.337525367736816, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2049265205860138, "tokens_per_second": 1015006.734584777, "eta_seconds": 35506.20070195198, "data_wait_seconds": 1.8268823623657227e-05, "mix": {"fineweb_hq": 0.50963, "dclm": 0.20775, "cosmopedia_v2": 0.20027, "finemath": 0.08235}}
{"step": 3750, "total_steps": 72479, "tokens": 1966080000, "loss": 4.87170934677124, "l_mix": 3.009169638156891, "l_a": 3.117002695798874, "l_c": 3.08586049079895, "z_loss": 16.80637836456299, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21548670530319214, "tokens_per_second": 1428311.4150701175, "eta_seconds": 25228.244745373726, "data_wait_seconds": 1.5974044799804688e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20773, "cosmopedia_v2": 0.20027, "finemath": 0.0824}}
{"step": 3760, "total_steps": 72479, "tokens": 1971322880, "loss": 4.934904098510742, "l_mix": 3.0521937906742096, "l_a": 3.1513669192790985, "l_c": 3.11886465549469, "z_loss": 16.40811848640442, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.23952683806419373, "tokens_per_second": 1307476.7947928845, "eta_seconds": 27555.783181381226, "data_wait_seconds": 3.293156623840332e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20771, "cosmopedia_v2": 0.20027, "finemath": 0.08245}}
{"step": 3770, "total_steps": 72479, "tokens": 1976565760, "loss": 3.958840847015381, "l_mix": 2.4302425384521484, "l_a": 2.5697942674160004, "l_c": 2.5205149352550507, "z_loss": 15.055690169334412, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.3582186698913574, "tokens_per_second": 1395496.329484939, "eta_seconds": 25813.972728466986, "data_wait_seconds": 5.0187110900878906e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20769, "cosmopedia_v2": 0.20053, "finemath": 0.08223}}
{"step": 3780, "total_steps": 72479, "tokens": 1981808640, "loss": 3.9498362243175507, "l_mix": 2.423307180404663, "l_a": 2.564812958240509, "l_c": 2.5185829401016235, "z_loss": 15.102477431297302, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.296061247587204, "tokens_per_second": 1459124.6416685893, "eta_seconds": 24684.70498230457, "data_wait_seconds": 2.22623348236084e-05, "mix": {"fineweb_hq": 0.50952, "dclm": 0.20767, "cosmopedia_v2": 0.20053, "finemath": 0.08228}}
{"step": 3790, "total_steps": 72479, "tokens": 1987051520, "loss": 3.85842227935791, "l_mix": 2.3680435121059418, "l_a": 2.5091919600963593, "l_c": 2.4535215497016907, "z_loss": 15.646866202354431, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.24272648990154266, "tokens_per_second": 1433454.8631752795, "eta_seconds": 25123.092018556596, "data_wait_seconds": 1.767277717590332e-05, "mix": {"fineweb_hq": 0.5095, "dclm": 0.20765, "cosmopedia_v2": 0.20053, "finemath": 0.08232}}
{"step": 3800, "total_steps": 72479, "tokens": 1992294400, "loss": 3.977390229701996, "l_mix": 2.4422293305397034, "l_a": 2.5802652835845947, "l_c": 2.5317607820034027, "z_loss": 15.530023455619812, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2908378541469574, "tokens_per_second": 1415491.424621373, "eta_seconds": 25438.215255618095, "data_wait_seconds": 2.1189451217651367e-05, "mix": {"fineweb_hq": 0.50947, "dclm": 0.20763, "cosmopedia_v2": 0.20053, "finemath": 0.08237}, "gates": {"gate_a2b/L1_absmean": 0.09024048, "gate_a2b/L1_absmax": 0.58894783, "gate_a2b/L1_rms": 0.13594092, "gate_a2b/L3_absmean": 0.25963041, "gate_a2b/L3_absmax": 0.47199357, "gate_a2b/L3_rms": 0.27028543, "gate_a2b/L5_absmean": 0.4033637, "gate_a2b/L5_absmax": 0.62418979, "gate_a2b/L5_rms": 0.41575783, "gate_c2b/L1_absmean": 0.01486188, "gate_c2b/L1_absmax": 0.55764836, "gate_c2b/L1_rms": 0.05304578, "gate_c2b/L3_absmean": 0.2617707, "gate_c2b/L3_absmax": 0.5802725, "gate_c2b/L3_rms": 0.27247956, "gate_c2b/L5_absmean": 0.52197742, "gate_c2b/L5_absmax": 0.77701724, "gate_c2b/L5_rms": 0.53783607, "gate_b2a/L7_absmean": 0.0479084, "gate_b2a/L7_absmax": 0.24302763, "gate_b2a/L7_rms": 0.05824107, "gate_b2a/L11_absmean": 0.09584368, "gate_b2a/L11_absmax": 0.18947652, "gate_b2a/L11_rms": 0.09835877, "gate_b2a/L14_absmean": 0.24100667, "gate_b2a/L14_absmax": 0.37602869, "gate_b2a/L14_rms": 0.24201766, "gate_b2c/L3_absmean": 0.02822962, "gate_b2c/L3_absmax": 0.36569431, "gate_b2c/L3_rms": 0.04623225, "gate_b2c/L5_absmean": 0.06952059, "gate_b2c/L5_absmax": 0.34511667, "gate_b2c/L5_rms": 0.07283135, "gate_b2c/L6_absmean": 0.11199646, "gate_b2c/L6_absmax": 0.30534518, "gate_b2c/L6_rms": 0.1133035}, "streams": {"cos_a2b/round0": 0.332042, "cos_c2b/round0": 0.225029, "cos_b2a/round0": 0.173291, "cos_b2c/round0": 0.096975, "rms_a/round0": 1.470717, "rms_b/round0": 2.616688, "rms_c/round0": 1.329134, "cos_a2b/round1": 0.38527, "cos_c2b/round1": 0.279227, "cos_b2a/round1": 0.190914, "cos_b2c/round1": 0.196179, "rms_a/round1": 2.467613, "rms_b/round1": 4.30342, "rms_c/round1": 2.195977, "cos_a2b/round2": 0.408674, "cos_c2b/round2": 0.422665, "cos_b2a/round2": 0.051428, "cos_b2c/round2": 0.11148, "rms_a/round2": 4.273885, "rms_b/round2": 7.610009, "rms_c/round2": 3.297534}, "alpha_mean": 0.47713755819098935, "alpha_hist": [0.0265, 0.09985, 0.14989, 0.1512, 0.12961, 0.10716, 0.10744, 0.12242, 0.09254, 0.01338], "alpha_by_type": {"special": {"share": 0.00113, "mean": 0.75515, "std": 0.07874}, "whitespace": {"share": 0.03369, "mean": 0.64139, "std": 0.1794}, "digit": {"share": 0.01059, "mean": 0.41301, "std": 0.24097}, "word_start": {"share": 0.57904, "mean": 0.49122, "std": 0.21817}, "word_cont": {"share": 0.25366, "mean": 0.33537, "std": 0.18855}, "punct": {"share": 0.12189, "mean": 0.66285, "std": 0.20043}}, "alpha_p05": 0.13008, "alpha_p50": 0.4535, "alpha_p95": 0.83548}
{"step": 3810, "total_steps": 72479, "tokens": 1997537280, "loss": 3.9411110877990723, "l_mix": 2.419825851917267, "l_a": 2.5550281703472137, "l_c": 2.5104728639125824, "z_loss": 16.34792971611023, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.3172493875026703, "tokens_per_second": 1473216.1915881652, "eta_seconds": 24437.915410900114, "data_wait_seconds": 1.8805265426635742e-05, "mix": {"fineweb_hq": 0.50945, "dclm": 0.20761, "cosmopedia_v2": 0.20052, "finemath": 0.08241}}
{"step": 3820, "total_steps": 72479, "tokens": 2002780160, "loss": 3.9557780027389526, "l_mix": 2.427912622690201, "l_a": 2.5691747963428497, "l_c": 2.518664985895157, "z_loss": 15.134443521499634, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.25164729356765747, "tokens_per_second": 1320958.100512375, "eta_seconds": 27250.743061447145, "data_wait_seconds": 1.6957521438598633e-05, "mix": {"fineweb_hq": 0.50942, "dclm": 0.20759, "cosmopedia_v2": 0.20052, "finemath": 0.08246}}
{"step": 3830, "total_steps": 72479, "tokens": 2008023040, "loss": 5.119175493717194, "l_mix": 3.1655110716819763, "l_a": 3.268110364675522, "l_c": 3.2383974194526672, "z_loss": 17.119885683059692, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2579226791858673, "tokens_per_second": 1436774.179808917, "eta_seconds": 25050.454982972144, "data_wait_seconds": 1.856684684753418e-05, "mix": {"fineweb_hq": 0.50966, "dclm": 0.20757, "cosmopedia_v2": 0.20052, "finemath": 0.08225}}
{"step": 3840, "total_steps": 72479, "tokens": 2013265920, "loss": 4.985528767108917, "l_mix": 3.081928104162216, "l_a": 3.184230625629425, "l_c": 3.1555038392543793, "z_loss": 16.801991939544678, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.23485103249549866, "tokens_per_second": 871992.3444529223, "eta_seconds": 41269.4036374569, "data_wait_seconds": 2.181529998779297e-05, "mix": {"fineweb_hq": 0.50964, "dclm": 0.20755, "cosmopedia_v2": 0.20052, "finemath": 0.08229}}
{"step": 3850, "total_steps": 72479, "tokens": 2018508800, "loss": 4.981998980045319, "l_mix": 3.079327642917633, "l_a": 3.1848602890968323, "l_c": 3.151798963546753, "z_loss": 16.734938859939575, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.241178497672081, "tokens_per_second": 1410103.2189462518, "eta_seconds": 25516.827894973754, "data_wait_seconds": 1.5676021575927734e-05, "mix": {"fineweb_hq": 0.50961, "dclm": 0.20753, "cosmopedia_v2": 0.20052, "finemath": 0.08234}}
{"step": 3860, "total_steps": 72479, "tokens": 2023751680, "loss": 4.9873740673065186, "l_mix": 3.0828448235988617, "l_a": 3.186929941177368, "l_c": 3.15601122379303, "z_loss": 16.46792483329773, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22625918686389923, "tokens_per_second": 1416777.5715928986, "eta_seconds": 25392.919109773637, "data_wait_seconds": 1.7076730728149414e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20751, "cosmopedia_v2": 0.20052, "finemath": 0.08238}}
{"step": 3870, "total_steps": 72479, "tokens": 2028994560, "loss": 3.9275215566158295, "l_mix": 2.4094834327697754, "l_a": 2.552931070327759, "l_c": 2.5022888779640198, "z_loss": 14.720649719238281, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.28201723098754883, "tokens_per_second": 1469888.0708126964, "eta_seconds": 24471.84660265446, "data_wait_seconds": 2.3663043975830078e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20749, "cosmopedia_v2": 0.20052, "finemath": 0.08243}}
{"step": 3880, "total_steps": 72479, "tokens": 2034237440, "loss": 5.305926561355591, "l_mix": 3.284795254468918, "l_a": 3.3744674921035767, "l_c": 3.356941133737564, "z_loss": 17.08717656135559, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.23854388296604156, "tokens_per_second": 1474279.2319374713, "eta_seconds": 24395.40063569546, "data_wait_seconds": 1.5735626220703125e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20773, "cosmopedia_v2": 0.20026, "finemath": 0.08247}}
{"step": 3890, "total_steps": 72479, "tokens": 2039480320, "loss": 3.8868855834007263, "l_mix": 2.386599063873291, "l_a": 2.5227291584014893, "l_c": 2.473065823316574, "z_loss": 15.480566501617432, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2742327153682709, "tokens_per_second": 1399351.962156425, "eta_seconds": 25697.887739825248, "data_wait_seconds": 3.835558891296387e-05, "mix": {"fineweb_hq": 0.50951, "dclm": 0.20771, "cosmopedia_v2": 0.20051, "finemath": 0.08226}}
{"step": 3900, "total_steps": 72479, "tokens": 2044723200, "loss": 3.947195053100586, "l_mix": 2.424022287130356, "l_a": 2.5588957965373993, "l_c": 2.5132091641426086, "z_loss": 15.411913871765137, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.27423757314682007, "tokens_per_second": 1453832.676637645, "eta_seconds": 24731.282581400872, "data_wait_seconds": 3.17692756652832e-05, "mix": {"fineweb_hq": 0.50949, "dclm": 0.20769, "cosmopedia_v2": 0.20051, "finemath": 0.08231}, "gates": {"gate_a2b/L1_absmean": 0.09258395, "gate_a2b/L1_absmax": 0.58888119, "gate_a2b/L1_rms": 0.13854848, "gate_a2b/L3_absmean": 0.26047856, "gate_a2b/L3_absmax": 0.46889278, "gate_a2b/L3_rms": 0.27097684, "gate_a2b/L5_absmean": 0.40618739, "gate_a2b/L5_absmax": 0.62334418, "gate_a2b/L5_rms": 0.41845453, "gate_c2b/L1_absmean": 0.01524699, "gate_c2b/L1_absmax": 0.55509251, "gate_c2b/L1_rms": 0.05282159, "gate_c2b/L3_absmean": 0.26072869, "gate_c2b/L3_absmax": 0.57496244, "gate_c2b/L3_rms": 0.27124089, "gate_c2b/L5_absmean": 0.52407813, "gate_c2b/L5_absmax": 0.7779476, "gate_c2b/L5_rms": 0.54001927, "gate_b2a/L7_absmean": 0.04891725, "gate_b2a/L7_absmax": 0.2453675, "gate_b2a/L7_rms": 0.05901166, "gate_b2a/L11_absmean": 0.09736311, "gate_b2a/L11_absmax": 0.19345397, "gate_b2a/L11_rms": 0.09984735, "gate_b2a/L14_absmean": 0.24314953, "gate_b2a/L14_absmax": 0.38046557, "gate_b2a/L14_rms": 0.24415672, "gate_b2c/L3_absmean": 0.02929831, "gate_b2c/L3_absmax": 0.36597127, "gate_b2c/L3_rms": 0.04712624, "gate_b2c/L5_absmean": 0.07052677, "gate_b2c/L5_absmax": 0.35190192, "gate_b2c/L5_rms": 0.07383154, "gate_b2c/L6_absmean": 0.11339863, "gate_b2c/L6_absmax": 0.30973652, "gate_b2c/L6_rms": 0.11470256}, "streams": {"cos_a2b/round0": 0.329685, "cos_c2b/round0": 0.226986, "cos_b2a/round0": 0.172288, "cos_b2c/round0": 0.090734, "rms_a/round0": 1.4641, "rms_b/round0": 2.574969, "rms_c/round0": 1.335417, "cos_a2b/round1": 0.380659, "cos_c2b/round1": 0.268304, "cos_b2a/round1": 0.184195, "cos_b2c/round1": 0.188918, "rms_a/round1": 2.469786, "rms_b/round1": 4.293665, "rms_c/round1": 2.175687, "cos_a2b/round2": 0.414133, "cos_c2b/round2": 0.420864, "cos_b2a/round2": 0.041659, "cos_b2c/round2": 0.113285, "rms_a/round2": 4.28722, "rms_b/round2": 7.671, "rms_c/round2": 3.297383}, "alpha_mean": 0.47797055488127427, "alpha_hist": [0.02502, 0.09726, 0.1443, 0.14835, 0.13271, 0.11618, 0.11852, 0.12511, 0.08327, 0.00929], "alpha_by_type": {"special": {"share": 0.00104, "mean": 0.77241, "std": 0.06864}, "whitespace": {"share": 0.03293, "mean": 0.67693, "std": 0.15453}, "digit": {"share": 0.01064, "mean": 0.53811, "std": 0.22187}, "word_start": {"share": 0.57339, "mean": 0.49102, "std": 0.21253}, "word_cont": {"share": 0.2571, "mean": 0.33719, "std": 0.18957}, "punct": {"share": 0.12491, "mean": 0.64782, "std": 0.18729}}, "alpha_p05": 0.14391, "alpha_p50": 0.46891, "alpha_p95": 0.82502}
{"step": 3910, "total_steps": 72479, "tokens": 2049966080, "loss": 3.9526631832122803, "l_mix": 2.4259188175201416, "l_a": 2.56397607922554, "l_c": 2.520191103219986, "z_loss": 14.94190764427185, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.284969687461853, "tokens_per_second": 1439153.417336072, "eta_seconds": 24979.896819162368, "data_wait_seconds": 3.9696693420410156e-05, "mix": {"fineweb_hq": 0.50946, "dclm": 0.20767, "cosmopedia_v2": 0.20051, "finemath": 0.08235}}
{"step": 3920, "total_steps": 72479, "tokens": 2055208960, "loss": 3.9052623212337494, "l_mix": 2.395579904317856, "l_a": 2.545387715101242, "l_c": 2.4816713631153107, "z_loss": 15.646105885505676, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.28035348653793335, "tokens_per_second": 1359424.396218256, "eta_seconds": 26441.088663697243, "data_wait_seconds": 2.0295381546020508e-05, "mix": {"fineweb_hq": 0.50944, "dclm": 0.20765, "cosmopedia_v2": 0.20051, "finemath": 0.0824}}
{"step": 3930, "total_steps": 72479, "tokens": 2060451840, "loss": 3.9469853937625885, "l_mix": 2.423410087823868, "l_a": 2.5657795071601868, "l_c": 2.5077193677425385, "z_loss": 15.256299495697021, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2660316824913025, "tokens_per_second": 1474617.933184219, "eta_seconds": 24372.020238757133, "data_wait_seconds": 1.9669532775878906e-05, "mix": {"fineweb_hq": 0.50941, "dclm": 0.20763, "cosmopedia_v2": 0.20051, "finemath": 0.08244}}
{"step": 3940, "total_steps": 72479, "tokens": 2065694720, "loss": 4.966131210327148, "l_mix": 3.0699659883975983, "l_a": 3.173185884952545, "l_c": 3.1419506669044495, "z_loss": 16.240925312042236, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2034338116645813, "tokens_per_second": 1413200.4349141002, "eta_seconds": 25427.51498246193, "data_wait_seconds": 2.396106719970703e-05, "mix": {"fineweb_hq": 0.50964, "dclm": 0.20761, "cosmopedia_v2": 0.20051, "finemath": 0.08223}}
{"step": 3950, "total_steps": 72479, "tokens": 2070937600, "loss": 5.0987953543663025, "l_mix": 3.1526932418346405, "l_a": 3.2560678720474243, "l_c": 3.2256096601486206, "z_loss": 15.988236665725708, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2247261255979538, "tokens_per_second": 1377200.8319933584, "eta_seconds": 26088.3754332304, "data_wait_seconds": 2.428889274597168e-05, "mix": {"fineweb_hq": 0.50962, "dclm": 0.20759, "cosmopedia_v2": 0.20051, "finemath": 0.08228}}
{"step": 3960, "total_steps": 72479, "tokens": 2076180480, "loss": 5.067540407180786, "l_mix": 3.1329891979694366, "l_a": 3.240999072790146, "l_c": 3.2019177675247192, "z_loss": 16.761089086532593, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.23214514553546906, "tokens_per_second": 1381799.7814982296, "eta_seconds": 25997.753041362765, "data_wait_seconds": 1.749396324157715e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20758, "cosmopedia_v2": 0.20051, "finemath": 0.08232}}
{"step": 3970, "total_steps": 72479, "tokens": 2081423360, "loss": 5.106565535068512, "l_mix": 3.1574412882328033, "l_a": 3.2588768005371094, "l_c": 3.2329003512859344, "z_loss": 15.910486340522766, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22600063681602478, "tokens_per_second": 1451155.9569335254, "eta_seconds": 24751.610204529763, "data_wait_seconds": 2.0891427993774414e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20756, "cosmopedia_v2": 0.2005, "finemath": 0.08237}}
{"step": 3980, "total_steps": 72479, "tokens": 2086666240, "loss": 3.905110776424408, "l_mix": 2.3978256285190582, "l_a": 2.5379876494407654, "l_c": 2.481111764907837, "z_loss": 15.551933765411377, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.26571977138519287, "tokens_per_second": 1449907.9925098768, "eta_seconds": 24769.29839515686, "data_wait_seconds": 3.0100345611572266e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20754, "cosmopedia_v2": 0.2005, "finemath": 0.08241}}
{"step": 3990, "total_steps": 72479, "tokens": 2091909120, "loss": 3.868111401796341, "l_mix": 2.3741211891174316, "l_a": 2.513451963663101, "l_c": 2.4611723721027374, "z_loss": 16.028381943702698, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2718973755836487, "tokens_per_second": 1392007.8298200753, "eta_seconds": 25795.80377550125, "data_wait_seconds": 2.9861927032470703e-05, "mix": {"fineweb_hq": 0.50952, "dclm": 0.20752, "cosmopedia_v2": 0.2005, "finemath": 0.08246}}
{"step": 4000, "total_steps": 72479, "tokens": 2097152000, "loss": 5.242976248264313, "l_mix": 3.246356099843979, "l_a": 3.333175927400589, "l_c": 3.316793769598007, "z_loss": 16.291202306747437, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2197742611169815, "tokens_per_second": 1474537.6431612384, "eta_seconds": 24348.458052945138, "data_wait_seconds": 1.9043684005737305e-05, "mix": {"fineweb_hq": 0.5095, "dclm": 0.20775, "cosmopedia_v2": 0.2005, "finemath": 0.08225}, "gates": {"gate_a2b/L1_absmean": 0.09484355, "gate_a2b/L1_absmax": 0.59326935, "gate_a2b/L1_rms": 0.14046848, "gate_a2b/L3_absmean": 0.26123446, "gate_a2b/L3_absmax": 0.47097394, "gate_a2b/L3_rms": 0.27156502, "gate_a2b/L5_absmean": 0.40921813, "gate_a2b/L5_absmax": 0.62509733, "gate_a2b/L5_rms": 0.42129433, "gate_c2b/L1_absmean": 0.01531897, "gate_c2b/L1_absmax": 0.55790633, "gate_c2b/L1_rms": 0.05262168, "gate_c2b/L3_absmean": 0.260236, "gate_c2b/L3_absmax": 0.57092917, "gate_c2b/L3_rms": 0.27058712, "gate_c2b/L5_absmean": 0.52658814, "gate_c2b/L5_absmax": 0.77554548, "gate_c2b/L5_rms": 0.54236799, "gate_b2a/L7_absmean": 0.05036599, "gate_b2a/L7_absmax": 0.24708055, "gate_b2a/L7_rms": 0.06003395, "gate_b2a/L11_absmean": 0.09879743, "gate_b2a/L11_absmax": 0.19559002, "gate_b2a/L11_rms": 0.10114867, "gate_b2a/L14_absmean": 0.24532112, "gate_b2a/L14_absmax": 0.3881138, "gate_b2a/L14_rms": 0.24631324, "gate_b2c/L3_absmean": 0.02983703, "gate_b2c/L3_absmax": 0.36974385, "gate_b2c/L3_rms": 0.04779431, "gate_b2c/L5_absmean": 0.07144928, "gate_b2c/L5_absmax": 0.35668051, "gate_b2c/L5_rms": 0.07477812, "gate_b2c/L6_absmean": 0.11485235, "gate_b2c/L6_absmax": 0.3134304, "gate_b2c/L6_rms": 0.11617751}, "streams": {"cos_a2b/round0": 0.354645, "cos_c2b/round0": 0.256352, "cos_b2a/round0": 0.184752, "cos_b2c/round0": 0.130995, "rms_a/round0": 1.430201, "rms_b/round0": 2.573375, "rms_c/round0": 1.296763, "cos_a2b/round1": 0.402029, "cos_c2b/round1": 0.28653, "cos_b2a/round1": 0.200004, "cos_b2c/round1": 0.210366, "rms_a/round1": 2.438504, "rms_b/round1": 4.19647, "rms_c/round1": 2.159403, "cos_a2b/round2": 0.437179, "cos_c2b/round2": 0.436796, "cos_b2a/round2": 0.093368, "cos_b2c/round2": 0.151098, "rms_a/round2": 4.305358, "rms_b/round2": 7.74685, "rms_c/round2": 3.293962}, "alpha_mean": 0.5027071677329076, "alpha_hist": [0.00598, 0.02327, 0.084, 0.1562, 0.21378, 0.22865, 0.17029, 0.09592, 0.02116, 0.00075], "alpha_by_type": {"special": {"share": 0.00034, "mean": 0.5609, "std": 0.16236}, "whitespace": {"share": 0.0524, "mean": 0.56645, "std": 0.12143}, "digit": {"share": 0.02573, "mean": 0.51549, "std": 0.1448}, "word_start": {"share": 0.60052, "mean": 0.51858, "std": 0.14739}, "word_cont": {"share": 0.22383, "mean": 0.39304, "std": 0.14634}, "punct": {"share": 0.09718, "mean": 0.61926, "std": 0.14644}}, "alpha_p05": 0.23971, "alpha_p50": 0.50976, "alpha_p95": 0.75347}
{"step": 4010, "total_steps": 72479, "tokens": 2102394880, "loss": 5.066747605800629, "l_mix": 3.136075437068939, "l_a": 3.2221852242946625, "l_c": 3.207854747772217, "z_loss": 16.600378274917603, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.25534239411354065, "tokens_per_second": 1449176.4596868283, "eta_seconds": 24770.948238945006, "data_wait_seconds": 1.6063451766967773e-05, "mix": {"fineweb_hq": 0.50948, "dclm": 0.20773, "cosmopedia_v2": 0.2005, "finemath": 0.08229}}
{"step": 4020, "total_steps": 72479, "tokens": 2107637760, "loss": 3.9164824783802032, "l_mix": 2.4039749205112457, "l_a": 2.5469546020030975, "l_c": 2.4896143972873688, "z_loss": 15.368208050727844, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22878654301166534, "tokens_per_second": 1413183.815194785, "eta_seconds": 25398.13420312405, "data_wait_seconds": 2.0295381546020508e-05, "mix": {"fineweb_hq": 0.50945, "dclm": 0.20771, "cosmopedia_v2": 0.2005, "finemath": 0.08234}}
{"step": 4030, "total_steps": 72479, "tokens": 2112880640, "loss": 3.9394712448120117, "l_mix": 2.4196884632110596, "l_a": 2.5559499859809875, "l_c": 2.5047871470451355, "z_loss": 15.615687608718872, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.3216230273246765, "tokens_per_second": 1417237.8613327013, "eta_seconds": 25321.78280804157, "data_wait_seconds": 2.473592758178711e-05, "mix": {"fineweb_hq": 0.50943, "dclm": 0.20769, "cosmopedia_v2": 0.2005, "finemath": 0.08238}}
{"step": 4040, "total_steps": 72479, "tokens": 2118123520, "loss": 3.9234712421894073, "l_mix": 2.4081741273403168, "l_a": 2.5490958094596863, "l_c": 2.4967114329338074, "z_loss": 15.549645185470581, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.3088960349559784, "tokens_per_second": 1360200.8545434899, "eta_seconds": 26379.7411331892, "data_wait_seconds": 2.9087066650390625e-05, "mix": {"fineweb_hq": 0.50941, "dclm": 0.20767, "cosmopedia_v2": 0.2005, "finemath": 0.08243}}
{"step": 4050, "total_steps": 72479, "tokens": 2123366400, "loss": 5.119623422622681, "l_mix": 3.1662232875823975, "l_a": 3.2668343484401703, "l_c": 3.2391396164894104, "z_loss": 16.078250885009766, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.24179372191429138, "tokens_per_second": 1400620.977151776, "eta_seconds": 25614.71242916584, "data_wait_seconds": 2.3573637008666992e-05, "mix": {"fineweb_hq": 0.50963, "dclm": 0.20765, "cosmopedia_v2": 0.20049, "finemath": 0.08222}}
{"step": 4060, "total_steps": 72479, "tokens": 2128609280, "loss": 5.042381286621094, "l_mix": 3.118232697248459, "l_a": 3.2188422083854675, "l_c": 3.1898061633110046, "z_loss": 15.540305018424988, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2800588011741638, "tokens_per_second": 1436271.1891370192, "eta_seconds": 24975.269951319693, "data_wait_seconds": 2.1398067474365234e-05, "mix": {"fineweb_hq": 0.50961, "dclm": 0.20764, "cosmopedia_v2": 0.20049, "finemath": 0.08227}}
{"step": 4070, "total_steps": 72479, "tokens": 2133852160, "loss": 5.033787488937378, "l_mix": 3.1118657290935516, "l_a": 3.2133749127388, "l_c": 3.1877526938915253, "z_loss": 15.834462404251099, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.23638668656349182, "tokens_per_second": 1402586.9677972244, "eta_seconds": 25571.332555818557, "data_wait_seconds": 1.850724220275879e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20762, "cosmopedia_v2": 0.20049, "finemath": 0.08231}}
{"step": 4080, "total_steps": 72479, "tokens": 2139095040, "loss": 5.091442286968231, "l_mix": 3.1453988552093506, "l_a": 3.2570612132549286, "l_c": 3.2242186963558197, "z_loss": 16.594975233078003, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2648213803768158, "tokens_per_second": 1391033.863473779, "eta_seconds": 25779.94386308193, "data_wait_seconds": 2.2858381271362305e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.2076, "cosmopedia_v2": 0.20049, "finemath": 0.08235}}
{"step": 4090, "total_steps": 72479, "tokens": 2144337920, "loss": 3.910928189754486, "l_mix": 2.4007644951343536, "l_a": 2.5370417833328247, "l_c": 2.4915571212768555, "z_loss": 15.839821696281433, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2659417688846588, "tokens_per_second": 1441321.4395653561, "eta_seconds": 24876.846377038954, "data_wait_seconds": 2.6911497116088867e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20758, "cosmopedia_v2": 0.20049, "finemath": 0.0824}}
{"step": 4100, "total_steps": 72479, "tokens": 2149580800, "loss": 3.916679263114929, "l_mix": 2.4035688936710358, "l_a": 2.543868839740753, "l_c": 2.494580775499344, "z_loss": 15.75391411781311, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.25890836119651794, "tokens_per_second": 1478153.111618537, "eta_seconds": 24253.434147119522, "data_wait_seconds": 1.6003847122192383e-05, "mix": {"fineweb_hq": 0.50951, "dclm": 0.20756, "cosmopedia_v2": 0.20049, "finemath": 0.08244}, "gates": {"gate_a2b/L1_absmean": 0.09687936, "gate_a2b/L1_absmax": 0.59050572, "gate_a2b/L1_rms": 0.14189632, "gate_a2b/L3_absmean": 0.26152515, "gate_a2b/L3_absmax": 0.46805218, "gate_a2b/L3_rms": 0.27156034, "gate_a2b/L5_absmean": 0.41120988, "gate_a2b/L5_absmax": 0.62879038, "gate_a2b/L5_rms": 0.4231579, "gate_c2b/L1_absmean": 0.01690873, "gate_c2b/L1_absmax": 0.56137305, "gate_c2b/L1_rms": 0.05297367, "gate_c2b/L3_absmean": 0.25902152, "gate_c2b/L3_absmax": 0.57042426, "gate_c2b/L3_rms": 0.26919213, "gate_c2b/L5_absmean": 0.52955371, "gate_c2b/L5_absmax": 0.77056348, "gate_c2b/L5_rms": 0.54540795, "gate_b2a/L7_absmean": 0.05124815, "gate_b2a/L7_absmax": 0.25049964, "gate_b2a/L7_rms": 0.06082332, "gate_b2a/L11_absmean": 0.10011553, "gate_b2a/L11_absmax": 0.19509579, "gate_b2a/L11_rms": 0.10251532, "gate_b2a/L14_absmean": 0.24772206, "gate_b2a/L14_absmax": 0.386518, "gate_b2a/L14_rms": 0.2487015, "gate_b2c/L3_absmean": 0.0303502, "gate_b2c/L3_absmax": 0.36904508, "gate_b2c/L3_rms": 0.04806674, "gate_b2c/L5_absmean": 0.0725617, "gate_b2c/L5_absmax": 0.35565415, "gate_b2c/L5_rms": 0.07587975, "gate_b2c/L6_absmean": 0.11579005, "gate_b2c/L6_absmax": 0.31952801, "gate_b2c/L6_rms": 0.1171701}, "streams": {"cos_a2b/round0": 0.328881, "cos_c2b/round0": 0.223432, "cos_b2a/round0": 0.172936, "cos_b2c/round0": 0.087971, "rms_a/round0": 1.459646, "rms_b/round0": 2.574955, "rms_c/round0": 1.33572, "cos_a2b/round1": 0.375406, "cos_c2b/round1": 0.260481, "cos_b2a/round1": 0.18053, "cos_b2c/round1": 0.182936, "rms_a/round1": 2.474128, "rms_b/round1": 4.358332, "rms_c/round1": 2.211019, "cos_a2b/round2": 0.414768, "cos_c2b/round2": 0.418783, "cos_b2a/round2": 0.038983, "cos_b2c/round2": 0.100187, "rms_a/round2": 4.33275, "rms_b/round2": 7.74806, "rms_c/round2": 3.330822}, "alpha_mean": 0.4743169638744149, "alpha_hist": [0.03499, 0.10582, 0.14433, 0.14153, 0.12927, 0.11241, 0.10358, 0.11636, 0.09566, 0.01605], "alpha_by_type": {"special": {"share": 0.00111, "mean": 0.73936, "std": 0.09979}, "whitespace": {"share": 0.04414, "mean": 0.63705, "std": 0.17595}, "digit": {"share": 0.01164, "mean": 0.43717, "std": 0.2159}, "word_start": {"share": 0.56036, "mean": 0.48677, "std": 0.22253}, "word_cont": {"share": 0.25473, "mean": 0.32166, "std": 0.19007}, "punct": {"share": 0.12801, "mean": 0.66856, "std": 0.19511}}, "alpha_p05": 0.12316, "alpha_p50": 0.45968, "alpha_p95": 0.85537}
{"step": 4110, "total_steps": 72479, "tokens": 2154823680, "loss": 5.29905903339386, "l_mix": 3.280347317457199, "l_a": 3.3745166659355164, "l_c": 3.348983973264694, "z_loss": 16.613572597503662, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18730519711971283, "tokens_per_second": 1461421.6708212737, "eta_seconds": 24527.51795575619, "data_wait_seconds": 2.300739288330078e-05, "mix": {"fineweb_hq": 0.50949, "dclm": 0.20779, "cosmopedia_v2": 0.20049, "finemath": 0.08224}}
{"step": 4120, "total_steps": 72479, "tokens": 2160066560, "loss": 5.286600768566132, "l_mix": 3.2718963027000427, "l_a": 3.366477370262146, "l_c": 3.3434876799583435, "z_loss": 17.15038013458252, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.27209270000457764, "tokens_per_second": 1405380.188101662, "eta_seconds": 25501.85614926815, "data_wait_seconds": 2.804398536682129e-05, "mix": {"fineweb_hq": 0.50947, "dclm": 0.20777, "cosmopedia_v2": 0.20049, "finemath": 0.08228}}
{"step": 4130, "total_steps": 72479, "tokens": 2165309440, "loss": 5.233900725841522, "l_mix": 3.239582806825638, "l_a": 3.3342688381671906, "l_c": 3.3079220354557037, "z_loss": 16.606196403503418, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.235849067568779, "tokens_per_second": 1446192.8278485783, "eta_seconds": 24778.549458932877, "data_wait_seconds": 3.3468008041381836e-05, "mix": {"fineweb_hq": 0.50944, "dclm": 0.20775, "cosmopedia_v2": 0.20048, "finemath": 0.08232}}
{"step": 4140, "total_steps": 72479, "tokens": 2170552320, "loss": 5.29242604970932, "l_mix": 3.2752197980880737, "l_a": 3.3681493401527405, "l_c": 3.3504059612751007, "z_loss": 16.396226406097412, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22017188370227814, "tokens_per_second": 1378239.560793103, "eta_seconds": 25996.436796069145, "data_wait_seconds": 1.913309097290039e-05, "mix": {"fineweb_hq": 0.50942, "dclm": 0.20773, "cosmopedia_v2": 0.20048, "finemath": 0.08237}}
{"step": 4150, "total_steps": 72479, "tokens": 2175795200, "loss": 5.21146696805954, "l_mix": 3.225970596075058, "l_a": 3.3199439644813538, "l_c": 3.292861044406891, "z_loss": 16.5480078458786, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22360575199127197, "tokens_per_second": 1445168.4668442823, "eta_seconds": 24788.857198238373, "data_wait_seconds": 1.9371509552001953e-05, "mix": {"fineweb_hq": 0.5094, "dclm": 0.20771, "cosmopedia_v2": 0.20048, "finemath": 0.08241}}
{"step": 4160, "total_steps": 72479, "tokens": 2181038080, "loss": 4.958337128162384, "l_mix": 3.0658954083919525, "l_a": 3.166660726070404, "l_c": 3.1359250843524933, "z_loss": 16.659494161605835, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1755272001028061, "tokens_per_second": 1462392.2805869512, "eta_seconds": 24493.31301012039, "data_wait_seconds": 2.053380012512207e-05, "mix": {"fineweb_hq": 0.50962, "dclm": 0.20769, "cosmopedia_v2": 0.20024, "finemath": 0.08245}}
{"step": 4170, "total_steps": 72479, "tokens": 2186280960, "loss": 5.0089240074157715, "l_mix": 3.0975181460380554, "l_a": 3.199272632598877, "l_c": 3.1666556894779205, "z_loss": 16.272780179977417, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21691152453422546, "tokens_per_second": 1263658.3156641405, "eta_seconds": 28341.19678401947, "data_wait_seconds": 1.7464160919189453e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20767, "cosmopedia_v2": 0.20048, "finemath": 0.08225}}
{"step": 4180, "total_steps": 72479, "tokens": 2191523840, "loss": 4.991647481918335, "l_mix": 3.086973339319229, "l_a": 3.1875021159648895, "l_c": 3.156079411506653, "z_loss": 15.996803879737854, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2940361797809601, "tokens_per_second": 1349248.6473615505, "eta_seconds": 26539.47156591415, "data_wait_seconds": 3.349781036376953e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20048, "finemath": 0.0823}}
{"step": 4190, "total_steps": 72479, "tokens": 2196766720, "loss": 5.103009521961212, "l_mix": 3.1542525589466095, "l_a": 3.2625170052051544, "l_c": 3.227790027856827, "z_loss": 16.649497985839844, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.33040595054626465, "tokens_per_second": 1444758.9577898982, "eta_seconds": 24781.367880749705, "data_wait_seconds": 1.952052116394043e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20764, "cosmopedia_v2": 0.20048, "finemath": 0.08234}}
{"step": 4200, "total_steps": 72479, "tokens": 2202009600, "loss": 3.9058754444122314, "l_mix": 2.397334337234497, "l_a": 2.538589656352997, "l_c": 2.4847571849823, "z_loss": 15.369924664497375, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.23685258626937866, "tokens_per_second": 1301889.8904979501, "eta_seconds": 27496.84179382324, "data_wait_seconds": 2.2470951080322266e-05, "mix": {"fineweb_hq": 0.50952, "dclm": 0.20762, "cosmopedia_v2": 0.20048, "finemath": 0.08238}, "gates": {"gate_a2b/L1_absmean": 0.0986174, "gate_a2b/L1_absmax": 0.58591604, "gate_a2b/L1_rms": 0.14358844, "gate_a2b/L3_absmean": 0.26167265, "gate_a2b/L3_absmax": 0.47142389, "gate_a2b/L3_rms": 0.27152401, "gate_a2b/L5_absmean": 0.41296053, "gate_a2b/L5_absmax": 0.62539804, "gate_a2b/L5_rms": 0.42468274, "gate_c2b/L1_absmean": 0.01431948, "gate_c2b/L1_absmax": 0.5580548, "gate_c2b/L1_rms": 0.05235875, "gate_c2b/L3_absmean": 0.25778159, "gate_c2b/L3_absmax": 0.57160312, "gate_c2b/L3_rms": 0.26792142, "gate_c2b/L5_absmean": 0.5311839, "gate_c2b/L5_absmax": 0.77446961, "gate_c2b/L5_rms": 0.54711562, "gate_b2a/L7_absmean": 0.05188444, "gate_b2a/L7_absmax": 0.25037998, "gate_b2a/L7_rms": 0.06156595, "gate_b2a/L11_absmean": 0.10169618, "gate_b2a/L11_absmax": 0.1984652, "gate_b2a/L11_rms": 0.10406237, "gate_b2a/L14_absmean": 0.24907045, "gate_b2a/L14_absmax": 0.38849768, "gate_b2a/L14_rms": 0.25007337, "gate_b2c/L3_absmean": 0.03072683, "gate_b2c/L3_absmax": 0.37344122, "gate_b2c/L3_rms": 0.04917704, "gate_b2c/L5_absmean": 0.07325006, "gate_b2c/L5_absmax": 0.36713997, "gate_b2c/L5_rms": 0.0766609, "gate_b2c/L6_absmean": 0.11654888, "gate_b2c/L6_absmax": 0.31526181, "gate_b2c/L6_rms": 0.1178918}, "streams": {"cos_a2b/round0": 0.330676, "cos_c2b/round0": 0.220715, "cos_b2a/round0": 0.177134, "cos_b2c/round0": 0.086844, "rms_a/round0": 1.44704, "rms_b/round0": 2.56097, "rms_c/round0": 1.346297, "cos_a2b/round1": 0.373033, "cos_c2b/round1": 0.268308, "cos_b2a/round1": 0.191041, "cos_b2c/round1": 0.182661, "rms_a/round1": 2.44002, "rms_b/round1": 4.2926, "rms_c/round1": 2.211832, "cos_a2b/round2": 0.413804, "cos_c2b/round2": 0.428725, "cos_b2a/round2": 0.043165, "cos_b2c/round2": 0.111009, "rms_a/round2": 4.318036, "rms_b/round2": 7.851502, "rms_c/round2": 3.34683}, "alpha_mean": 0.469523538537473, "alpha_hist": [0.03085, 0.10536, 0.1488, 0.14491, 0.13106, 0.11383, 0.11176, 0.12486, 0.07979, 0.00877], "alpha_by_type": {"special": {"share": 0.00111, "mean": 0.75775, "std": 0.09357}, "whitespace": {"share": 0.03194, "mean": 0.6973, "std": 0.14391}, "digit": {"share": 0.01299, "mean": 0.57152, "std": 0.20805}, "word_start": {"share": 0.56602, "mean": 0.47885, "std": 0.21404}, "word_cont": {"share": 0.26677, "mean": 0.32796, "std": 0.18732}, "punct": {"share": 0.12117, "mean": 0.66398, "std": 0.18022}}, "alpha_p05": 0.12421, "alpha_p50": 0.45956, "alpha_p95": 0.82976}
{"step": 4210, "total_steps": 72479, "tokens": 2207252480, "loss": 3.884353816509247, "l_mix": 2.383951485157013, "l_a": 2.5287151634693146, "l_c": 2.4675460755825043, "z_loss": 15.239593863487244, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.24858196079730988, "tokens_per_second": 1462082.2101396157, "eta_seconds": 24480.57792084217, "data_wait_seconds": 2.7120113372802734e-05, "mix": {"fineweb_hq": 0.5095, "dclm": 0.2076, "cosmopedia_v2": 0.20048, "finemath": 0.08242}}
{"step": 4220, "total_steps": 72479, "tokens": 2212495360, "loss": 3.396174281835556, "l_mix": 2.0965665727853775, "l_a": 2.188634902238846, "l_c": 2.1373570263385773, "z_loss": 18.100989818572998, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.34628158807754517, "tokens_per_second": 1482924.900899652, "eta_seconds": 24132.964906239507, "data_wait_seconds": 2.962350845336914e-05, "mix": {"fineweb_hq": 0.50948, "dclm": 0.20758, "cosmopedia_v2": 0.20047, "finemath": 0.08246}}
{"step": 4230, "total_steps": 72479, "tokens": 2217738240, "loss": 5.185253202915192, "l_mix": 3.2058229446411133, "l_a": 3.3080720603466034, "l_c": 3.284544199705124, "z_loss": 16.45422601699829, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2622598707675934, "tokens_per_second": 1479150.9558496606, "eta_seconds": 24190.99387421608, "data_wait_seconds": 2.0295381546020508e-05, "mix": {"fineweb_hq": 0.50969, "dclm": 0.20757, "cosmopedia_v2": 0.20047, "finemath": 0.08227}}
{"step": 4240, "total_steps": 72479, "tokens": 2222981120, "loss": 4.977046608924866, "l_mix": 3.0769669711589813, "l_a": 3.177531599998474, "l_c": 3.150465428829193, "z_loss": 16.804792642593384, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.24040472507476807, "tokens_per_second": 1455380.6491961507, "eta_seconds": 24582.495893263815, "data_wait_seconds": 1.6629695892333984e-05, "mix": {"fineweb_hq": 0.50967, "dclm": 0.20755, "cosmopedia_v2": 0.20047, "finemath": 0.08231}}
{"step": 4250, "total_steps": 72479, "tokens": 2228224000, "loss": 5.180212318897247, "l_mix": 3.203896939754486, "l_a": 3.305890679359436, "l_c": 3.2764923572540283, "z_loss": 16.00404703617096, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.25202828645706177, "tokens_per_second": 1472100.8742954961, "eta_seconds": 24299.72468369007, "data_wait_seconds": 2.0831823348999023e-05, "mix": {"fineweb_hq": 0.50965, "dclm": 0.20753, "cosmopedia_v2": 0.20047, "finemath": 0.08235}}
{"step": 4260, "total_steps": 72479, "tokens": 2233466880, "loss": 5.067972123622894, "l_mix": 3.133943200111389, "l_a": 3.237250417470932, "l_c": 3.2039357125759125, "z_loss": 16.729759216308594, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2124069333076477, "tokens_per_second": 1431907.9340157234, "eta_seconds": 24978.14435017109, "data_wait_seconds": 2.1845102310180664e-05, "mix": {"fineweb_hq": 0.50962, "dclm": 0.20751, "cosmopedia_v2": 0.20047, "finemath": 0.08239}}
{"step": 4270, "total_steps": 72479, "tokens": 2238709760, "loss": 5.0875091552734375, "l_mix": 3.1446653306484222, "l_a": 3.249161571264267, "l_c": 3.221508502960205, "z_loss": 16.428120851516724, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.30523985624313354, "tokens_per_second": 1342703.3305785512, "eta_seconds": 26633.701859211924, "data_wait_seconds": 1.874566078186035e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20749, "cosmopedia_v2": 0.20047, "finemath": 0.08244}}
{"step": 4280, "total_steps": 72479, "tokens": 2243952640, "loss": 5.062064349651337, "l_mix": 3.1308065950870514, "l_a": 3.2341873347759247, "l_c": 3.197877049446106, "z_loss": 16.38254690170288, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21009814739227295, "tokens_per_second": 1397173.7713495616, "eta_seconds": 25591.60359663963, "data_wait_seconds": 2.2411346435546875e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20771, "cosmopedia_v2": 0.20047, "finemath": 0.08224}}
{"step": 4290, "total_steps": 72479, "tokens": 2249195520, "loss": 5.080406606197357, "l_mix": 3.140470117330551, "l_a": 3.2441572844982147, "l_c": 3.216629534959793, "z_loss": 17.002913236618042, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.27125951647758484, "tokens_per_second": 1433250.162828923, "eta_seconds": 24943.77838509083, "data_wait_seconds": 2.2113323211669922e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20769, "cosmopedia_v2": 0.20047, "finemath": 0.08228}}
{"step": 4300, "total_steps": 72479, "tokens": 2254438400, "loss": 5.08098304271698, "l_mix": 3.1426542699337006, "l_a": 3.2411956787109375, "l_c": 3.2144651114940643, "z_loss": 16.30358874797821, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2635476589202881, "tokens_per_second": 1375313.6595421087, "eta_seconds": 25990.748585963247, "data_wait_seconds": 3.319978713989258e-05, "mix": {"fineweb_hq": 0.50953, "dclm": 0.20767, "cosmopedia_v2": 0.20047, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.0996512, "gate_a2b/L1_absmax": 0.58916157, "gate_a2b/L1_rms": 0.1445419, "gate_a2b/L3_absmean": 0.26218042, "gate_a2b/L3_absmax": 0.46781459, "gate_a2b/L3_rms": 0.27194709, "gate_a2b/L5_absmean": 0.41444167, "gate_a2b/L5_absmax": 0.63182861, "gate_a2b/L5_rms": 0.42613015, "gate_c2b/L1_absmean": 0.01540209, "gate_c2b/L1_absmax": 0.55422324, "gate_c2b/L1_rms": 0.05225285, "gate_c2b/L3_absmean": 0.25637588, "gate_c2b/L3_absmax": 0.56090122, "gate_c2b/L3_rms": 0.26635286, "gate_c2b/L5_absmean": 0.53280103, "gate_c2b/L5_absmax": 0.77428073, "gate_c2b/L5_rms": 0.54873884, "gate_b2a/L7_absmean": 0.05264405, "gate_b2a/L7_absmax": 0.25427544, "gate_b2a/L7_rms": 0.06217925, "gate_b2a/L11_absmean": 0.10194969, "gate_b2a/L11_absmax": 0.20334417, "gate_b2a/L11_rms": 0.10431572, "gate_b2a/L14_absmean": 0.25052345, "gate_b2a/L14_absmax": 0.39356339, "gate_b2a/L14_rms": 0.25153539, "gate_b2c/L3_absmean": 0.03110608, "gate_b2c/L3_absmax": 0.37552077, "gate_b2c/L3_rms": 0.04945234, "gate_b2c/L5_absmean": 0.07373529, "gate_b2c/L5_absmax": 0.36770305, "gate_b2c/L5_rms": 0.07713153, "gate_b2c/L6_absmean": 0.11788492, "gate_b2c/L6_absmax": 0.32974339, "gate_b2c/L6_rms": 0.11927629}, "streams": {"cos_a2b/round0": 0.352194, "cos_c2b/round0": 0.245579, "cos_b2a/round0": 0.182262, "cos_b2c/round0": 0.121005, "rms_a/round0": 1.42151, "rms_b/round0": 2.543458, "rms_c/round0": 1.317512, "cos_a2b/round1": 0.396555, "cos_c2b/round1": 0.283484, "cos_b2a/round1": 0.195228, "cos_b2c/round1": 0.19875, "rms_a/round1": 2.459107, "rms_b/round1": 4.229972, "rms_c/round1": 2.188586, "cos_a2b/round2": 0.460052, "cos_c2b/round2": 0.414228, "cos_b2a/round2": 0.081927, "cos_b2c/round2": 0.159843, "rms_a/round2": 4.414876, "rms_b/round2": 7.747181, "rms_c/round2": 3.293893}, "alpha_mean": 0.5066476915011577, "alpha_hist": [0.00932, 0.03717, 0.09239, 0.14554, 0.1774, 0.21062, 0.18141, 0.11412, 0.03015, 0.00188], "alpha_by_type": {"special": {"share": 0.00136, "mean": 0.58601, "std": 0.10088}, "whitespace": {"share": 0.02687, "mean": 0.58726, "std": 0.12148}, "digit": {"share": 0.02737, "mean": 0.63056, "std": 0.13086}, "word_start": {"share": 0.5853, "mean": 0.52243, "std": 0.1604}, "word_cont": {"share": 0.24765, "mean": 0.38873, "std": 0.16262}, "punct": {"share": 0.11145, "mean": 0.63496, "std": 0.14446}}, "alpha_p05": 0.20435, "alpha_p50": 0.52896, "alpha_p95": 0.78956}
{"step": 4310, "total_steps": 72479, "tokens": 2259681280, "loss": 3.914313167333603, "l_mix": 2.4025204181671143, "l_a": 2.543535739183426, "l_c": 2.490830034017563, "z_loss": 14.82986044883728, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.28181323409080505, "tokens_per_second": 1432504.4496101206, "eta_seconds": 24949.443390369415, "data_wait_seconds": 4.100799560546875e-05, "mix": {"fineweb_hq": 0.50951, "dclm": 0.20766, "cosmopedia_v2": 0.20046, "finemath": 0.08237}}
{"step": 4320, "total_steps": 72479, "tokens": 2264924160, "loss": 3.877196043729782, "l_mix": 2.379809558391571, "l_a": 2.5194320380687714, "l_c": 2.4666846990585327, "z_loss": 15.513875484466553, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.27089744806289673, "tokens_per_second": 1432029.342258752, "eta_seconds": 24954.059765028953, "data_wait_seconds": 2.193450927734375e-05, "mix": {"fineweb_hq": 0.50949, "dclm": 0.20764, "cosmopedia_v2": 0.20046, "finemath": 0.08241}}
{"step": 4330, "total_steps": 72479, "tokens": 2270167040, "loss": 3.5034185349941254, "l_mix": 2.1616911590099335, "l_a": 2.248449742794037, "l_c": 2.217667192220688, "z_loss": 18.92216682434082, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.353636234998703, "tokens_per_second": 1460872.2594332886, "eta_seconds": 24457.787244081497, "data_wait_seconds": 2.726912498474121e-05, "mix": {"fineweb_hq": 0.50947, "dclm": 0.20762, "cosmopedia_v2": 0.20046, "finemath": 0.08245}}
{"step": 4340, "total_steps": 72479, "tokens": 2275409920, "loss": 5.082935929298401, "l_mix": 3.1431699693202972, "l_a": 3.2489910423755646, "l_c": 3.211381644010544, "z_loss": 16.54264760017395, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2519507110118866, "tokens_per_second": 1417466.4280604075, "eta_seconds": 25203.037846112253, "data_wait_seconds": 1.7911195755004883e-05, "mix": {"fineweb_hq": 0.50968, "dclm": 0.2076, "cosmopedia_v2": 0.20046, "finemath": 0.08226}}
{"step": 4350, "total_steps": 72479, "tokens": 2280652800, "loss": 5.028476655483246, "l_mix": 3.1097083389759064, "l_a": 3.2116627395153046, "l_c": 3.17875012755394, "z_loss": 16.444539546966553, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18990546464920044, "tokens_per_second": 1466962.5747395852, "eta_seconds": 24349.09912977219, "data_wait_seconds": 2.5510787963867188e-05, "mix": {"fineweb_hq": 0.50966, "dclm": 0.20759, "cosmopedia_v2": 0.20046, "finemath": 0.0823}}
{"step": 4360, "total_steps": 72479, "tokens": 2285895680, "loss": 5.094328165054321, "l_mix": 3.150915652513504, "l_a": 3.2546900808811188, "l_c": 3.217973977327347, "z_loss": 16.132232308387756, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2159302681684494, "tokens_per_second": 1422487.0215131582, "eta_seconds": 25106.71361627579, "data_wait_seconds": 1.8328428268432617e-05, "mix": {"fineweb_hq": 0.50963, "dclm": 0.20757, "cosmopedia_v2": 0.20046, "finemath": 0.08234}}
{"step": 4370, "total_steps": 72479, "tokens": 2291138560, "loss": 4.991636872291565, "l_mix": 3.086583763360977, "l_a": 3.1860580146312714, "l_c": 3.158608853816986, "z_loss": 16.529665231704712, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1981324851512909, "tokens_per_second": 1346224.9933558775, "eta_seconds": 26525.084267663955, "data_wait_seconds": 3.4481287002563477e-05, "mix": {"fineweb_hq": 0.50961, "dclm": 0.20755, "cosmopedia_v2": 0.20046, "finemath": 0.08238}}
{"step": 4380, "total_steps": 72479, "tokens": 2296381440, "loss": 5.019151926040649, "l_mix": 3.103341221809387, "l_a": 3.2053670585155487, "l_c": 3.175062209367752, "z_loss": 16.818216562271118, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21137398481369019, "tokens_per_second": 1473699.6685128934, "eta_seconds": 24227.113077950475, "data_wait_seconds": 1.6242265701293945e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20753, "cosmopedia_v2": 0.20046, "finemath": 0.08242}}
{"step": 4390, "total_steps": 72479, "tokens": 2301624320, "loss": 5.226943254470825, "l_mix": 3.2368663549423218, "l_a": 3.325727790594101, "l_c": 3.30237939953804, "z_loss": 16.447185516357422, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18523265421390533, "tokens_per_second": 1464939.2616571025, "eta_seconds": 24368.41346693039, "data_wait_seconds": 1.704692840576172e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20774, "cosmopedia_v2": 0.20046, "finemath": 0.08223}}
{"step": 4400, "total_steps": 72479, "tokens": 2306867200, "loss": 4.7225449830293655, "l_mix": 2.922739624977112, "l_a": 3.0055590122938156, "l_c": 2.9883713126182556, "z_loss": 16.2604501247406, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22346536815166473, "tokens_per_second": 1457746.4853547083, "eta_seconds": 24485.054919075967, "data_wait_seconds": 1.633167266845703e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20773, "cosmopedia_v2": 0.20045, "finemath": 0.08227}, "gates": {"gate_a2b/L1_absmean": 0.10120808, "gate_a2b/L1_absmax": 0.58990908, "gate_a2b/L1_rms": 0.14648046, "gate_a2b/L3_absmean": 0.26257274, "gate_a2b/L3_absmax": 0.47563177, "gate_a2b/L3_rms": 0.27217552, "gate_a2b/L5_absmean": 0.41686031, "gate_a2b/L5_absmax": 0.63607645, "gate_a2b/L5_rms": 0.42836776, "gate_c2b/L1_absmean": 0.0150745, "gate_c2b/L1_absmax": 0.55564135, "gate_c2b/L1_rms": 0.05227702, "gate_c2b/L3_absmean": 0.25630069, "gate_c2b/L3_absmax": 0.56342232, "gate_c2b/L3_rms": 0.26611382, "gate_c2b/L5_absmean": 0.53353035, "gate_c2b/L5_absmax": 0.77896422, "gate_c2b/L5_rms": 0.54945838, "gate_b2a/L7_absmean": 0.05329271, "gate_b2a/L7_absmax": 0.25439182, "gate_b2a/L7_rms": 0.06258029, "gate_b2a/L11_absmean": 0.10369922, "gate_b2a/L11_absmax": 0.20605378, "gate_b2a/L11_rms": 0.10598104, "gate_b2a/L14_absmean": 0.25268957, "gate_b2a/L14_absmax": 0.39632699, "gate_b2a/L14_rms": 0.25367093, "gate_b2c/L3_absmean": 0.0316788, "gate_b2c/L3_absmax": 0.37528187, "gate_b2c/L3_rms": 0.05035619, "gate_b2c/L5_absmean": 0.07411017, "gate_b2c/L5_absmax": 0.37015638, "gate_b2c/L5_rms": 0.07748417, "gate_b2c/L6_absmean": 0.11824303, "gate_b2c/L6_absmax": 0.32532629, "gate_b2c/L6_rms": 0.11962801}, "streams": {"cos_a2b/round0": 0.359375, "cos_c2b/round0": 0.25058, "cos_b2a/round0": 0.185688, "cos_b2c/round0": 0.124537, "rms_a/round0": 1.400675, "rms_b/round0": 2.507223, "rms_c/round0": 1.304821, "cos_a2b/round1": 0.398799, "cos_c2b/round1": 0.288082, "cos_b2a/round1": 0.204851, "cos_b2c/round1": 0.199464, "rms_a/round1": 2.39795, "rms_b/round1": 4.171546, "rms_c/round1": 2.155349, "cos_a2b/round2": 0.444364, "cos_c2b/round2": 0.442221, "cos_b2a/round2": 0.086787, "cos_b2c/round2": 0.147907, "rms_a/round2": 4.304713, "rms_b/round2": 7.835701, "rms_c/round2": 3.293576}, "alpha_mean": 0.4949943091586846, "alpha_hist": [0.00583, 0.02361, 0.0889, 0.17502, 0.21124, 0.21848, 0.17183, 0.08864, 0.01585, 0.00061], "alpha_by_type": {"special": {"share": 0.00035, "mean": 0.63881, "std": 0.10384}, "whitespace": {"share": 0.05548, "mean": 0.56099, "std": 0.12186}, "digit": {"share": 0.02747, "mean": 0.50433, "std": 0.12944}, "word_start": {"share": 0.58115, "mean": 0.51088, "std": 0.1494}, "word_cont": {"share": 0.22984, "mean": 0.38855, "std": 0.14153}, "punct": {"share": 0.10571, "mean": 0.60158, "std": 0.14208}}, "alpha_p05": 0.22919, "alpha_p50": 0.48966, "alpha_p95": 0.74017}
{"step": 4410, "total_steps": 72479, "tokens": 2312110080, "loss": 4.575762510299683, "l_mix": 2.831722691655159, "l_a": 2.9145571179687977, "l_c": 2.893316574394703, "z_loss": 16.7755868434906, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17408739030361176, "tokens_per_second": 1327146.5194408614, "eta_seconds": 26890.595235133173, "data_wait_seconds": 3.0219554901123047e-05, "mix": {"fineweb_hq": 0.50952, "dclm": 0.20771, "cosmopedia_v2": 0.20045, "finemath": 0.08231}}
{"step": 4420, "total_steps": 72479, "tokens": 2317352960, "loss": 5.068797528743744, "l_mix": 3.137606233358383, "l_a": 3.227790594100952, "l_c": 3.2041416466236115, "z_loss": 16.11642062664032, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20596356689929962, "tokens_per_second": 1370939.9926472509, "eta_seconds": 26027.774507546426, "data_wait_seconds": 2.664327621459961e-05, "mix": {"fineweb_hq": 0.5095, "dclm": 0.20769, "cosmopedia_v2": 0.20045, "finemath": 0.08235}}
{"step": 4430, "total_steps": 72479, "tokens": 2322595840, "loss": 3.899411052465439, "l_mix": 2.3939016461372375, "l_a": 2.5341778993606567, "l_c": 2.4790020287036896, "z_loss": 15.553606390953064, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.24007298052310944, "tokens_per_second": 1473896.6254773105, "eta_seconds": 24206.089826989177, "data_wait_seconds": 2.2470951080322266e-05, "mix": {"fineweb_hq": 0.50948, "dclm": 0.20767, "cosmopedia_v2": 0.20045, "finemath": 0.08239}}
{"step": 4440, "total_steps": 72479, "tokens": 2327838720, "loss": 3.4235381484031677, "l_mix": 2.113326922059059, "l_a": 2.2002585530281067, "l_c": 2.1609209030866623, "z_loss": 18.572901487350464, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.28633013367652893, "tokens_per_second": 1406189.5445138693, "eta_seconds": 25367.868344044684, "data_wait_seconds": 2.625584602355957e-05, "mix": {"fineweb_hq": 0.50946, "dclm": 0.20766, "cosmopedia_v2": 0.20045, "finemath": 0.08243}}
{"step": 4450, "total_steps": 72479, "tokens": 2333081600, "loss": 4.9611018896102905, "l_mix": 3.0683973133563995, "l_a": 3.1700068712234497, "l_c": 3.1335355043411255, "z_loss": 16.417104959487915, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2331729531288147, "tokens_per_second": 1450793.2968331939, "eta_seconds": 24584.334949612617, "data_wait_seconds": 1.5228986740112305e-05, "mix": {"fineweb_hq": 0.50966, "dclm": 0.20764, "cosmopedia_v2": 0.20045, "finemath": 0.08225}}
{"step": 4460, "total_steps": 72479, "tokens": 2338324480, "loss": 5.096235990524292, "l_mix": 3.1514916718006134, "l_a": 3.2504720389842987, "l_c": 3.2264134287834167, "z_loss": 16.78579616546631, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.23654906451702118, "tokens_per_second": 1441379.635161111, "eta_seconds": 24741.258029508594, "data_wait_seconds": 1.7344951629638672e-05, "mix": {"fineweb_hq": 0.50964, "dclm": 0.20762, "cosmopedia_v2": 0.20045, "finemath": 0.08229}}
{"step": 4470, "total_steps": 72479, "tokens": 2343567360, "loss": 5.056493878364563, "l_mix": 3.1276009678840637, "l_a": 3.2309662103652954, "l_c": 3.193472772836685, "z_loss": 15.612926244735718, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.25217241048812866, "tokens_per_second": 1402985.0891198732, "eta_seconds": 25414.598393464086, "data_wait_seconds": 2.7120113372802734e-05, "mix": {"fineweb_hq": 0.50962, "dclm": 0.20761, "cosmopedia_v2": 0.20045, "finemath": 0.08233}}
{"step": 4480, "total_steps": 72479, "tokens": 2348810240, "loss": 5.017019987106323, "l_mix": 3.1016765236854553, "l_a": 3.2104839086532593, "l_c": 3.1684696674346924, "z_loss": 16.573899269104004, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2518146336078644, "tokens_per_second": 1347860.1922448783, "eta_seconds": 26450.116946196555, "data_wait_seconds": 1.946091651916504e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20759, "cosmopedia_v2": 0.20045, "finemath": 0.08237}}
{"step": 4490, "total_steps": 72479, "tokens": 2354053120, "loss": 5.077692091464996, "l_mix": 3.1400696635246277, "l_a": 3.241363614797592, "l_c": 3.2119726836681366, "z_loss": 16.21688437461853, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.25071683526039124, "tokens_per_second": 1464809.867526368, "eta_seconds": 24334.77383122444, "data_wait_seconds": 1.800060272216797e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20757, "cosmopedia_v2": 0.20045, "finemath": 0.08241}}
{"step": 4500, "total_steps": 72479, "tokens": 2359296000, "loss": 3.6810393929481506, "l_mix": 2.277003139257431, "l_a": 2.3522706627845764, "l_c": 2.3217256367206573, "z_loss": 18.373247385025024, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.39678651094436646, "tokens_per_second": 1442387.9391878885, "eta_seconds": 24709.423161196708, "data_wait_seconds": 3.975629806518555e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20756, "cosmopedia_v2": 0.20044, "finemath": 0.08244}, "gates": {"gate_a2b/L1_absmean": 0.10321295, "gate_a2b/L1_absmax": 0.59026694, "gate_a2b/L1_rms": 0.14866768, "gate_a2b/L3_absmean": 0.26315501, "gate_a2b/L3_absmax": 0.46868771, "gate_a2b/L3_rms": 0.27263343, "gate_a2b/L5_absmean": 0.41880652, "gate_a2b/L5_absmax": 0.63675731, "gate_a2b/L5_rms": 0.43019697, "gate_c2b/L1_absmean": 0.01431703, "gate_c2b/L1_absmax": 0.55520207, "gate_c2b/L1_rms": 0.05182222, "gate_c2b/L3_absmean": 0.25473154, "gate_c2b/L3_absmax": 0.55656755, "gate_c2b/L3_rms": 0.26443097, "gate_c2b/L5_absmean": 0.53515559, "gate_c2b/L5_absmax": 0.78004158, "gate_c2b/L5_rms": 0.55105752, "gate_b2a/L7_absmean": 0.05378129, "gate_b2a/L7_absmax": 0.25280997, "gate_b2a/L7_rms": 0.06288093, "gate_b2a/L11_absmean": 0.10453296, "gate_b2a/L11_absmax": 0.21396932, "gate_b2a/L11_rms": 0.1068247, "gate_b2a/L14_absmean": 0.25389165, "gate_b2a/L14_absmax": 0.40119585, "gate_b2a/L14_rms": 0.25488749, "gate_b2c/L3_absmean": 0.03223815, "gate_b2c/L3_absmax": 0.37940088, "gate_b2c/L3_rms": 0.05088172, "gate_b2c/L5_absmean": 0.07447396, "gate_b2c/L5_absmax": 0.37400013, "gate_b2c/L5_rms": 0.07793323, "gate_b2c/L6_absmean": 0.11957207, "gate_b2c/L6_absmax": 0.32505348, "gate_b2c/L6_rms": 0.12095582}, "streams": {"cos_a2b/round0": 0.393896, "cos_c2b/round0": 0.283333, "cos_b2a/round0": 0.194618, "cos_b2c/round0": 0.136242, "rms_a/round0": 1.426705, "rms_b/round0": 2.479865, "rms_c/round0": 1.273154, "cos_a2b/round1": 0.393215, "cos_c2b/round1": 0.257576, "cos_b2a/round1": 0.189476, "cos_b2c/round1": 0.189993, "rms_a/round1": 2.430184, "rms_b/round1": 4.201899, "rms_c/round1": 2.201316, "cos_a2b/round2": 0.411913, "cos_c2b/round2": 0.466641, "cos_b2a/round2": 0.097396, "cos_b2c/round2": 0.141443, "rms_a/round2": 4.366665, "rms_b/round2": 8.718239, "rms_c/round2": 3.441661}, "alpha_mean": 0.477319411003748, "alpha_hist": [0.00301, 0.02164, 0.09756, 0.20406, 0.24695, 0.20506, 0.13866, 0.0668, 0.0154, 0.00087], "alpha_by_type": {"special": {"share": 0.00066, "mean": 0.64497, "std": 0.12956}, "whitespace": {"share": 0.09473, "mean": 0.53602, "std": 0.12017}, "digit": {"share": 0.08762, "mean": 0.48349, "std": 0.12384}, "word_start": {"share": 0.4075, "mean": 0.47661, "std": 0.15885}, "word_cont": {"share": 0.21609, "mean": 0.39494, "std": 0.12775}, "punct": {"share": 0.19341, "mean": 0.53874, "std": 0.14247}}, "alpha_p05": 0.25128, "alpha_p50": 0.46005, "alpha_p95": 0.73602}
{"step": 4510, "total_steps": 72479, "tokens": 2364538880, "loss": 5.119603216648102, "l_mix": 3.1696780622005463, "l_a": 3.2614645063877106, "l_c": 3.232658952474594, "z_loss": 16.88043713569641, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2161960005760193, "tokens_per_second": 1467212.1619406121, "eta_seconds": 24287.783318853377, "data_wait_seconds": 2.8014183044433594e-05, "mix": {"fineweb_hq": 0.50953, "dclm": 0.20776, "cosmopedia_v2": 0.20044, "finemath": 0.08226}}
{"step": 4520, "total_steps": 72479, "tokens": 2369781760, "loss": 5.146549105644226, "l_mix": 3.1857344210147858, "l_a": 3.2750028669834137, "l_c": 3.2555340230464935, "z_loss": 16.53626847267151, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2101224958896637, "tokens_per_second": 1467524.1184951463, "eta_seconds": 24279.047780513763, "data_wait_seconds": 1.7434358596801758e-05, "mix": {"fineweb_hq": 0.50951, "dclm": 0.20774, "cosmopedia_v2": 0.20044, "finemath": 0.0823}}
{"step": 4530, "total_steps": 72479, "tokens": 2375024640, "loss": 5.135849237442017, "l_mix": 3.1778875589370728, "l_a": 3.273052752017975, "l_c": 3.248026520013809, "z_loss": 16.37772226333618, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1950366199016571, "tokens_per_second": 1464284.914170772, "eta_seconds": 24329.17594604492, "data_wait_seconds": 1.7404556274414062e-05, "mix": {"fineweb_hq": 0.50949, "dclm": 0.20773, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 4540, "total_steps": 72479, "tokens": 2380267520, "loss": 5.0523505210876465, "l_mix": 3.1267590820789337, "l_a": 3.21693953871727, "l_c": 3.1960604190826416, "z_loss": 16.914396047592163, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18736578524112701, "tokens_per_second": 1372638.6483341344, "eta_seconds": 25949.73008754253, "data_wait_seconds": 3.1620264053344727e-05, "mix": {"fineweb_hq": 0.50947, "dclm": 0.20771, "cosmopedia_v2": 0.20044, "finemath": 0.08238}}
{"step": 4550, "total_steps": 72479, "tokens": 2385510400, "loss": 5.1216002106666565, "l_mix": 3.17033115029335, "l_a": 3.2621657252311707, "l_c": 3.236633151769638, "z_loss": 16.29271388053894, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18169856071472168, "tokens_per_second": 1394454.957172803, "eta_seconds": 25539.985618615152, "data_wait_seconds": 4.151463508605957e-05, "mix": {"fineweb_hq": 0.50945, "dclm": 0.20769, "cosmopedia_v2": 0.20044, "finemath": 0.08242}}
{"step": 4560, "total_steps": 72479, "tokens": 2390753280, "loss": 5.031843900680542, "l_mix": 3.111948937177658, "l_a": 3.2137970328330994, "l_c": 3.18046572804451, "z_loss": 16.16038429737091, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16504298150539398, "tokens_per_second": 1382575.3312012516, "eta_seconds": 25755.6430151701, "data_wait_seconds": 1.8328428268432617e-05, "mix": {"fineweb_hq": 0.50965, "dclm": 0.20768, "cosmopedia_v2": 0.20044, "finemath": 0.08224}}
{"step": 4570, "total_steps": 72479, "tokens": 2395996160, "loss": 5.113113462924957, "l_mix": 3.1615926325321198, "l_a": 3.2649415731430054, "l_c": 3.2346960604190826, "z_loss": 16.295130968093872, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16994163393974304, "tokens_per_second": 1461672.7764376851, "eta_seconds": 24358.30670580864, "data_wait_seconds": 1.665949821472168e-05, "mix": {"fineweb_hq": 0.50963, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08228}}
{"step": 4580, "total_steps": 72479, "tokens": 2401239040, "loss": 4.994641721248627, "l_mix": 3.0885313153266907, "l_a": 3.1934120059013367, "l_c": 3.1548847258090973, "z_loss": 16.211785554885864, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.30549049377441406, "tokens_per_second": 1424239.427952738, "eta_seconds": 24994.83598988056, "data_wait_seconds": 1.6927719116210938e-05, "mix": {"fineweb_hq": 0.50961, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08231}}
{"step": 4590, "total_steps": 72479, "tokens": 2406481920, "loss": 5.108257412910461, "l_mix": 3.159060388803482, "l_a": 3.2612718641757965, "l_c": 3.2306120097637177, "z_loss": 16.318002343177795, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2647210657596588, "tokens_per_second": 1482891.2010064402, "eta_seconds": 24002.69689903259, "data_wait_seconds": 1.5616416931152344e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20763, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 4600, "total_steps": 72479, "tokens": 2411724800, "loss": 5.1831799149513245, "l_mix": 3.2066254019737244, "l_a": 3.3101710379123688, "l_c": 3.2729540169239044, "z_loss": 16.169375896453857, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.29380524158477783, "tokens_per_second": 1468688.614893013, "eta_seconds": 24231.23920967579, "data_wait_seconds": 1.7762184143066406e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20761, "cosmopedia_v2": 0.20043, "finemath": 0.08239}, "gates": {"gate_a2b/L1_absmean": 0.10573294, "gate_a2b/L1_absmax": 0.59284025, "gate_a2b/L1_rms": 0.15063602, "gate_a2b/L3_absmean": 0.2633675, "gate_a2b/L3_absmax": 0.4707832, "gate_a2b/L3_rms": 0.27255097, "gate_a2b/L5_absmean": 0.42115274, "gate_a2b/L5_absmax": 0.63549334, "gate_a2b/L5_rms": 0.43236363, "gate_c2b/L1_absmean": 0.01455404, "gate_c2b/L1_absmax": 0.5552156, "gate_c2b/L1_rms": 0.05162428, "gate_c2b/L3_absmean": 0.25367773, "gate_c2b/L3_absmax": 0.5514372, "gate_c2b/L3_rms": 0.26325256, "gate_c2b/L5_absmean": 0.536892, "gate_c2b/L5_absmax": 0.77963769, "gate_c2b/L5_rms": 0.55281103, "gate_b2a/L7_absmean": 0.05455748, "gate_b2a/L7_absmax": 0.25421169, "gate_b2a/L7_rms": 0.06360336, "gate_b2a/L11_absmean": 0.10547058, "gate_b2a/L11_absmax": 0.21702519, "gate_b2a/L11_rms": 0.10769459, "gate_b2a/L14_absmean": 0.25583291, "gate_b2a/L14_absmax": 0.40466294, "gate_b2a/L14_rms": 0.25684908, "gate_b2c/L3_absmean": 0.03289608, "gate_b2c/L3_absmax": 0.38012782, "gate_b2c/L3_rms": 0.05146305, "gate_b2c/L5_absmean": 0.07535154, "gate_b2c/L5_absmax": 0.37889323, "gate_b2c/L5_rms": 0.07881208, "gate_b2c/L6_absmean": 0.12001932, "gate_b2c/L6_absmax": 0.32837197, "gate_b2c/L6_rms": 0.121443}, "streams": {"cos_a2b/round0": 0.358588, "cos_c2b/round0": 0.247726, "cos_b2a/round0": 0.187386, "cos_b2c/round0": 0.119288, "rms_a/round0": 1.410126, "rms_b/round0": 2.501191, "rms_c/round0": 1.328169, "cos_a2b/round1": 0.379295, "cos_c2b/round1": 0.265305, "cos_b2a/round1": 0.19376, "cos_b2c/round1": 0.184079, "rms_a/round1": 2.42576, "rms_b/round1": 4.288011, "rms_c/round1": 2.194279, "cos_a2b/round2": 0.455889, "cos_c2b/round2": 0.423869, "cos_b2a/round2": 0.079967, "cos_b2c/round2": 0.138909, "rms_a/round2": 4.426981, "rms_b/round2": 7.902877, "rms_c/round2": 3.339}, "alpha_mean": 0.5265546709035931, "alpha_hist": [0.00813, 0.03049, 0.07137, 0.12668, 0.18553, 0.21333, 0.1908, 0.1333, 0.03888, 0.0015], "alpha_by_type": {"special": {"share": 0.00104, "mean": 0.60907, "std": 0.08151}, "whitespace": {"share": 0.036, "mean": 0.62112, "std": 0.11578}, "digit": {"share": 0.04802, "mean": 0.55546, "std": 0.14006}, "word_start": {"share": 0.53638, "mean": 0.5496, "std": 0.16024}, "word_cont": {"share": 0.24739, "mean": 0.41486, "std": 0.16714}, "punct": {"share": 0.13118, "mean": 0.60579, "std": 0.1541}}, "alpha_p05": 0.23336, "alpha_p50": 0.53643, "alpha_p95": 0.77764}
{"step": 4610, "total_steps": 72479, "tokens": 2416967680, "loss": 3.5975221991539, "l_mix": 2.2221633791923523, "l_a": 2.3058548867702484, "l_c": 2.272640824317932, "z_loss": 18.1005277633667, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.3007087707519531, "tokens_per_second": 1386749.8965636776, "eta_seconds": 25659.206725144388, "data_wait_seconds": 3.287196159362793e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20759, "cosmopedia_v2": 0.20043, "finemath": 0.08243}}
{"step": 4620, "total_steps": 72479, "tokens": 2422210560, "loss": 3.3440368473529816, "l_mix": 2.06400266289711, "l_a": 2.146107703447342, "l_c": 2.1145591139793396, "z_loss": 18.340934991836548, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.36656054854393005, "tokens_per_second": 1362052.7420092511, "eta_seconds": 26120.61801624298, "data_wait_seconds": 2.5510787963867188e-05, "mix": {"fineweb_hq": 0.50952, "dclm": 0.20758, "cosmopedia_v2": 0.20043, "finemath": 0.08247}}
{"step": 4630, "total_steps": 72479, "tokens": 2427453440, "loss": 5.146164178848267, "l_mix": 3.1851163804531097, "l_a": 3.2778753340244293, "l_c": 3.2533282935619354, "z_loss": 16.866278886795044, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18350736796855927, "tokens_per_second": 1366365.7228085224, "eta_seconds": 26034.330280828475, "data_wait_seconds": 2.6106834411621094e-05, "mix": {"fineweb_hq": 0.5095, "dclm": 0.20778, "cosmopedia_v2": 0.20043, "finemath": 0.08229}}
{"step": 4640, "total_steps": 72479, "tokens": 2432696320, "loss": 5.147611737251282, "l_mix": 3.185877948999405, "l_a": 3.280958414077759, "l_c": 3.2524121403694153, "z_loss": 17.22674536705017, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.221127450466156, "tokens_per_second": 1456936.3619391823, "eta_seconds": 24412.3041754961, "data_wait_seconds": 2.8967857360839844e-05, "mix": {"fineweb_hq": 0.50948, "dclm": 0.20776, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 4650, "total_steps": 72479, "tokens": 2437939200, "loss": 5.126808702945709, "l_mix": 3.173087954521179, "l_a": 3.2681988775730133, "l_c": 3.2388038337230682, "z_loss": 16.198005080223083, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19535893201828003, "tokens_per_second": 1449719.7835028907, "eta_seconds": 24530.210014843942, "data_wait_seconds": 2.8401613235473633e-05, "mix": {"fineweb_hq": 0.50946, "dclm": 0.20774, "cosmopedia_v2": 0.20043, "finemath": 0.08237}}
{"step": 4660, "total_steps": 72479, "tokens": 2443182080, "loss": 5.109325408935547, "l_mix": 3.1624996066093445, "l_a": 3.2559815645217896, "l_c": 3.227874279022217, "z_loss": 16.68985366821289, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2090800553560257, "tokens_per_second": 1420386.991971249, "eta_seconds": 25033.09877729416, "data_wait_seconds": 1.8537044525146484e-05, "mix": {"fineweb_hq": 0.50944, "dclm": 0.20773, "cosmopedia_v2": 0.20043, "finemath": 0.0824}}
{"step": 4670, "total_steps": 72479, "tokens": 2448424960, "loss": 5.032355964183807, "l_mix": 3.1137464940547943, "l_a": 3.211717873811722, "l_c": 3.178207904100418, "z_loss": 16.31707274913788, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1668059229850769, "tokens_per_second": 1393049.5201074341, "eta_seconds": 25520.589525961874, "data_wait_seconds": 1.800060272216797e-05, "mix": {"fineweb_hq": 0.50964, "dclm": 0.20771, "cosmopedia_v2": 0.20043, "finemath": 0.08223}}
{"step": 4680, "total_steps": 72479, "tokens": 2453667840, "loss": 5.010696589946747, "l_mix": 3.0985634326934814, "l_a": 3.203043222427368, "l_c": 3.165266901254654, "z_loss": 16.401668071746826, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18312035501003265, "tokens_per_second": 1393916.7400923613, "eta_seconds": 25500.95073084831, "data_wait_seconds": 3.203749656677246e-05, "mix": {"fineweb_hq": 0.50962, "dclm": 0.20769, "cosmopedia_v2": 0.20043, "finemath": 0.08226}}
{"step": 4690, "total_steps": 72479, "tokens": 2458910720, "loss": 4.928800404071808, "l_mix": 3.0469658970832825, "l_a": 3.150767534971237, "l_c": 3.116524338722229, "z_loss": 16.468472480773926, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18266956508159637, "tokens_per_second": 1370977.3434842357, "eta_seconds": 25923.81223578453, "data_wait_seconds": 1.800060272216797e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20768, "cosmopedia_v2": 0.20043, "finemath": 0.0823}}
{"step": 4700, "total_steps": 72479, "tokens": 2464153600, "loss": 5.060483038425446, "l_mix": 3.129997581243515, "l_a": 3.232138305902481, "l_c": 3.197362571954727, "z_loss": 16.350086569786072, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21484185755252838, "tokens_per_second": 1340627.5067469613, "eta_seconds": 26506.77848482132, "data_wait_seconds": 2.148747444152832e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.10695155, "gate_a2b/L1_absmax": 0.59446567, "gate_a2b/L1_rms": 0.15237096, "gate_a2b/L3_absmean": 0.26353303, "gate_a2b/L3_absmax": 0.46611166, "gate_a2b/L3_rms": 0.27259517, "gate_a2b/L5_absmean": 0.42308447, "gate_a2b/L5_absmax": 0.64037526, "gate_a2b/L5_rms": 0.43417022, "gate_c2b/L1_absmean": 0.01424855, "gate_c2b/L1_absmax": 0.56205052, "gate_c2b/L1_rms": 0.05174413, "gate_c2b/L3_absmean": 0.25269052, "gate_c2b/L3_absmax": 0.55214423, "gate_c2b/L3_rms": 0.26208779, "gate_c2b/L5_absmean": 0.53805667, "gate_c2b/L5_absmax": 0.77420402, "gate_c2b/L5_rms": 0.55395353, "gate_b2a/L7_absmean": 0.0548558, "gate_b2a/L7_absmax": 0.25434148, "gate_b2a/L7_rms": 0.06369845, "gate_b2a/L11_absmean": 0.10599954, "gate_b2a/L11_absmax": 0.22015266, "gate_b2a/L11_rms": 0.10816791, "gate_b2a/L14_absmean": 0.25742477, "gate_b2a/L14_absmax": 0.40052637, "gate_b2a/L14_rms": 0.25838608, "gate_b2c/L3_absmean": 0.03345506, "gate_b2c/L3_absmax": 0.38173872, "gate_b2c/L3_rms": 0.052241, "gate_b2c/L5_absmean": 0.07525679, "gate_b2c/L5_absmax": 0.38369104, "gate_b2c/L5_rms": 0.07875762, "gate_b2c/L6_absmean": 0.12080424, "gate_b2c/L6_absmax": 0.32632074, "gate_b2c/L6_rms": 0.1221917}, "streams": {"cos_a2b/round0": 0.357275, "cos_c2b/round0": 0.240559, "cos_b2a/round0": 0.18974, "cos_b2c/round0": 0.116742, "rms_a/round0": 1.402213, "rms_b/round0": 2.515577, "rms_c/round0": 1.334747, "cos_a2b/round1": 0.385912, "cos_c2b/round1": 0.270868, "cos_b2a/round1": 0.200337, "cos_b2c/round1": 0.188351, "rms_a/round1": 2.409728, "rms_b/round1": 4.258856, "rms_c/round1": 2.211066, "cos_a2b/round2": 0.463564, "cos_c2b/round2": 0.411269, "cos_b2a/round2": 0.077452, "cos_b2c/round2": 0.150076, "rms_a/round2": 4.385461, "rms_b/round2": 7.73855, "rms_c/round2": 3.33354}, "alpha_mean": 0.48678495148122636, "alpha_hist": [0.00859, 0.03703, 0.10335, 0.1739, 0.20024, 0.20293, 0.15607, 0.09441, 0.02238, 0.0011], "alpha_by_type": {"special": {"share": 0.00154, "mean": 0.56877, "std": 0.10221}, "whitespace": {"share": 0.02559, "mean": 0.55397, "std": 0.12282}, "digit": {"share": 0.03015, "mean": 0.5594, "std": 0.13869}, "word_start": {"share": 0.57907, "mean": 0.50436, "std": 0.15618}, "word_cont": {"share": 0.25946, "mean": 0.37801, "std": 0.1528}, "punct": {"share": 0.10419, "mean": 0.62127, "std": 0.15149}}, "alpha_p05": 0.20786, "alpha_p50": 0.48231, "alpha_p95": 0.78299}
{"step": 4710, "total_steps": 72479, "tokens": 2469396480, "loss": 4.979973793029785, "l_mix": 3.07987904548645, "l_a": 3.180612325668335, "l_c": 3.147636115550995, "z_loss": 16.201826095581055, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2435617744922638, "tokens_per_second": 1467559.1802844352, "eta_seconds": 24210.589902830125, "data_wait_seconds": 2.8342008590698242e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20764, "cosmopedia_v2": 0.20042, "finemath": 0.08238}}
{"step": 4720, "total_steps": 72479, "tokens": 2474639360, "loss": 3.50514155626297, "l_mix": 2.1655871123075485, "l_a": 2.246865764260292, "l_c": 2.2118823677301407, "z_loss": 19.299780130386353, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.29227498173713684, "tokens_per_second": 1467436.8630468806, "eta_seconds": 24209.035145974158, "data_wait_seconds": 3.2514333724975586e-05, "mix": {"fineweb_hq": 0.50953, "dclm": 0.20763, "cosmopedia_v2": 0.20042, "finemath": 0.08242}}
{"step": 4730, "total_steps": 72479, "tokens": 2479882240, "loss": 3.523367941379547, "l_mix": 2.176498517394066, "l_a": 2.2605535984039307, "l_c": 2.223134845495224, "z_loss": 17.628700971603394, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.3112010061740875, "tokens_per_second": 1473901.3673169347, "eta_seconds": 24099.297618985176, "data_wait_seconds": 2.41696834564209e-05, "mix": {"fineweb_hq": 0.50951, "dclm": 0.20761, "cosmopedia_v2": 0.20042, "finemath": 0.08245}}
{"step": 4740, "total_steps": 72479, "tokens": 2485125120, "loss": 5.045640647411346, "l_mix": 3.1210231482982635, "l_a": 3.2172513902187347, "l_c": 3.192619025707245, "z_loss": 16.562838315963745, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17568224668502808, "tokens_per_second": 1441926.2038969628, "eta_seconds": 24630.071036934853, "data_wait_seconds": 2.899765968322754e-05, "mix": {"fineweb_hq": 0.5097, "dclm": 0.20759, "cosmopedia_v2": 0.20042, "finemath": 0.08228}}
{"step": 4750, "total_steps": 72479, "tokens": 2490368000, "loss": 5.013160467147827, "l_mix": 3.1001080870628357, "l_a": 3.2035125494003296, "l_c": 3.1680977046489716, "z_loss": 15.692376494407654, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2090580016374588, "tokens_per_second": 1399564.1956899734, "eta_seconds": 25371.827931404114, "data_wait_seconds": 1.633167266845703e-05, "mix": {"fineweb_hq": 0.50968, "dclm": 0.20758, "cosmopedia_v2": 0.20042, "finemath": 0.08232}}
{"step": 4760, "total_steps": 72479, "tokens": 2495610880, "loss": 5.048200368881226, "l_mix": 3.121103435754776, "l_a": 3.2279871106147766, "l_c": 3.1902143359184265, "z_loss": 16.36404538154602, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20129325985908508, "tokens_per_second": 1440332.9272750912, "eta_seconds": 24650.036390662193, "data_wait_seconds": 1.665949821472168e-05, "mix": {"fineweb_hq": 0.50966, "dclm": 0.20756, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 4770, "total_steps": 72479, "tokens": 2500853760, "loss": 4.970437824726105, "l_mix": 3.0722516477108, "l_a": 3.1798212230205536, "l_c": 3.1420595943927765, "z_loss": 16.217331290245056, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18008512258529663, "tokens_per_second": 1348751.6209394203, "eta_seconds": 26319.9062309742, "data_wait_seconds": 2.9295682907104492e-05, "mix": {"fineweb_hq": 0.50964, "dclm": 0.20755, "cosmopedia_v2": 0.20042, "finemath": 0.08239}}
{"step": 4780, "total_steps": 72479, "tokens": 2506096640, "loss": 5.021695673465729, "l_mix": 3.105190187692642, "l_a": 3.2076941430568695, "l_c": 3.1752383708953857, "z_loss": 16.25739073753357, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.23178043961524963, "tokens_per_second": 1466180.106854663, "eta_seconds": 24208.330986118315, "data_wait_seconds": 2.1725893020629883e-05, "mix": {"fineweb_hq": 0.50962, "dclm": 0.20753, "cosmopedia_v2": 0.20042, "finemath": 0.08243}}
{"step": 4790, "total_steps": 72479, "tokens": 2511339520, "loss": 4.877016961574554, "l_mix": 3.016019344329834, "l_a": 3.119297683238983, "l_c": 3.0785505175590515, "z_loss": 16.43136978149414, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18063826858997345, "tokens_per_second": 1379469.7770413992, "eta_seconds": 25726.210912799837, "data_wait_seconds": 2.03549861907959e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20772, "cosmopedia_v2": 0.20042, "finemath": 0.08225}}
{"step": 4800, "total_steps": 72479, "tokens": 2516582400, "loss": 4.9241902232170105, "l_mix": 3.04428893327713, "l_a": 3.1475369334220886, "l_c": 3.1134126782417297, "z_loss": 16.16375160217285, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17895671725273132, "tokens_per_second": 1437703.1980104332, "eta_seconds": 24680.537402367594, "data_wait_seconds": 1.6361474990844727e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20771, "cosmopedia_v2": 0.20042, "finemath": 0.08229}, "gates": {"gate_a2b/L1_absmean": 0.10846456, "gate_a2b/L1_absmax": 0.59605426, "gate_a2b/L1_rms": 0.15409334, "gate_a2b/L3_absmean": 0.26352218, "gate_a2b/L3_absmax": 0.46311587, "gate_a2b/L3_rms": 0.2724621, "gate_a2b/L5_absmean": 0.42367211, "gate_a2b/L5_absmax": 0.63327706, "gate_a2b/L5_rms": 0.43469045, "gate_c2b/L1_absmean": 0.01440792, "gate_c2b/L1_absmax": 0.56307691, "gate_c2b/L1_rms": 0.05186716, "gate_c2b/L3_absmean": 0.25153968, "gate_c2b/L3_absmax": 0.54511935, "gate_c2b/L3_rms": 0.26083881, "gate_c2b/L5_absmean": 0.53845036, "gate_c2b/L5_absmax": 0.77866399, "gate_c2b/L5_rms": 0.55427957, "gate_b2a/L7_absmean": 0.05513614, "gate_b2a/L7_absmax": 0.25606817, "gate_b2a/L7_rms": 0.06391876, "gate_b2a/L11_absmean": 0.10621753, "gate_b2a/L11_absmax": 0.22669484, "gate_b2a/L11_rms": 0.10844415, "gate_b2a/L14_absmean": 0.25731921, "gate_b2a/L14_absmax": 0.40662581, "gate_b2a/L14_rms": 0.25832546, "gate_b2c/L3_absmean": 0.03346711, "gate_b2c/L3_absmax": 0.38560712, "gate_b2c/L3_rms": 0.05247282, "gate_b2c/L5_absmean": 0.07617071, "gate_b2c/L5_absmax": 0.39552423, "gate_b2c/L5_rms": 0.07977992, "gate_b2c/L6_absmean": 0.12123173, "gate_b2c/L6_absmax": 0.33238688, "gate_b2c/L6_rms": 0.12266336}, "streams": {"cos_a2b/round0": 0.358407, "cos_c2b/round0": 0.237814, "cos_b2a/round0": 0.191195, "cos_b2c/round0": 0.12126, "rms_a/round0": 1.395206, "rms_b/round0": 2.506546, "rms_c/round0": 1.33165, "cos_a2b/round1": 0.391866, "cos_c2b/round1": 0.268443, "cos_b2a/round1": 0.202267, "cos_b2c/round1": 0.19561, "rms_a/round1": 2.438571, "rms_b/round1": 4.264128, "rms_c/round1": 2.203663, "cos_a2b/round2": 0.466809, "cos_c2b/round2": 0.408707, "cos_b2a/round2": 0.074064, "cos_b2c/round2": 0.155143, "rms_a/round2": 4.448084, "rms_b/round2": 7.825398, "rms_c/round2": 3.354112}, "alpha_mean": 0.48133926070664046, "alpha_hist": [0.01137, 0.04401, 0.10942, 0.17053, 0.20265, 0.1931, 0.14857, 0.09294, 0.02657, 0.00084], "alpha_by_type": {"special": {"share": 0.00151, "mean": 0.60075, "std": 0.10102}, "whitespace": {"share": 0.02727, "mean": 0.55538, "std": 0.12208}, "digit": {"share": 0.03795, "mean": 0.52909, "std": 0.12755}, "word_start": {"share": 0.57893, "mean": 0.50569, "std": 0.15819}, "word_cont": {"share": 0.25595, "mean": 0.35662, "std": 0.15452}, "punct": {"share": 0.09839, "mean": 0.62172, "std": 0.16198}}, "alpha_p05": 0.18478, "alpha_p50": 0.48206, "alpha_p95": 0.76348}
{"step": 4810, "total_steps": 72479, "tokens": 2521825280, "loss": 4.9699108600616455, "l_mix": 3.0734916031360626, "l_a": 3.1767669916152954, "l_c": 3.139197736978531, "z_loss": 16.296311140060425, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20133471488952637, "tokens_per_second": 1122329.7131745517, "eta_seconds": 31611.071377277378, "data_wait_seconds": 1.952052116394043e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20769, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 4820, "total_steps": 72479, "tokens": 2527068160, "loss": 4.993420779705048, "l_mix": 3.08741095662117, "l_a": 3.1916075944900513, "l_c": 3.15620157122612, "z_loss": 16.671290397644043, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20675399899482727, "tokens_per_second": 1418842.3991513103, "eta_seconds": 25001.22762980461, "data_wait_seconds": 2.5063753128051758e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20768, "cosmopedia_v2": 0.20041, "finemath": 0.08237}}
{"step": 4830, "total_steps": 72479, "tokens": 2532311040, "loss": 4.983514189720154, "l_mix": 3.0824241042137146, "l_a": 3.1851228177547455, "l_c": 3.1464402973651886, "z_loss": 16.21099042892456, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2018444985151291, "tokens_per_second": 1451240.8080323052, "eta_seconds": 24439.471875166895, "data_wait_seconds": 1.856684684753418e-05, "mix": {"fineweb_hq": 0.50952, "dclm": 0.20766, "cosmopedia_v2": 0.20041, "finemath": 0.0824}}
{"step": 4840, "total_steps": 72479, "tokens": 2537553920, "loss": 3.4147322177886963, "l_mix": 2.1092449575662613, "l_a": 2.1926520615816116, "l_c": 2.152832120656967, "z_loss": 18.419228553771973, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2613900303840637, "tokens_per_second": 1357005.9492585112, "eta_seconds": 26132.763862514494, "data_wait_seconds": 2.7507543563842773e-05, "mix": {"fineweb_hq": 0.5095, "dclm": 0.20764, "cosmopedia_v2": 0.20041, "finemath": 0.08244}}
{"step": 4850, "total_steps": 72479, "tokens": 2542796800, "loss": 5.00495719909668, "l_mix": 3.094433009624481, "l_a": 3.201731860637665, "l_c": 3.1612090468406677, "z_loss": 16.41779088973999, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20774605870246887, "tokens_per_second": 1409928.094140397, "eta_seconds": 25148.142872929573, "data_wait_seconds": 1.621246337890625e-05, "mix": {"fineweb_hq": 0.50969, "dclm": 0.20763, "cosmopedia_v2": 0.20041, "finemath": 0.08227}}
{"step": 4860, "total_steps": 72479, "tokens": 2548039680, "loss": 5.034760296344757, "l_mix": 3.1139306724071503, "l_a": 3.220460057258606, "l_c": 3.1769488751888275, "z_loss": 16.068764686584473, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20379455387592316, "tokens_per_second": 1471083.4843772452, "eta_seconds": 24099.128736400606, "data_wait_seconds": 1.5914440155029297e-05, "mix": {"fineweb_hq": 0.50967, "dclm": 0.20761, "cosmopedia_v2": 0.20041, "finemath": 0.0823}}
{"step": 4870, "total_steps": 72479, "tokens": 2553282560, "loss": 5.071923792362213, "l_mix": 3.137841373682022, "l_a": 3.2373573184013367, "l_c": 3.2042458057403564, "z_loss": 16.016361713409424, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18225160241127014, "tokens_per_second": 1201078.8816200444, "eta_seconds": 29512.28927128315, "data_wait_seconds": 3.343820571899414e-05, "mix": {"fineweb_hq": 0.50965, "dclm": 0.2076, "cosmopedia_v2": 0.20041, "finemath": 0.08234}}
{"step": 4880, "total_steps": 72479, "tokens": 2558525440, "loss": 5.032746970653534, "l_mix": 3.1122672855854034, "l_a": 3.2171711325645447, "l_c": 3.1789607405662537, "z_loss": 16.400824308395386, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19112573564052582, "tokens_per_second": 1451176.1632363882, "eta_seconds": 24422.496323919295, "data_wait_seconds": 2.2709369659423828e-05, "mix": {"fineweb_hq": 0.50963, "dclm": 0.20758, "cosmopedia_v2": 0.20041, "finemath": 0.08238}}
{"step": 4890, "total_steps": 72479, "tokens": 2563768320, "loss": 5.009899199008942, "l_mix": 3.098695456981659, "l_a": 3.2002808451652527, "l_c": 3.1649338603019714, "z_loss": 16.392499923706055, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.23640252649784088, "tokens_per_second": 1444634.7172678076, "eta_seconds": 24529.454545450208, "data_wait_seconds": 3.0249357223510742e-05, "mix": {"fineweb_hq": 0.50961, "dclm": 0.20757, "cosmopedia_v2": 0.20041, "finemath": 0.08241}}
{"step": 4900, "total_steps": 72479, "tokens": 2569011200, "loss": 5.068064451217651, "l_mix": 3.134988844394684, "l_a": 3.236729472875595, "l_c": 3.2014295756816864, "z_loss": 16.278749704360962, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19489063322544098, "tokens_per_second": 1394238.259136634, "eta_seconds": 25412.34148454666, "data_wait_seconds": 1.990795135498047e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20755, "cosmopedia_v2": 0.20041, "finemath": 0.08245}, "gates": {"gate_a2b/L1_absmean": 0.11086241, "gate_a2b/L1_absmax": 0.59582204, "gate_a2b/L1_rms": 0.15646543, "gate_a2b/L3_absmean": 0.26441452, "gate_a2b/L3_absmax": 0.46352589, "gate_a2b/L3_rms": 0.27334359, "gate_a2b/L5_absmean": 0.42644054, "gate_a2b/L5_absmax": 0.6370517, "gate_a2b/L5_rms": 0.43734363, "gate_c2b/L1_absmean": 0.01428629, "gate_c2b/L1_absmax": 0.56548482, "gate_c2b/L1_rms": 0.05169364, "gate_c2b/L3_absmean": 0.25088033, "gate_c2b/L3_absmax": 0.54058355, "gate_c2b/L3_rms": 0.26005968, "gate_c2b/L5_absmean": 0.54011059, "gate_c2b/L5_absmax": 0.77961165, "gate_c2b/L5_rms": 0.55586582, "gate_b2a/L7_absmean": 0.05604666, "gate_b2a/L7_absmax": 0.25501764, "gate_b2a/L7_rms": 0.06446642, "gate_b2a/L11_absmean": 0.10769749, "gate_b2a/L11_absmax": 0.23127131, "gate_b2a/L11_rms": 0.10992488, "gate_b2a/L14_absmean": 0.25959486, "gate_b2a/L14_absmax": 0.40821677, "gate_b2a/L14_rms": 0.26057616, "gate_b2c/L3_absmean": 0.0345643, "gate_b2c/L3_absmax": 0.38757083, "gate_b2c/L3_rms": 0.05347564, "gate_b2c/L5_absmean": 0.07751115, "gate_b2c/L5_absmax": 0.40310615, "gate_b2c/L5_rms": 0.08109464, "gate_b2c/L6_absmean": 0.12322198, "gate_b2c/L6_absmax": 0.33404857, "gate_b2c/L6_rms": 0.12464284}, "streams": {"cos_a2b/round0": 0.350865, "cos_c2b/round0": 0.233154, "cos_b2a/round0": 0.18933, "cos_b2c/round0": 0.113977, "rms_a/round0": 1.37531, "rms_b/round0": 2.475613, "rms_c/round0": 1.326196, "cos_a2b/round1": 0.387753, "cos_c2b/round1": 0.274402, "cos_b2a/round1": 0.200349, "cos_b2c/round1": 0.188928, "rms_a/round1": 2.374032, "rms_b/round1": 4.217027, "rms_c/round1": 2.170131, "cos_a2b/round2": 0.465087, "cos_c2b/round2": 0.424658, "cos_b2a/round2": 0.08229, "cos_b2c/round2": 0.152171, "rms_a/round2": 4.339268, "rms_b/round2": 7.804284, "rms_c/round2": 3.307659}, "alpha_mean": 0.5009275382342935, "alpha_hist": [0.00734, 0.02997, 0.07832, 0.16315, 0.21556, 0.21858, 0.1635, 0.0948, 0.02718, 0.0016], "alpha_by_type": {"special": {"share": 0.00089, "mean": 0.58345, "std": 0.09912}, "whitespace": {"share": 0.02274, "mean": 0.54844, "std": 0.10886}, "digit": {"share": 0.02242, "mean": 0.52501, "std": 0.13788}, "word_start": {"share": 0.5612, "mean": 0.5282, "std": 0.14884}, "word_cont": {"share": 0.28325, "mean": 0.39793, "std": 0.15086}, "punct": {"share": 0.10951, "mean": 0.6121, "std": 0.1579}}, "alpha_p05": 0.21601, "alpha_p50": 0.50635, "alpha_p95": 0.77526}
{"step": 4910, "total_steps": 72479, "tokens": 2574254080, "loss": 5.090772032737732, "l_mix": 3.1507086753845215, "l_a": 3.2441856265068054, "l_c": 3.2172337770462036, "z_loss": 16.373786687850952, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16585278511047363, "tokens_per_second": 1388909.1914069685, "eta_seconds": 25506.070584869383, "data_wait_seconds": 2.5212764739990234e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20774, "cosmopedia_v2": 0.20041, "finemath": 0.08228}}
{"step": 4920, "total_steps": 72479, "tokens": 2579496960, "loss": 5.0968011021614075, "l_mix": 3.1552065014839172, "l_a": 3.2458913922309875, "l_c": 3.2207146883010864, "z_loss": 16.12725853919983, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.23574692010879517, "tokens_per_second": 1314519.3405666002, "eta_seconds": 26945.493990778923, "data_wait_seconds": 3.0368566513061523e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20772, "cosmopedia_v2": 0.20041, "finemath": 0.08232}}
{"step": 4930, "total_steps": 72479, "tokens": 2584739840, "loss": 5.098412752151489, "l_mix": 3.1553572714328766, "l_a": 3.2499627470970154, "l_c": 3.221348524093628, "z_loss": 16.619434118270874, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20940864086151123, "tokens_per_second": 1356886.0439742433, "eta_seconds": 26100.29800901413, "data_wait_seconds": 4.786252975463867e-05, "mix": {"fineweb_hq": 0.50953, "dclm": 0.20771, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 4940, "total_steps": 72479, "tokens": 2589982720, "loss": 5.285346627235413, "l_mix": 3.2729252576828003, "l_a": 3.3624053299427032, "l_c": 3.3403522670269012, "z_loss": 15.940410614013672, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21669608354568481, "tokens_per_second": 1431757.3679069674, "eta_seconds": 24731.76532959938, "data_wait_seconds": 2.440810203552246e-05, "mix": {"fineweb_hq": 0.50951, "dclm": 0.20769, "cosmopedia_v2": 0.2004, "finemath": 0.08239}}
{"step": 4950, "total_steps": 72479, "tokens": 2595225600, "loss": 3.5298414528369904, "l_mix": 2.17434099316597, "l_a": 2.3158114850521088, "l_c": 2.1966062486171722, "z_loss": 17.750956535339355, "lr": 0.0023, "wd": 0.1, "grad_norm": 1.6246657371520996, "tokens_per_second": 1353386.5821121852, "eta_seconds": 26160.037952160834, "data_wait_seconds": 3.1322240829467773e-05, "mix": {"fineweb_hq": 0.50949, "dclm": 0.20768, "cosmopedia_v2": 0.2004, "finemath": 0.08242}}
{"step": 4960, "total_steps": 72479, "tokens": 2600468480, "loss": 5.07830548286438, "l_mix": 3.140513002872467, "l_a": 3.2424961626529694, "l_c": 3.2114125192165375, "z_loss": 16.197792768478394, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16771456599235535, "tokens_per_second": 1382927.9044681746, "eta_seconds": 25597.430898332597, "data_wait_seconds": 0.02268114686012268, "mix": {"fineweb_hq": 0.50968, "dclm": 0.20766, "cosmopedia_v2": 0.2004, "finemath": 0.08226}}
{"step": 4970, "total_steps": 72479, "tokens": 2605711360, "loss": 5.074735999107361, "l_mix": 3.137740910053253, "l_a": 3.2426280975341797, "l_c": 3.208597868680954, "z_loss": 16.273556113243103, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22217994928359985, "tokens_per_second": 1232679.3319542867, "eta_seconds": 28713.192210245135, "data_wait_seconds": 1.5914440155029297e-05, "mix": {"fineweb_hq": 0.50966, "dclm": 0.20765, "cosmopedia_v2": 0.2004, "finemath": 0.08229}}
{"step": 4980, "total_steps": 72479, "tokens": 2610954240, "loss": 4.925274133682251, "l_mix": 3.041254937648773, "l_a": 3.1486458778381348, "l_c": 3.125893473625183, "z_loss": 16.57299542427063, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18454772233963013, "tokens_per_second": 1338543.262491523, "eta_seconds": 26438.380218005183, "data_wait_seconds": 2.434849739074707e-05, "mix": {"fineweb_hq": 0.50964, "dclm": 0.20763, "cosmopedia_v2": 0.2004, "finemath": 0.08233}}
{"step": 4990, "total_steps": 72479, "tokens": 2616197120, "loss": 5.006377577781677, "l_mix": 3.0961475670337677, "l_a": 3.2026342153549194, "l_c": 3.1594611406326294, "z_loss": 16.013018131256104, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18085716664791107, "tokens_per_second": 1403335.612140773, "eta_seconds": 25213.977701330186, "data_wait_seconds": 2.1785497665405273e-05, "mix": {"fineweb_hq": 0.50962, "dclm": 0.20762, "cosmopedia_v2": 0.2004, "finemath": 0.08236}}
{"step": 5000, "total_steps": 72479, "tokens": 2621440000, "loss": 4.948380470275879, "l_mix": 3.0603906214237213, "l_a": 3.1672704815864563, "l_c": 3.120684713125229, "z_loss": 16.032867312431335, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20245561003684998, "tokens_per_second": 1473302.4576569044, "eta_seconds": 24013.01224207878, "data_wait_seconds": 1.5169382095336914e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.2076, "cosmopedia_v2": 0.2004, "finemath": 0.0824}, "gates": {"gate_a2b/L1_absmean": 0.11279674, "gate_a2b/L1_absmax": 0.60782164, "gate_a2b/L1_rms": 0.15771808, "gate_a2b/L3_absmean": 0.26370499, "gate_a2b/L3_absmax": 0.46402645, "gate_a2b/L3_rms": 0.2726157, "gate_a2b/L5_absmean": 0.42581135, "gate_a2b/L5_absmax": 0.62978888, "gate_a2b/L5_rms": 0.43653667, "gate_c2b/L1_absmean": 0.01478594, "gate_c2b/L1_absmax": 0.57921451, "gate_c2b/L1_rms": 0.0524095, "gate_c2b/L3_absmean": 0.25102934, "gate_c2b/L3_absmax": 0.54644817, "gate_c2b/L3_rms": 0.26032031, "gate_c2b/L5_absmean": 0.54119247, "gate_c2b/L5_absmax": 0.77093655, "gate_c2b/L5_rms": 0.55707222, "gate_b2a/L7_absmean": 0.05703822, "gate_b2a/L7_absmax": 0.25410846, "gate_b2a/L7_rms": 0.06528492, "gate_b2a/L11_absmean": 0.10873943, "gate_b2a/L11_absmax": 0.22588788, "gate_b2a/L11_rms": 0.11098062, "gate_b2a/L14_absmean": 0.26166949, "gate_b2a/L14_absmax": 0.41343021, "gate_b2a/L14_rms": 0.26268199, "gate_b2c/L3_absmean": 0.03464257, "gate_b2c/L3_absmax": 0.39119664, "gate_b2c/L3_rms": 0.05357244, "gate_b2c/L5_absmean": 0.07645825, "gate_b2c/L5_absmax": 0.40632713, "gate_b2c/L5_rms": 0.08019634, "gate_b2c/L6_absmean": 0.12227327, "gate_b2c/L6_absmax": 0.33458221, "gate_b2c/L6_rms": 0.12374863}, "streams": {"cos_a2b/round0": 0.357961, "cos_c2b/round0": 0.228376, "cos_b2a/round0": 0.194445, "cos_b2c/round0": 0.113725, "rms_a/round0": 1.368104, "rms_b/round0": 2.490008, "rms_c/round0": 1.338569, "cos_a2b/round1": 0.388385, "cos_c2b/round1": 0.265578, "cos_b2a/round1": 0.200876, "cos_b2c/round1": 0.192777, "rms_a/round1": 2.380427, "rms_b/round1": 4.255219, "rms_c/round1": 2.19458, "cos_a2b/round2": 0.459865, "cos_c2b/round2": 0.421824, "cos_b2a/round2": 0.081599, "cos_b2c/round2": 0.150166, "rms_a/round2": 4.416722, "rms_b/round2": 8.009358, "rms_c/round2": 3.370422}, "alpha_mean": 0.47002159774602603, "alpha_hist": [0.01241, 0.04973, 0.11746, 0.17899, 0.2124, 0.18423, 0.13452, 0.08475, 0.0248, 0.00072], "alpha_by_type": {"special": {"share": 0.0011, "mean": 0.56009, "std": 0.10242}, "whitespace": {"share": 0.02301, "mean": 0.54749, "std": 0.11209}, "digit": {"share": 0.02782, "mean": 0.54293, "std": 0.1223}, "word_start": {"share": 0.55945, "mean": 0.50036, "std": 0.16002}, "word_cont": {"share": 0.2632, "mean": 0.3436, "std": 0.1496}, "punct": {"share": 0.12543, "mean": 0.56883, "std": 0.16711}}, "alpha_p05": 0.1747, "alpha_p50": 0.45762, "alpha_p95": 0.75238}
{"step": 5010, "total_steps": 72479, "tokens": 2626682880, "loss": 4.960787534713745, "l_mix": 3.068155735731125, "l_a": 3.170900344848633, "l_c": 3.132372349500656, "z_loss": 16.49901819229126, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2195725440979004, "tokens_per_second": 1467985.0503105, "eta_seconds": 24096.421870589256, "data_wait_seconds": 2.232193946838379e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20758, "cosmopedia_v2": 0.2004, "finemath": 0.08244}}
{"step": 5020, "total_steps": 72479, "tokens": 2631925760, "loss": 5.115553438663483, "l_mix": 3.16697895526886, "l_a": 3.2585619390010834, "l_c": 3.2313980162143707, "z_loss": 15.863423824310303, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18814823031425476, "tokens_per_second": 1374425.6133923517, "eta_seconds": 25732.890778064724, "data_wait_seconds": 2.4825334548950195e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20777, "cosmopedia_v2": 0.2004, "finemath": 0.08227}}
{"step": 5030, "total_steps": 72479, "tokens": 2637168640, "loss": 5.098487675189972, "l_mix": 3.1553524136543274, "l_a": 3.2541574835777283, "l_c": 3.2173995077610016, "z_loss": 16.68056559562683, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22319161891937256, "tokens_per_second": 1254709.5361321142, "eta_seconds": 28183.974293375017, "data_wait_seconds": 2.014636993408203e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20775, "cosmopedia_v2": 0.2004, "finemath": 0.08231}}
{"step": 5040, "total_steps": 72479, "tokens": 2642411520, "loss": 5.175404608249664, "l_mix": 3.203665792942047, "l_a": 3.2954616844654083, "l_c": 3.2717102468013763, "z_loss": 15.870907425880432, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21820692718029022, "tokens_per_second": 1427575.372002797, "eta_seconds": 24767.48977701664, "data_wait_seconds": 1.710653305053711e-05, "mix": {"fineweb_hq": 0.50952, "dclm": 0.20774, "cosmopedia_v2": 0.2004, "finemath": 0.08234}}
{"step": 5050, "total_steps": 72479, "tokens": 2647654400, "loss": 5.433340609073639, "l_mix": 3.3655820786952972, "l_a": 3.459650218486786, "l_c": 3.4275991022586823, "z_loss": 15.83612072467804, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22342067956924438, "tokens_per_second": 1168555.1850338944, "eta_seconds": 30252.927722001077, "data_wait_seconds": 4.214048385620117e-05, "mix": {"fineweb_hq": 0.5095, "dclm": 0.20772, "cosmopedia_v2": 0.2004, "finemath": 0.08238}}
{"step": 5060, "total_steps": 72479, "tokens": 2652897280, "loss": 5.277560889720917, "l_mix": 3.2661966383457184, "l_a": 3.3614442348480225, "l_c": 3.3377252519130707, "z_loss": 16.133049249649048, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.23944249749183655, "tokens_per_second": 1345795.0076319051, "eta_seconds": 26264.75241143703, "data_wait_seconds": 3.1620264053344727e-05, "mix": {"fineweb_hq": 0.50949, "dclm": 0.20771, "cosmopedia_v2": 0.2004, "finemath": 0.08241}}
{"step": 5070, "total_steps": 72479, "tokens": 2658140160, "loss": 4.828400671482086, "l_mix": 2.985613763332367, "l_a": 3.0911688804626465, "l_c": 3.046061009168625, "z_loss": 16.179307222366333, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18251876533031464, "tokens_per_second": 1470820.9699744338, "eta_seconds": 24028.57350654602, "data_wait_seconds": 1.6689300537109375e-05, "mix": {"fineweb_hq": 0.50966, "dclm": 0.20769, "cosmopedia_v2": 0.20039, "finemath": 0.08225}}
{"step": 5080, "total_steps": 72479, "tokens": 2663383040, "loss": 5.034051239490509, "l_mix": 3.1134766340255737, "l_a": 3.219350904226303, "l_c": 3.17717307806015, "z_loss": 16.17278552055359, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1951257884502411, "tokens_per_second": 1466335.1647672525, "eta_seconds": 24098.50609946251, "data_wait_seconds": 1.5556812286376953e-05, "mix": {"fineweb_hq": 0.50965, "dclm": 0.20768, "cosmopedia_v2": 0.20039, "finemath": 0.08228}}
{"step": 5090, "total_steps": 72479, "tokens": 2668625920, "loss": 4.92135214805603, "l_mix": 3.0429558157920837, "l_a": 3.15237757563591, "l_c": 3.1034480035305023, "z_loss": 16.48655676841736, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20907746255397797, "tokens_per_second": 1419566.9879424586, "eta_seconds": 24888.74729554653, "data_wait_seconds": 1.9878149032592773e-05, "mix": {"fineweb_hq": 0.50963, "dclm": 0.20766, "cosmopedia_v2": 0.20039, "finemath": 0.08232}}
{"step": 5100, "total_steps": 72479, "tokens": 2673868800, "loss": 4.977375090122223, "l_mix": 3.077986925840378, "l_a": 3.1848041713237762, "l_c": 3.1410982608795166, "z_loss": 16.17458200454712, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1768762469291687, "tokens_per_second": 1456619.2414068156, "eta_seconds": 24252.04895541668, "data_wait_seconds": 1.7255544662475586e-05, "mix": {"fineweb_hq": 0.50961, "dclm": 0.20765, "cosmopedia_v2": 0.20039, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.11431416, "gate_a2b/L1_absmax": 0.60738462, "gate_a2b/L1_rms": 0.15954976, "gate_a2b/L3_absmean": 0.26279211, "gate_a2b/L3_absmax": 0.45779032, "gate_a2b/L3_rms": 0.27154315, "gate_a2b/L5_absmean": 0.4259761, "gate_a2b/L5_absmax": 0.62830371, "gate_a2b/L5_rms": 0.43666595, "gate_c2b/L1_absmean": 0.01490555, "gate_c2b/L1_absmax": 0.57726085, "gate_c2b/L1_rms": 0.05223598, "gate_c2b/L3_absmean": 0.24976416, "gate_c2b/L3_absmax": 0.53961611, "gate_c2b/L3_rms": 0.25884569, "gate_c2b/L5_absmean": 0.54350066, "gate_c2b/L5_absmax": 0.77707261, "gate_c2b/L5_rms": 0.55942357, "gate_b2a/L7_absmean": 0.05674601, "gate_b2a/L7_absmax": 0.25532937, "gate_b2a/L7_rms": 0.06522778, "gate_b2a/L11_absmean": 0.10854292, "gate_b2a/L11_absmax": 0.23119444, "gate_b2a/L11_rms": 0.11071189, "gate_b2a/L14_absmean": 0.26363039, "gate_b2a/L14_absmax": 0.41495171, "gate_b2a/L14_rms": 0.2646378, "gate_b2c/L3_absmean": 0.03467371, "gate_b2c/L3_absmax": 0.39249116, "gate_b2c/L3_rms": 0.05389453, "gate_b2c/L5_absmean": 0.07716184, "gate_b2c/L5_absmax": 0.40806732, "gate_b2c/L5_rms": 0.0808387, "gate_b2c/L6_absmean": 0.12283465, "gate_b2c/L6_absmax": 0.33651218, "gate_b2c/L6_rms": 0.12430244}, "streams": {"cos_a2b/round0": 0.359904, "cos_c2b/round0": 0.235824, "cos_b2a/round0": 0.192181, "cos_b2c/round0": 0.11493, "rms_a/round0": 1.368727, "rms_b/round0": 2.446294, "rms_c/round0": 1.34553, "cos_a2b/round1": 0.396891, "cos_c2b/round1": 0.27826, "cos_b2a/round1": 0.206171, "cos_b2c/round1": 0.182018, "rms_a/round1": 2.398766, "rms_b/round1": 4.193364, "rms_c/round1": 2.208785, "cos_a2b/round2": 0.453298, "cos_c2b/round2": 0.438494, "cos_b2a/round2": 0.078645, "cos_b2c/round2": 0.145664, "rms_a/round2": 4.407037, "rms_b/round2": 8.075343, "rms_c/round2": 3.396577}, "alpha_mean": 0.48779511801603803, "alpha_hist": [0.00743, 0.0379, 0.10661, 0.17467, 0.20036, 0.19061, 0.15541, 0.09959, 0.02678, 0.00063], "alpha_by_type": {"special": {"share": 0.00136, "mean": 0.59739, "std": 0.08911}, "whitespace": {"share": 0.02663, "mean": 0.56098, "std": 0.12067}, "digit": {"share": 0.02736, "mean": 0.51243, "std": 0.13393}, "word_start": {"share": 0.58734, "mean": 0.50794, "std": 0.16297}, "word_cont": {"share": 0.24887, "mean": 0.37531, "std": 0.14622}, "punct": {"share": 0.10844, "mean": 0.61129, "std": 0.15963}}, "alpha_p05": 0.19621, "alpha_p50": 0.48999, "alpha_p95": 0.76907}
{"step": 5110, "total_steps": 72479, "tokens": 2679111680, "loss": 5.027117431163788, "l_mix": 3.1097581684589386, "l_a": 3.2140729129314423, "l_c": 3.171757459640503, "z_loss": 16.099403381347656, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17404477298259735, "tokens_per_second": 1433520.181522544, "eta_seconds": 24639.1775485754, "data_wait_seconds": 1.7017126083374023e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20763, "cosmopedia_v2": 0.20039, "finemath": 0.08239}}
{"step": 5120, "total_steps": 72479, "tokens": 2684354560, "loss": 4.950729250907898, "l_mix": 3.061189830303192, "l_a": 3.165048062801361, "l_c": 3.1279261112213135, "z_loss": 16.470251083374023, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2096102237701416, "tokens_per_second": 1406928.6283032403, "eta_seconds": 25101.14207754135, "data_wait_seconds": 2.41696834564209e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20762, "cosmopedia_v2": 0.20039, "finemath": 0.08242}}
{"step": 5130, "total_steps": 72479, "tokens": 2689597440, "loss": 3.640519440174103, "l_mix": 2.2495167553424835, "l_a": 2.3312799632549286, "l_c": 2.299351543188095, "z_loss": 18.131494998931885, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.308331161737442, "tokens_per_second": 1425864.8602738539, "eta_seconds": 24764.108784627915, "data_wait_seconds": 2.625584602355957e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.2076, "cosmopedia_v2": 0.20039, "finemath": 0.08246}}
{"step": 5140, "total_steps": 72479, "tokens": 2694840320, "loss": 5.150492489337921, "l_mix": 3.1881408095359802, "l_a": 3.2845785915851593, "l_c": 3.251023143529892, "z_loss": 16.711584329605103, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22266539931297302, "tokens_per_second": 1464553.6838510255, "eta_seconds": 24106.340396595002, "data_wait_seconds": 2.3543834686279297e-05, "mix": {"fineweb_hq": 0.50953, "dclm": 0.20778, "cosmopedia_v2": 0.20039, "finemath": 0.0823}}
{"step": 5150, "total_steps": 72479, "tokens": 2700083200, "loss": 5.133489191532135, "l_mix": 3.1773060858249664, "l_a": 3.272120952606201, "l_c": 3.243044435977936, "z_loss": 16.333844900131226, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19598259031772614, "tokens_per_second": 1472306.8672526863, "eta_seconds": 23975.83515851498, "data_wait_seconds": 2.014636993408203e-05, "mix": {"fineweb_hq": 0.50951, "dclm": 0.20777, "cosmopedia_v2": 0.20039, "finemath": 0.08233}}
{"step": 5160, "total_steps": 72479, "tokens": 2705326080, "loss": 5.15442419052124, "l_mix": 3.191536545753479, "l_a": 3.2828378081321716, "l_c": 3.2546599209308624, "z_loss": 16.381951928138733, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19116155803203583, "tokens_per_second": 1393081.2017444056, "eta_seconds": 25335.596968650818, "data_wait_seconds": 2.238154411315918e-05, "mix": {"fineweb_hq": 0.5095, "dclm": 0.20775, "cosmopedia_v2": 0.20039, "finemath": 0.08236}}
{"step": 5170, "total_steps": 72479, "tokens": 2710568960, "loss": 5.128995835781097, "l_mix": 3.174994170665741, "l_a": 3.266805410385132, "l_c": 3.2411599159240723, "z_loss": 16.11961317062378, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20898127555847168, "tokens_per_second": 1312943.9232307894, "eta_seconds": 26877.995600271228, "data_wait_seconds": 3.781914710998535e-05, "mix": {"fineweb_hq": 0.50948, "dclm": 0.20774, "cosmopedia_v2": 0.20039, "finemath": 0.0824}}
{"step": 5180, "total_steps": 72479, "tokens": 2715811840, "loss": 5.0752551555633545, "l_mix": 3.13960599899292, "l_a": 3.2437773644924164, "l_c": 3.203037828207016, "z_loss": 16.04624044895172, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19531196355819702, "tokens_per_second": 1261168.2352256335, "eta_seconds": 27977.281005406378, "data_wait_seconds": 1.767277717590332e-05, "mix": {"fineweb_hq": 0.50965, "dclm": 0.20772, "cosmopedia_v2": 0.20039, "finemath": 0.08224}}
{"step": 5190, "total_steps": 72479, "tokens": 2721054720, "loss": 5.061066210269928, "l_mix": 3.1303211450576782, "l_a": 3.235773265361786, "l_c": 3.1946495473384857, "z_loss": 16.18091356754303, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19022060930728912, "tokens_per_second": 1240527.6815843384, "eta_seconds": 28438.55542743206, "data_wait_seconds": 2.1636486053466797e-05, "mix": {"fineweb_hq": 0.50963, "dclm": 0.20771, "cosmopedia_v2": 0.20039, "finemath": 0.08227}}
{"step": 5200, "total_steps": 72479, "tokens": 2726297600, "loss": 4.877596616744995, "l_mix": 3.0159508883953094, "l_a": 3.121392399072647, "l_c": 3.078675150871277, "z_loss": 16.252899527549744, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16378504037857056, "tokens_per_second": 1412585.9470138769, "eta_seconds": 24970.921186470987, "data_wait_seconds": 1.7940998077392578e-05, "mix": {"fineweb_hq": 0.50962, "dclm": 0.20769, "cosmopedia_v2": 0.20038, "finemath": 0.08231}, "gates": {"gate_a2b/L1_absmean": 0.1151737, "gate_a2b/L1_absmax": 0.60158324, "gate_a2b/L1_rms": 0.16038965, "gate_a2b/L3_absmean": 0.2630738, "gate_a2b/L3_absmax": 0.45812222, "gate_a2b/L3_rms": 0.27167553, "gate_a2b/L5_absmean": 0.42761451, "gate_a2b/L5_absmax": 0.62708068, "gate_a2b/L5_rms": 0.43818814, "gate_c2b/L1_absmean": 0.01395212, "gate_c2b/L1_absmax": 0.5713715, "gate_c2b/L1_rms": 0.05180651, "gate_c2b/L3_absmean": 0.24889068, "gate_c2b/L3_absmax": 0.53698283, "gate_c2b/L3_rms": 0.25782818, "gate_c2b/L5_absmean": 0.54451609, "gate_c2b/L5_absmax": 0.78424948, "gate_c2b/L5_rms": 0.56030887, "gate_b2a/L7_absmean": 0.05775932, "gate_b2a/L7_absmax": 0.25796506, "gate_b2a/L7_rms": 0.06600825, "gate_b2a/L11_absmean": 0.1093217, "gate_b2a/L11_absmax": 0.23629297, "gate_b2a/L11_rms": 0.11147771, "gate_b2a/L14_absmean": 0.26419443, "gate_b2a/L14_absmax": 0.41815734, "gate_b2a/L14_rms": 0.2652044, "gate_b2c/L3_absmean": 0.03524151, "gate_b2c/L3_absmax": 0.39583096, "gate_b2c/L3_rms": 0.05444991, "gate_b2c/L5_absmean": 0.07747009, "gate_b2c/L5_absmax": 0.41527471, "gate_b2c/L5_rms": 0.08126002, "gate_b2c/L6_absmean": 0.1234903, "gate_b2c/L6_absmax": 0.34153262, "gate_b2c/L6_rms": 0.12498544}, "streams": {"cos_a2b/round0": 0.36028, "cos_c2b/round0": 0.235828, "cos_b2a/round0": 0.192997, "cos_b2c/round0": 0.114979, "rms_a/round0": 1.36507, "rms_b/round0": 2.442641, "rms_c/round0": 1.338558, "cos_a2b/round1": 0.393639, "cos_c2b/round1": 0.274004, "cos_b2a/round1": 0.205244, "cos_b2c/round1": 0.188635, "rms_a/round1": 2.394517, "rms_b/round1": 4.216248, "rms_c/round1": 2.209837, "cos_a2b/round2": 0.462042, "cos_c2b/round2": 0.43166, "cos_b2a/round2": 0.071725, "cos_b2c/round2": 0.145645, "rms_a/round2": 4.42989, "rms_b/round2": 8.019641, "rms_c/round2": 3.429251}, "alpha_mean": 0.4841301151167414, "alpha_hist": [0.01071, 0.05424, 0.11514, 0.16119, 0.18777, 0.18105, 0.14954, 0.10742, 0.03197, 0.00096], "alpha_by_type": {"special": {"share": 0.00128, "mean": 0.61163, "std": 0.11017}, "whitespace": {"share": 0.02893, "mean": 0.60095, "std": 0.11815}, "digit": {"share": 0.03288, "mean": 0.62459, "std": 0.12552}, "word_start": {"share": 0.5883, "mean": 0.48843, "std": 0.17286}, "word_cont": {"share": 0.24387, "mean": 0.37553, "std": 0.15807}, "punct": {"share": 0.10474, "mean": 0.6349, "std": 0.1612}}, "alpha_p05": 0.17583, "alpha_p50": 0.46709, "alpha_p95": 0.77594}
{"step": 5210, "total_steps": 72479, "tokens": 2731540480, "loss": 4.902662992477417, "l_mix": 3.031137228012085, "l_a": 3.139796167612076, "l_c": 3.093301147222519, "z_loss": 15.964714288711548, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19173762202262878, "tokens_per_second": 1449303.7751110315, "eta_seconds": 24334.670258688926, "data_wait_seconds": 1.671910285949707e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20768, "cosmopedia_v2": 0.20038, "finemath": 0.08234}}
{"step": 5220, "total_steps": 72479, "tokens": 2736783360, "loss": 4.874728262424469, "l_mix": 3.014066606760025, "l_a": 3.121399074792862, "l_c": 3.075384795665741, "z_loss": 16.264171481132507, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19376413524150848, "tokens_per_second": 1464739.8129735542, "eta_seconds": 24074.641980552675, "data_wait_seconds": 1.7851591110229492e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20038, "finemath": 0.08238}}
{"step": 5230, "total_steps": 72479, "tokens": 2742026240, "loss": 4.934805929660797, "l_mix": 3.0507218837738037, "l_a": 3.159846007823944, "l_c": 3.1150298416614532, "z_loss": 16.21293544769287, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20540830492973328, "tokens_per_second": 1461274.3503624373, "eta_seconds": 24128.14794378281, "data_wait_seconds": 1.874566078186035e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20038, "finemath": 0.08241}}
{"step": 5240, "total_steps": 72479, "tokens": 2747269120, "loss": 3.426772266626358, "l_mix": 2.1140070855617523, "l_a": 2.1943628787994385, "l_c": 2.1753848791122437, "z_loss": 18.4079110622406, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.26359671354293823, "tokens_per_second": 1427044.9978104597, "eta_seconds": 24703.216006565097, "data_wait_seconds": 3.3974647521972656e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20763, "cosmopedia_v2": 0.20038, "finemath": 0.08244}}
{"step": 5250, "total_steps": 72479, "tokens": 2752512000, "loss": 3.417265474796295, "l_mix": 2.1094759106636047, "l_a": 2.2063536942005157, "l_c": 2.146735653281212, "z_loss": 18.627126216888428, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.300226628780365, "tokens_per_second": 1369918.6340180528, "eta_seconds": 25729.526613283157, "data_wait_seconds": 1.9043684005737305e-05, "mix": {"fineweb_hq": 0.50952, "dclm": 0.20762, "cosmopedia_v2": 0.20038, "finemath": 0.08248}}
{"step": 5260, "total_steps": 72479, "tokens": 2757754880, "loss": 4.920335948467255, "l_mix": 3.0426188707351685, "l_a": 3.146191358566284, "l_c": 3.1075998842716217, "z_loss": 15.795187711715698, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19728679955005646, "tokens_per_second": 1414160.4098513783, "eta_seconds": 24920.875189614297, "data_wait_seconds": 3.1381845474243164e-05, "mix": {"fineweb_hq": 0.5097, "dclm": 0.2076, "cosmopedia_v2": 0.20038, "finemath": 0.08232}}
{"step": 5270, "total_steps": 72479, "tokens": 2762997760, "loss": 4.967726945877075, "l_mix": 3.073131740093231, "l_a": 3.175107002258301, "l_c": 3.134858548641205, "z_loss": 16.054940462112427, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18266728520393372, "tokens_per_second": 1393407.897853826, "eta_seconds": 25288.267883563043, "data_wait_seconds": 2.199411392211914e-05, "mix": {"fineweb_hq": 0.50968, "dclm": 0.20759, "cosmopedia_v2": 0.20038, "finemath": 0.08235}}
{"step": 5280, "total_steps": 72479, "tokens": 2768240640, "loss": 4.963239848613739, "l_mix": 3.0695070326328278, "l_a": 3.1731068193912506, "l_c": 3.133986711502075, "z_loss": 16.04821765422821, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19152027368545532, "tokens_per_second": 1349423.512951015, "eta_seconds": 26108.65230512619, "data_wait_seconds": 4.583597183227539e-05, "mix": {"fineweb_hq": 0.50966, "dclm": 0.20758, "cosmopedia_v2": 0.20038, "finemath": 0.08239}}
{"step": 5290, "total_steps": 72479, "tokens": 2773483520, "loss": 5.012654960155487, "l_mix": 3.1000094413757324, "l_a": 3.2069068253040314, "l_c": 3.163217455148697, "z_loss": 16.081820726394653, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22968995571136475, "tokens_per_second": 1448588.5011694313, "eta_seconds": 24317.731642603874, "data_wait_seconds": 2.7179718017578125e-05, "mix": {"fineweb_hq": 0.50964, "dclm": 0.20756, "cosmopedia_v2": 0.20038, "finemath": 0.08242}}
{"step": 5300, "total_steps": 72479, "tokens": 2778726400, "loss": 4.869256615638733, "l_mix": 3.0106116235256195, "l_a": 3.1180510222911835, "l_c": 3.0720236897468567, "z_loss": 16.22566330432892, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17207027971744537, "tokens_per_second": 1404848.0451877147, "eta_seconds": 25071.141090774538, "data_wait_seconds": 1.9550323486328125e-05, "mix": {"fineweb_hq": 0.50962, "dclm": 0.20774, "cosmopedia_v2": 0.20038, "finemath": 0.08226}, "gates": {"gate_a2b/L1_absmean": 0.11675163, "gate_a2b/L1_absmax": 0.59924442, "gate_a2b/L1_rms": 0.16153394, "gate_a2b/L3_absmean": 0.26420686, "gate_a2b/L3_absmax": 0.4580819, "gate_a2b/L3_rms": 0.27274376, "gate_a2b/L5_absmean": 0.42998824, "gate_a2b/L5_absmax": 0.63166827, "gate_a2b/L5_rms": 0.44041228, "gate_c2b/L1_absmean": 0.01433813, "gate_c2b/L1_absmax": 0.56966794, "gate_c2b/L1_rms": 0.05166971, "gate_c2b/L3_absmean": 0.24745302, "gate_c2b/L3_absmax": 0.5329392, "gate_c2b/L3_rms": 0.25632957, "gate_c2b/L5_absmean": 0.54480028, "gate_c2b/L5_absmax": 0.76594186, "gate_c2b/L5_rms": 0.56059134, "gate_b2a/L7_absmean": 0.05755283, "gate_b2a/L7_absmax": 0.25905466, "gate_b2a/L7_rms": 0.06571507, "gate_b2a/L11_absmean": 0.11002044, "gate_b2a/L11_absmax": 0.23717953, "gate_b2a/L11_rms": 0.1122051, "gate_b2a/L14_absmean": 0.26501748, "gate_b2a/L14_absmax": 0.42345339, "gate_b2a/L14_rms": 0.26605755, "gate_b2c/L3_absmean": 0.03617702, "gate_b2c/L3_absmax": 0.39873773, "gate_b2c/L3_rms": 0.05536744, "gate_b2c/L5_absmean": 0.0782612, "gate_b2c/L5_absmax": 0.42032793, "gate_b2c/L5_rms": 0.08205169, "gate_b2c/L6_absmean": 0.12430036, "gate_b2c/L6_absmax": 0.34571257, "gate_b2c/L6_rms": 0.12581873}, "streams": {"cos_a2b/round0": 0.35362, "cos_c2b/round0": 0.230927, "cos_b2a/round0": 0.188829, "cos_b2c/round0": 0.109581, "rms_a/round0": 1.351673, "rms_b/round0": 2.429976, "rms_c/round0": 1.335241, "cos_a2b/round1": 0.390218, "cos_c2b/round1": 0.276235, "cos_b2a/round1": 0.201903, "cos_b2c/round1": 0.182314, "rms_a/round1": 2.36824, "rms_b/round1": 4.218016, "rms_c/round1": 2.208076, "cos_a2b/round2": 0.459095, "cos_c2b/round2": 0.438344, "cos_b2a/round2": 0.081288, "cos_b2c/round2": 0.148597, "rms_a/round2": 4.41232, "rms_b/round2": 8.17707, "rms_c/round2": 3.446459}, "alpha_mean": 0.47420195129917886, "alpha_hist": [0.0105, 0.04158, 0.10936, 0.17625, 0.21581, 0.20775, 0.14047, 0.0788, 0.01906, 0.00043], "alpha_by_type": {"special": {"share": 0.00122, "mean": 0.55948, "std": 0.08518}, "whitespace": {"share": 0.02113, "mean": 0.54207, "std": 0.12374}, "digit": {"share": 0.02571, "mean": 0.55953, "std": 0.12777}, "word_start": {"share": 0.57748, "mean": 0.49908, "std": 0.15241}, "word_cont": {"share": 0.27489, "mean": 0.36025, "std": 0.14325}, "punct": {"share": 0.09956, "mean": 0.60705, "std": 0.14905}}, "alpha_p05": 0.20948, "alpha_p50": 0.4792, "alpha_p95": 0.76136}
{"step": 5310, "total_steps": 72479, "tokens": 2783969280, "loss": 4.914379715919495, "l_mix": 3.040139853954315, "l_a": 3.143927901983261, "l_c": 3.0981918275356293, "z_loss": 16.03893208503723, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17993128299713135, "tokens_per_second": 1467731.281569488, "eta_seconds": 23993.42516863346, "data_wait_seconds": 2.2083520889282227e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20772, "cosmopedia_v2": 0.20038, "finemath": 0.0823}}
{"step": 5320, "total_steps": 72479, "tokens": 2789212160, "loss": 4.861094236373901, "l_mix": 3.0067967772483826, "l_a": 3.1093095541000366, "l_c": 3.0663056075572968, "z_loss": 16.12966763973236, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17976050078868866, "tokens_per_second": 1315875.2752235632, "eta_seconds": 26758.35503180027, "data_wait_seconds": 1.71661376953125e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20771, "cosmopedia_v2": 0.20038, "finemath": 0.08233}}
{"step": 5330, "total_steps": 72479, "tokens": 2794455040, "loss": 4.939672768115997, "l_mix": 3.055009424686432, "l_a": 3.158671736717224, "l_c": 3.1181083619594574, "z_loss": 16.291659832000732, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19869807362556458, "tokens_per_second": 1410532.942685053, "eta_seconds": 24958.94554932117, "data_wait_seconds": 2.6553869247436523e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20769, "cosmopedia_v2": 0.20038, "finemath": 0.08236}}
{"step": 5340, "total_steps": 72479, "tokens": 2799697920, "loss": 5.029156744480133, "l_mix": 3.111954092979431, "l_a": 3.2115811109542847, "l_c": 3.17375710606575, "z_loss": 16.0116628408432, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1652447134256363, "tokens_per_second": 1314537.963952271, "eta_seconds": 26777.600189018252, "data_wait_seconds": 2.4318695068359375e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20768, "cosmopedia_v2": 0.20037, "finemath": 0.0824}}
{"step": 5350, "total_steps": 72479, "tokens": 2804940800, "loss": 3.5385273694992065, "l_mix": 2.185759201645851, "l_a": 2.2690517008304596, "l_c": 2.2340716123580933, "z_loss": 18.310681343078613, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2542920708656311, "tokens_per_second": 1453009.1346011404, "eta_seconds": 24222.097655057907, "data_wait_seconds": 2.5212764739990234e-05, "mix": {"fineweb_hq": 0.50953, "dclm": 0.20766, "cosmopedia_v2": 0.20037, "finemath": 0.08243}}
{"step": 5360, "total_steps": 72479, "tokens": 2810183680, "loss": 3.502460867166519, "l_mix": 2.1641308963298798, "l_a": 2.2484461814165115, "l_c": 2.2064954340457916, "z_loss": 18.474799633026123, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2982175350189209, "tokens_per_second": 1465679.8630127034, "eta_seconds": 24009.12174618244, "data_wait_seconds": 2.1517276763916016e-05, "mix": {"fineweb_hq": 0.50951, "dclm": 0.20765, "cosmopedia_v2": 0.20037, "finemath": 0.08246}}
{"step": 5370, "total_steps": 72479, "tokens": 2815426560, "loss": 4.914342284202576, "l_mix": 3.0396373569965363, "l_a": 3.1414906084537506, "l_c": 3.1022115349769592, "z_loss": 15.942068815231323, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.194574236869812, "tokens_per_second": 1442354.6374438256, "eta_seconds": 24393.753435254097, "data_wait_seconds": 1.8596649169921875e-05, "mix": {"fineweb_hq": 0.50968, "dclm": 0.20764, "cosmopedia_v2": 0.20037, "finemath": 0.08231}}
{"step": 5380, "total_steps": 72479, "tokens": 2820669440, "loss": 4.949142277240753, "l_mix": 3.0609080493450165, "l_a": 3.1653126180171967, "l_c": 3.1234543919563293, "z_loss": 16.04185152053833, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16971559822559357, "tokens_per_second": 1389920.5943108667, "eta_seconds": 25310.223228573803, "data_wait_seconds": 2.2202730178833008e-05, "mix": {"fineweb_hq": 0.50967, "dclm": 0.20762, "cosmopedia_v2": 0.20037, "finemath": 0.08234}}
{"step": 5390, "total_steps": 72479, "tokens": 2825912320, "loss": 4.987985551357269, "l_mix": 3.0851417183876038, "l_a": 3.1905942857265472, "l_c": 3.146887183189392, "z_loss": 15.9944349527359, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17196132242679596, "tokens_per_second": 1468464.8050985842, "eta_seconds": 23952.877528882025, "data_wait_seconds": 3.707408905029297e-05, "mix": {"fineweb_hq": 0.50965, "dclm": 0.20761, "cosmopedia_v2": 0.20037, "finemath": 0.08237}}
{"step": 5400, "total_steps": 72479, "tokens": 2831155200, "loss": 4.896331250667572, "l_mix": 3.027981698513031, "l_a": 3.134852945804596, "l_c": 3.0874751210212708, "z_loss": 16.510790944099426, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18864941596984863, "tokens_per_second": 1437423.9905893193, "eta_seconds": 24466.486563634873, "data_wait_seconds": 2.6047229766845703e-05, "mix": {"fineweb_hq": 0.50963, "dclm": 0.20759, "cosmopedia_v2": 0.20037, "finemath": 0.08241}, "gates": {"gate_a2b/L1_absmean": 0.11917993, "gate_a2b/L1_absmax": 0.60134333, "gate_a2b/L1_rms": 0.16448893, "gate_a2b/L3_absmean": 0.26471898, "gate_a2b/L3_absmax": 0.45694908, "gate_a2b/L3_rms": 0.27305412, "gate_a2b/L5_absmean": 0.43214831, "gate_a2b/L5_absmax": 0.63328379, "gate_a2b/L5_rms": 0.44248495, "gate_c2b/L1_absmean": 0.0142843, "gate_c2b/L1_absmax": 0.57114989, "gate_c2b/L1_rms": 0.05181082, "gate_c2b/L3_absmean": 0.24618077, "gate_c2b/L3_absmax": 0.52909374, "gate_c2b/L3_rms": 0.25495934, "gate_c2b/L5_absmean": 0.5454101, "gate_c2b/L5_absmax": 0.78817362, "gate_c2b/L5_rms": 0.56105959, "gate_b2a/L7_absmean": 0.05856624, "gate_b2a/L7_absmax": 0.25842068, "gate_b2a/L7_rms": 0.06642792, "gate_b2a/L11_absmean": 0.11046908, "gate_b2a/L11_absmax": 0.24654144, "gate_b2a/L11_rms": 0.11265133, "gate_b2a/L14_absmean": 0.26550981, "gate_b2a/L14_absmax": 0.42577934, "gate_b2a/L14_rms": 0.26659983, "gate_b2c/L3_absmean": 0.0368281, "gate_b2c/L3_absmax": 0.40291923, "gate_b2c/L3_rms": 0.05625395, "gate_b2c/L5_absmean": 0.07925124, "gate_b2c/L5_absmax": 0.42926881, "gate_b2c/L5_rms": 0.08311582, "gate_b2c/L6_absmean": 0.12531491, "gate_b2c/L6_absmax": 0.34314045, "gate_b2c/L6_rms": 0.12683472}, "streams": {"cos_a2b/round0": 0.357721, "cos_c2b/round0": 0.238025, "cos_b2a/round0": 0.192295, "cos_b2c/round0": 0.113121, "rms_a/round0": 1.348508, "rms_b/round0": 2.420459, "rms_c/round0": 1.342133, "cos_a2b/round1": 0.393446, "cos_c2b/round1": 0.272528, "cos_b2a/round1": 0.198152, "cos_b2c/round1": 0.179933, "rms_a/round1": 2.374521, "rms_b/round1": 4.180122, "rms_c/round1": 2.204186, "cos_a2b/round2": 0.468118, "cos_c2b/round2": 0.423928, "cos_b2a/round2": 0.073578, "cos_b2c/round2": 0.146157, "rms_a/round2": 4.432454, "rms_b/round2": 8.035332, "rms_c/round2": 3.397759}, "alpha_mean": 0.46732281219035343, "alpha_hist": [0.01288, 0.04915, 0.12381, 0.19038, 0.19792, 0.17439, 0.14174, 0.088, 0.02119, 0.00053], "alpha_by_type": {"special": {"share": 0.00136, "mean": 0.5713, "std": 0.09824}, "whitespace": {"share": 0.02754, "mean": 0.55371, "std": 0.12323}, "digit": {"share": 0.04338, "mean": 0.4993, "std": 0.13121}, "word_start": {"share": 0.56682, "mean": 0.49204, "std": 0.16399}, "word_cont": {"share": 0.25125, "mean": 0.34457, "std": 0.14591}, "punct": {"share": 0.10965, "mean": 0.58519, "std": 0.1729}}, "alpha_p05": 0.19993, "alpha_p50": 0.46417, "alpha_p95": 0.74092}
{"step": 5410, "total_steps": 72479, "tokens": 2836398080, "loss": 4.9052735567092896, "l_mix": 3.0329810678958893, "l_a": 3.1406856775283813, "l_c": 3.094951242208481, "z_loss": 16.015175104141235, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18371464312076569, "tokens_per_second": 1416578.3363413564, "eta_seconds": 24822.821985840797, "data_wait_seconds": 1.8328428268432617e-05, "mix": {"fineweb_hq": 0.50961, "dclm": 0.20758, "cosmopedia_v2": 0.20037, "finemath": 0.08244}}
{"step": 5420, "total_steps": 72479, "tokens": 2841640960, "loss": 5.162306845188141, "l_mix": 3.195210188627243, "l_a": 3.2900859713554382, "l_c": 3.2615014016628265, "z_loss": 16.204354763031006, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1644289344549179, "tokens_per_second": 1364260.920580381, "eta_seconds": 25770.897972393035, "data_wait_seconds": 2.4706125259399414e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20775, "cosmopedia_v2": 0.20037, "finemath": 0.08229}}
{"step": 5430, "total_steps": 72479, "tokens": 2846883840, "loss": 5.145462393760681, "l_mix": 3.1853775084018707, "l_a": 3.280368834733963, "l_c": 3.2478465139865875, "z_loss": 16.201850414276123, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17117169499397278, "tokens_per_second": 1472511.1430228453, "eta_seconds": 23872.81500623226, "data_wait_seconds": 2.9712915420532227e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20773, "cosmopedia_v2": 0.20037, "finemath": 0.08232}}
{"step": 5440, "total_steps": 72479, "tokens": 2852126720, "loss": 5.1686437129974365, "l_mix": 3.199948102235794, "l_a": 3.2899420261383057, "l_c": 3.2669014930725098, "z_loss": 16.425134420394897, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1892758011817932, "tokens_per_second": 1384954.312453977, "eta_seconds": 25378.269099521636, "data_wait_seconds": 2.8461217880249023e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20772, "cosmopedia_v2": 0.20037, "finemath": 0.08235}}
{"step": 5450, "total_steps": 72479, "tokens": 2857369600, "loss": 5.204125344753265, "l_mix": 3.2221516966819763, "l_a": 3.312543421983719, "l_c": 3.2885748744010925, "z_loss": 16.37972855567932, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15998482704162598, "tokens_per_second": 1434520.6974241005, "eta_seconds": 24497.729740047453, "data_wait_seconds": 5.9038400650024414e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20771, "cosmopedia_v2": 0.20037, "finemath": 0.08239}}
{"step": 5460, "total_steps": 72479, "tokens": 2862612480, "loss": 5.142950117588043, "l_mix": 3.184664875268936, "l_a": 3.276638686656952, "l_c": 3.2455706894397736, "z_loss": 16.224073886871338, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1768665462732315, "tokens_per_second": 1419709.4097667355, "eta_seconds": 24749.612301135065, "data_wait_seconds": 2.0295381546020508e-05, "mix": {"fineweb_hq": 0.50952, "dclm": 0.20769, "cosmopedia_v2": 0.20037, "finemath": 0.08242}}
{"step": 5470, "total_steps": 72479, "tokens": 2867855360, "loss": 4.28997340798378, "l_mix": 2.6556850522756577, "l_a": 2.7393476516008377, "l_c": 2.7024209797382355, "z_loss": 17.576170325279236, "lr": 0.0023, "wd": 0.1, "grad_norm": 1.1466469764709473, "tokens_per_second": 1455002.589104402, "eta_seconds": 24145.671530127525, "data_wait_seconds": 2.2917985916137695e-05, "mix": {"fineweb_hq": 0.50951, "dclm": 0.20768, "cosmopedia_v2": 0.20037, "finemath": 0.08245}}
{"step": 5480, "total_steps": 72479, "tokens": 2873098240, "loss": 4.914914309978485, "l_mix": 3.040016859769821, "l_a": 3.143494725227356, "l_c": 3.1006877422332764, "z_loss": 16.42643642425537, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15486912429332733, "tokens_per_second": 1275228.7068934923, "eta_seconds": 27545.46813611984, "data_wait_seconds": 3.454089164733887e-05, "mix": {"fineweb_hq": 0.50967, "dclm": 0.20766, "cosmopedia_v2": 0.20036, "finemath": 0.0823}}
{"step": 5490, "total_steps": 72479, "tokens": 2878341120, "loss": 4.901122629642487, "l_mix": 3.0316294729709625, "l_a": 3.136649042367935, "l_c": 3.0896150171756744, "z_loss": 16.13743805885315, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19494123756885529, "tokens_per_second": 1445685.8832942145, "eta_seconds": 24294.024890089036, "data_wait_seconds": 2.8401613235473633e-05, "mix": {"fineweb_hq": 0.50965, "dclm": 0.20765, "cosmopedia_v2": 0.20036, "finemath": 0.08233}}
{"step": 5500, "total_steps": 72479, "tokens": 2883584000, "loss": 4.930951118469238, "l_mix": 3.049053132534027, "l_a": 3.160392701625824, "l_c": 3.1072945296764374, "z_loss": 15.917225003242493, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16624923050403595, "tokens_per_second": 1395465.1577814, "eta_seconds": 25164.573802638057, "data_wait_seconds": 1.5854835510253906e-05, "mix": {"fineweb_hq": 0.50964, "dclm": 0.20764, "cosmopedia_v2": 0.20036, "finemath": 0.08236}, "gates": {"gate_a2b/L1_absmean": 0.12168308, "gate_a2b/L1_absmax": 0.60519934, "gate_a2b/L1_rms": 0.16600096, "gate_a2b/L3_absmean": 0.26489249, "gate_a2b/L3_absmax": 0.45985183, "gate_a2b/L3_rms": 0.27302808, "gate_a2b/L5_absmean": 0.43212232, "gate_a2b/L5_absmax": 0.62590945, "gate_a2b/L5_rms": 0.44244179, "gate_c2b/L1_absmean": 0.0149759, "gate_c2b/L1_absmax": 0.57279319, "gate_c2b/L1_rms": 0.05209314, "gate_c2b/L3_absmean": 0.24612729, "gate_c2b/L3_absmax": 0.53075856, "gate_c2b/L3_rms": 0.25485715, "gate_c2b/L5_absmean": 0.54456055, "gate_c2b/L5_absmax": 0.78953755, "gate_c2b/L5_rms": 0.56014091, "gate_b2a/L7_absmean": 0.05922947, "gate_b2a/L7_absmax": 0.25846168, "gate_b2a/L7_rms": 0.06713888, "gate_b2a/L11_absmean": 0.1105257, "gate_b2a/L11_absmax": 0.24266453, "gate_b2a/L11_rms": 0.11287709, "gate_b2a/L14_absmean": 0.26533529, "gate_b2a/L14_absmax": 0.44006217, "gate_b2a/L14_rms": 0.26647905, "gate_b2c/L3_absmean": 0.03782214, "gate_b2c/L3_absmax": 0.40828186, "gate_b2c/L3_rms": 0.05678403, "gate_b2c/L5_absmean": 0.07937752, "gate_b2c/L5_absmax": 0.43316653, "gate_b2c/L5_rms": 0.08333009, "gate_b2c/L6_absmean": 0.12509885, "gate_b2c/L6_absmax": 0.35076973, "gate_b2c/L6_rms": 0.12675235}, "streams": {"cos_a2b/round0": 0.359458, "cos_c2b/round0": 0.236382, "cos_b2a/round0": 0.191029, "cos_b2c/round0": 0.109722, "rms_a/round0": 1.354425, "rms_b/round0": 2.449139, "rms_c/round0": 1.347103, "cos_a2b/round1": 0.392842, "cos_c2b/round1": 0.269254, "cos_b2a/round1": 0.199976, "cos_b2c/round1": 0.185033, "rms_a/round1": 2.386115, "rms_b/round1": 4.222157, "rms_c/round1": 2.233759, "cos_a2b/round2": 0.477124, "cos_c2b/round2": 0.418516, "cos_b2a/round2": 0.06372, "cos_b2c/round2": 0.137195, "rms_a/round2": 4.454847, "rms_b/round2": 8.046185, "rms_c/round2": 3.46285}, "alpha_mean": 0.45712622023881266, "alpha_hist": [0.01741, 0.06038, 0.13377, 0.17839, 0.19579, 0.18208, 0.13148, 0.08049, 0.01979, 0.00041], "alpha_by_type": {"special": {"share": 0.0013, "mean": 0.56229, "std": 0.09367}, "whitespace": {"share": 0.02898, "mean": 0.53865, "std": 0.11971}, "digit": {"share": 0.04489, "mean": 0.50051, "std": 0.12788}, "word_start": {"share": 0.57509, "mean": 0.47964, "std": 0.16659}, "word_cont": {"share": 0.25269, "mean": 0.33789, "std": 0.15644}, "punct": {"share": 0.09706, "mean": 0.58836, "std": 0.17116}}, "alpha_p05": 0.15104, "alpha_p50": 0.45362, "alpha_p95": 0.75055}
{"step": 5510, "total_steps": 72479, "tokens": 2888826880, "loss": 4.911926567554474, "l_mix": 3.038111984729767, "l_a": 3.1431890428066254, "l_c": 3.0973396003246307, "z_loss": 16.5605411529541, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19647128880023956, "tokens_per_second": 1462709.5850112631, "eta_seconds": 24004.110885572434, "data_wait_seconds": 1.6927719116210938e-05, "mix": {"fineweb_hq": 0.50962, "dclm": 0.20762, "cosmopedia_v2": 0.20036, "finemath": 0.0824}}
{"step": 5520, "total_steps": 72479, "tokens": 2894069760, "loss": 4.91522353887558, "l_mix": 3.0398280024528503, "l_a": 3.1453458070755005, "l_c": 3.100564181804657, "z_loss": 16.22523808479309, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1884874552488327, "tokens_per_second": 1397620.1656546898, "eta_seconds": 25118.26965200901, "data_wait_seconds": 2.1219253540039062e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20761, "cosmopedia_v2": 0.20036, "finemath": 0.08243}}
{"step": 5530, "total_steps": 72479, "tokens": 2899312640, "loss": 3.7381178438663483, "l_mix": 2.2956662476062775, "l_a": 2.433458983898163, "l_c": 2.3695211708545685, "z_loss": 15.574886679649353, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2126796990633011, "tokens_per_second": 1357478.2404848202, "eta_seconds": 25857.178601598738, "data_wait_seconds": 2.22623348236084e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20759, "cosmopedia_v2": 0.20054, "finemath": 0.08228}}
{"step": 5540, "total_steps": 72479, "tokens": 2904555520, "loss": 5.130177557468414, "l_mix": 3.175769805908203, "l_a": 3.2707995176315308, "l_c": 3.2383777499198914, "z_loss": 16.546179056167603, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.23923246562480927, "tokens_per_second": 1429689.0121545035, "eta_seconds": 24547.516371488575, "data_wait_seconds": 1.65402889251709e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20776, "cosmopedia_v2": 0.20036, "finemath": 0.08231}}
{"step": 5550, "total_steps": 72479, "tokens": 2909798400, "loss": 5.129030764102936, "l_mix": 3.1735765039920807, "l_a": 3.2715014815330505, "l_c": 3.2413655817508698, "z_loss": 15.940436840057373, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2342061698436737, "tokens_per_second": 1270206.0441598434, "eta_seconds": 27625.49565351009, "data_wait_seconds": 2.440810203552246e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20775, "cosmopedia_v2": 0.20036, "finemath": 0.08234}}
{"step": 5560, "total_steps": 72479, "tokens": 2915041280, "loss": 5.111136794090271, "l_mix": 3.1655033826828003, "l_a": 3.253058910369873, "l_c": 3.227074056863785, "z_loss": 15.93494701385498, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21299369633197784, "tokens_per_second": 1389490.8688554382, "eta_seconds": 25250.132590579986, "data_wait_seconds": 2.60770320892334e-05, "mix": {"fineweb_hq": 0.50953, "dclm": 0.20773, "cosmopedia_v2": 0.20036, "finemath": 0.08237}}
{"step": 5570, "total_steps": 72479, "tokens": 2920284160, "loss": 5.04694926738739, "l_mix": 3.1240449845790863, "l_a": 3.2160635590553284, "l_c": 3.187994599342346, "z_loss": 16.867607355117798, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17864687740802765, "tokens_per_second": 1435246.9662874562, "eta_seconds": 24441.49795539379, "data_wait_seconds": 3.2454729080200195e-05, "mix": {"fineweb_hq": 0.50952, "dclm": 0.20772, "cosmopedia_v2": 0.20036, "finemath": 0.08241}}
{"step": 5580, "total_steps": 72479, "tokens": 2925527040, "loss": 3.5027237832546234, "l_mix": 2.1634725481271744, "l_a": 2.2563930451869965, "l_c": 2.201703354716301, "z_loss": 18.222923517227173, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.27736231684684753, "tokens_per_second": 1454995.1762371564, "eta_seconds": 24106.157521915433, "data_wait_seconds": 2.086162567138672e-05, "mix": {"fineweb_hq": 0.5095, "dclm": 0.20771, "cosmopedia_v2": 0.20036, "finemath": 0.08244}}
{"step": 5590, "total_steps": 72479, "tokens": 2930769920, "loss": 4.837318301200867, "l_mix": 2.9913129806518555, "l_a": 3.0972468554973602, "l_c": 3.0506934225559235, "z_loss": 16.233044028282166, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18757419288158417, "tokens_per_second": 1463638.2624354856, "eta_seconds": 23960.22359626293, "data_wait_seconds": 3.135204315185547e-05, "mix": {"fineweb_hq": 0.50966, "dclm": 0.20769, "cosmopedia_v2": 0.20036, "finemath": 0.08229}}
{"step": 5600, "total_steps": 72479, "tokens": 2936012800, "loss": 4.89291125535965, "l_mix": 3.025477319955826, "l_a": 3.1318697035312653, "l_c": 3.08744353055954, "z_loss": 16.397505521774292, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1986023336648941, "tokens_per_second": 1387436.817118581, "eta_seconds": 25272.39923243523, "data_wait_seconds": 2.276897430419922e-05, "mix": {"fineweb_hq": 0.50964, "dclm": 0.20768, "cosmopedia_v2": 0.20036, "finemath": 0.08232}, "gates": {"gate_a2b/L1_absmean": 0.12371383, "gate_a2b/L1_absmax": 0.60690325, "gate_a2b/L1_rms": 0.16811968, "gate_a2b/L3_absmean": 0.26555094, "gate_a2b/L3_absmax": 0.45382741, "gate_a2b/L3_rms": 0.27363518, "gate_a2b/L5_absmean": 0.43369928, "gate_a2b/L5_absmax": 0.62958854, "gate_a2b/L5_rms": 0.44376445, "gate_c2b/L1_absmean": 0.01476842, "gate_c2b/L1_absmax": 0.57517844, "gate_c2b/L1_rms": 0.05212598, "gate_c2b/L3_absmean": 0.24505253, "gate_c2b/L3_absmax": 0.53394121, "gate_c2b/L3_rms": 0.25364941, "gate_c2b/L5_absmean": 0.54621708, "gate_c2b/L5_absmax": 0.79810125, "gate_c2b/L5_rms": 0.56180519, "gate_b2a/L7_absmean": 0.05932716, "gate_b2a/L7_absmax": 0.26197761, "gate_b2a/L7_rms": 0.06722175, "gate_b2a/L11_absmean": 0.11155725, "gate_b2a/L11_absmax": 0.246677, "gate_b2a/L11_rms": 0.11379424, "gate_b2a/L14_absmean": 0.26852867, "gate_b2a/L14_absmax": 0.43487188, "gate_b2a/L14_rms": 0.26963741, "gate_b2c/L3_absmean": 0.03785485, "gate_b2c/L3_absmax": 0.41016045, "gate_b2c/L3_rms": 0.05745326, "gate_b2c/L5_absmean": 0.08012884, "gate_b2c/L5_absmax": 0.4382427, "gate_b2c/L5_rms": 0.08407128, "gate_b2c/L6_absmean": 0.12671383, "gate_b2c/L6_absmax": 0.35300377, "gate_b2c/L6_rms": 0.12834537}, "streams": {"cos_a2b/round0": 0.357967, "cos_c2b/round0": 0.232507, "cos_b2a/round0": 0.190123, "cos_b2c/round0": 0.115059, "rms_a/round0": 1.346788, "rms_b/round0": 2.408565, "rms_c/round0": 1.345543, "cos_a2b/round1": 0.395586, "cos_c2b/round1": 0.274699, "cos_b2a/round1": 0.198433, "cos_b2c/round1": 0.180941, "rms_a/round1": 2.369079, "rms_b/round1": 4.149008, "rms_c/round1": 2.20923, "cos_a2b/round2": 0.474177, "cos_c2b/round2": 0.433737, "cos_b2a/round2": 0.07493, "cos_b2c/round2": 0.141295, "rms_a/round2": 4.40536, "rms_b/round2": 7.981289, "rms_c/round2": 3.42216}, "alpha_mean": 0.4733777406062245, "alpha_hist": [0.01372, 0.05522, 0.12666, 0.17458, 0.17726, 0.17813, 0.14766, 0.09819, 0.02785, 0.00073], "alpha_by_type": {"special": {"share": 0.0009, "mean": 0.645, "std": 0.09998}, "whitespace": {"share": 0.02486, "mean": 0.60198, "std": 0.13122}, "digit": {"share": 0.03755, "mean": 0.63214, "std": 0.12719}, "word_start": {"share": 0.57379, "mean": 0.48493, "std": 0.16574}, "word_cont": {"share": 0.25156, "mean": 0.35227, "std": 0.16162}, "punct": {"share": 0.11134, "mean": 0.60383, "std": 0.17064}}, "alpha_p05": 0.19315, "alpha_p50": 0.46636, "alpha_p95": 0.78166}
{"step": 5610, "total_steps": 72479, "tokens": 2941255680, "loss": 4.920823454856873, "l_mix": 3.042881488800049, "l_a": 3.148675948381424, "l_c": 3.1057406961917877, "z_loss": 16.170067071914673, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19117994606494904, "tokens_per_second": 1431662.8490818054, "eta_seconds": 24488.038014316557, "data_wait_seconds": 1.5437602996826172e-05, "mix": {"fineweb_hq": 0.50963, "dclm": 0.20766, "cosmopedia_v2": 0.20036, "finemath": 0.08235}}
{"step": 5620, "total_steps": 72479, "tokens": 2946498560, "loss": 4.971851646900177, "l_mix": 3.0747485160827637, "l_a": 3.1817378103733063, "l_c": 3.1365222334861755, "z_loss": 16.25071358680725, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17645657062530518, "tokens_per_second": 1440231.5188957162, "eta_seconds": 24338.70591783524, "data_wait_seconds": 2.4825334548950195e-05, "mix": {"fineweb_hq": 0.50961, "dclm": 0.20765, "cosmopedia_v2": 0.20036, "finemath": 0.08238}}
{"step": 5630, "total_steps": 72479, "tokens": 2951741440, "loss": 4.928400874137878, "l_mix": 3.0479597747325897, "l_a": 3.1564191579818726, "l_c": 3.1063098907470703, "z_loss": 16.2223938703537, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22474388778209686, "tokens_per_second": 1464398.709899024, "eta_seconds": 23933.46038553715, "data_wait_seconds": 2.3245811462402344e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20764, "cosmopedia_v2": 0.20036, "finemath": 0.08242}}
{"step": 5640, "total_steps": 72479, "tokens": 2956984320, "loss": 3.7405619621276855, "l_mix": 2.296955317258835, "l_a": 2.432041436433792, "l_c": 2.3747698366642, "z_loss": 15.63184940814972, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.24609053134918213, "tokens_per_second": 1399335.399427125, "eta_seconds": 25042.52064683437, "data_wait_seconds": 1.9818544387817383e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20762, "cosmopedia_v2": 0.20053, "finemath": 0.08227}}
{"step": 5650, "total_steps": 72479, "tokens": 2962227200, "loss": 3.7925228476524353, "l_mix": 2.3296788036823273, "l_a": 2.4669012427330017, "l_c": 2.4039913713932037, "z_loss": 15.762891411781311, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2610551118850708, "tokens_per_second": 1426032.0371680069, "eta_seconds": 24570.024963521955, "data_wait_seconds": 3.439188003540039e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20761, "cosmopedia_v2": 0.20053, "finemath": 0.0823}}
{"step": 5660, "total_steps": 72479, "tokens": 2967470080, "loss": 3.7596468329429626, "l_mix": 2.3093592524528503, "l_a": 2.440748393535614, "l_c": 2.388360947370529, "z_loss": 15.54713749885559, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2160082906484604, "tokens_per_second": 1351566.4763627106, "eta_seconds": 25919.84965939522, "data_wait_seconds": 1.7970800399780273e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.2076, "cosmopedia_v2": 0.20053, "finemath": 0.08233}}
{"step": 5670, "total_steps": 72479, "tokens": 2972712960, "loss": 3.7532577216625214, "l_mix": 2.3045639991760254, "l_a": 2.4455865919589996, "l_c": 2.378203868865967, "z_loss": 15.565669894218445, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22609645128250122, "tokens_per_second": 1419887.1563570432, "eta_seconds": 24668.97234416008, "data_wait_seconds": 2.1636486053466797e-05, "mix": {"fineweb_hq": 0.50952, "dclm": 0.20758, "cosmopedia_v2": 0.20053, "finemath": 0.08236}}
{"step": 5680, "total_steps": 72479, "tokens": 2977955840, "loss": 5.184573590755463, "l_mix": 3.209995448589325, "l_a": 3.3033081889152527, "l_c": 3.273119270801544, "z_loss": 16.49919581413269, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20292219519615173, "tokens_per_second": 1306145.862431449, "eta_seconds": 26813.172341108322, "data_wait_seconds": 3.3468008041381836e-05, "mix": {"fineweb_hq": 0.50951, "dclm": 0.20775, "cosmopedia_v2": 0.20035, "finemath": 0.08239}}
{"step": 5690, "total_steps": 72479, "tokens": 2983198720, "loss": 5.2283390164375305, "l_mix": 3.2370744943618774, "l_a": 3.3300183415412903, "l_c": 3.3020914793014526, "z_loss": 16.31402063369751, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21555964648723602, "tokens_per_second": 1339726.375541132, "eta_seconds": 26137.181346344947, "data_wait_seconds": 2.3931264877319336e-05, "mix": {"fineweb_hq": 0.50949, "dclm": 0.20773, "cosmopedia_v2": 0.20035, "finemath": 0.08243}}
{"step": 5700, "total_steps": 72479, "tokens": 2988441600, "loss": 4.861319601535797, "l_mix": 3.006424367427826, "l_a": 3.1102048456668854, "l_c": 3.0674109160900116, "z_loss": 16.10486078262329, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2055196911096573, "tokens_per_second": 1262577.3424090191, "eta_seconds": 27730.12565329075, "data_wait_seconds": 2.944469451904297e-05, "mix": {"fineweb_hq": 0.50965, "dclm": 0.20772, "cosmopedia_v2": 0.20035, "finemath": 0.08228}, "gates": {"gate_a2b/L1_absmean": 0.12453456, "gate_a2b/L1_absmax": 0.61569083, "gate_a2b/L1_rms": 0.16927144, "gate_a2b/L3_absmean": 0.26540118, "gate_a2b/L3_absmax": 0.45092228, "gate_a2b/L3_rms": 0.27331546, "gate_a2b/L5_absmean": 0.43436137, "gate_a2b/L5_absmax": 0.63409001, "gate_a2b/L5_rms": 0.44439667, "gate_c2b/L1_absmean": 0.01537273, "gate_c2b/L1_absmax": 0.58517575, "gate_c2b/L1_rms": 0.0537282, "gate_c2b/L3_absmean": 0.24399844, "gate_c2b/L3_absmax": 0.5167498, "gate_c2b/L3_rms": 0.2521936, "gate_c2b/L5_absmean": 0.54521346, "gate_c2b/L5_absmax": 0.78793788, "gate_c2b/L5_rms": 0.56084394, "gate_b2a/L7_absmean": 0.05973323, "gate_b2a/L7_absmax": 0.25991645, "gate_b2a/L7_rms": 0.06737626, "gate_b2a/L11_absmean": 0.11237881, "gate_b2a/L11_absmax": 0.24683823, "gate_b2a/L11_rms": 0.11455718, "gate_b2a/L14_absmean": 0.2690227, "gate_b2a/L14_absmax": 0.44337597, "gate_b2a/L14_rms": 0.27015084, "gate_b2c/L3_absmean": 0.03875057, "gate_b2c/L3_absmax": 0.41833034, "gate_b2c/L3_rms": 0.05810708, "gate_b2c/L5_absmean": 0.0801601, "gate_b2c/L5_absmax": 0.43782613, "gate_b2c/L5_rms": 0.0842444, "gate_b2c/L6_absmean": 0.1269749, "gate_b2c/L6_absmax": 0.35149446, "gate_b2c/L6_rms": 0.12860595}, "streams": {"cos_a2b/round0": 0.3713, "cos_c2b/round0": 0.244611, "cos_b2a/round0": 0.192192, "cos_b2c/round0": 0.122895, "rms_a/round0": 1.355505, "rms_b/round0": 2.391051, "rms_c/round0": 1.347878, "cos_a2b/round1": 0.393542, "cos_c2b/round1": 0.269866, "cos_b2a/round1": 0.199367, "cos_b2c/round1": 0.171073, "rms_a/round1": 2.358571, "rms_b/round1": 4.180341, "rms_c/round1": 2.22676, "cos_a2b/round2": 0.470219, "cos_c2b/round2": 0.436983, "cos_b2a/round2": 0.087142, "cos_b2c/round2": 0.13541, "rms_a/round2": 4.528024, "rms_b/round2": 8.288503, "rms_c/round2": 3.43454}, "alpha_mean": 0.4481728937638394, "alpha_hist": [0.01865, 0.05972, 0.13904, 0.19621, 0.19604, 0.17471, 0.12634, 0.07471, 0.01439, 0.00018], "alpha_by_type": {"special": {"share": 0.00067, "mean": 0.58296, "std": 0.10543}, "whitespace": {"share": 0.03703, "mean": 0.52949, "std": 0.1345}, "digit": {"share": 0.0687, "mean": 0.51237, "std": 0.13184}, "word_start": {"share": 0.51488, "mean": 0.46596, "std": 0.16402}, "word_cont": {"share": 0.25835, "mean": 0.32776, "std": 0.14681}, "punct": {"share": 0.12038, "mean": 0.5681, "std": 0.16946}}, "alpha_p05": 0.16132, "alpha_p50": 0.43686, "alpha_p95": 0.74503}
{"step": 5710, "total_steps": 72479, "tokens": 2993684480, "loss": 4.923932313919067, "l_mix": 3.0443228781223297, "l_a": 3.157147616147995, "l_c": 3.1027550101280212, "z_loss": 16.3857421875, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17814740538597107, "tokens_per_second": 1459360.334124282, "eta_seconds": 23987.348877072334, "data_wait_seconds": 3.007054328918457e-05, "mix": {"fineweb_hq": 0.50963, "dclm": 0.20771, "cosmopedia_v2": 0.20035, "finemath": 0.08231}}
{"step": 5720, "total_steps": 72479, "tokens": 2998927360, "loss": 4.929732441902161, "l_mix": 3.048478841781616, "l_a": 3.1567194163799286, "l_c": 3.1087192595005035, "z_loss": 16.218945384025574, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19896100461483002, "tokens_per_second": 1391487.7887967876, "eta_seconds": 25153.61102972031, "data_wait_seconds": 3.17692756652832e-05, "mix": {"fineweb_hq": 0.50962, "dclm": 0.20769, "cosmopedia_v2": 0.20035, "finemath": 0.08234}}
{"step": 5730, "total_steps": 72479, "tokens": 3004170240, "loss": 4.806049823760986, "l_mix": 2.9708410799503326, "l_a": 3.082167625427246, "l_c": 3.029797285795212, "z_loss": 16.19284760951996, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1620139479637146, "tokens_per_second": 1428538.9277626232, "eta_seconds": 24497.547131466865, "data_wait_seconds": 2.0235776901245117e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20768, "cosmopedia_v2": 0.20035, "finemath": 0.08237}}
{"step": 5740, "total_steps": 72479, "tokens": 3009413120, "loss": 4.639226973056793, "l_mix": 2.8659680485725403, "l_a": 2.9802514612674713, "l_c": 2.9249361753463745, "z_loss": 17.02546238899231, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22999736666679382, "tokens_per_second": 1443171.067811916, "eta_seconds": 24245.53652191162, "data_wait_seconds": 4.369020462036133e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20035, "finemath": 0.0824}}
{"step": 5750, "total_steps": 72479, "tokens": 3014656000, "loss": 3.8057226836681366, "l_mix": 2.3377596735954285, "l_a": 2.4747509360313416, "l_c": 2.4133033752441406, "z_loss": 15.465824007987976, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.27556276321411133, "tokens_per_second": 1414968.532920288, "eta_seconds": 24725.08267006874, "data_wait_seconds": 2.428889274597168e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20052, "finemath": 0.08226}}
{"step": 5760, "total_steps": 72479, "tokens": 3019898880, "loss": 3.7533772587776184, "l_mix": 2.304469048976898, "l_a": 2.4432644844055176, "l_c": 2.3812860250473022, "z_loss": 15.430013298988342, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19697725772857666, "tokens_per_second": 1421986.074869782, "eta_seconds": 24599.37666773796, "data_wait_seconds": 2.086162567138672e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20764, "cosmopedia_v2": 0.20052, "finemath": 0.08229}}
{"step": 5770, "total_steps": 72479, "tokens": 3025141760, "loss": 3.78641414642334, "l_mix": 2.325680524110794, "l_a": 2.4647068679332733, "l_c": 2.3992433547973633, "z_loss": 15.485063672065735, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20186355710029602, "tokens_per_second": 1463803.7937558878, "eta_seconds": 23893.043822669984, "data_wait_seconds": 2.810359001159668e-05, "mix": {"fineweb_hq": 0.50953, "dclm": 0.20763, "cosmopedia_v2": 0.20052, "finemath": 0.08232}}
{"step": 5780, "total_steps": 72479, "tokens": 3030384640, "loss": 3.738992065191269, "l_mix": 2.295310765504837, "l_a": 2.435558319091797, "l_c": 2.3715270459651947, "z_loss": 15.556597709655762, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19744640588760376, "tokens_per_second": 1337246.5099706817, "eta_seconds": 26150.36573381424, "data_wait_seconds": 3.987550735473633e-05, "mix": {"fineweb_hq": 0.50952, "dclm": 0.20761, "cosmopedia_v2": 0.20052, "finemath": 0.08235}}
{"step": 5790, "total_steps": 72479, "tokens": 3035627520, "loss": 3.7727136313915253, "l_mix": 2.315539240837097, "l_a": 2.457844167947769, "l_c": 2.394185423851013, "z_loss": 15.654692769050598, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21271708607673645, "tokens_per_second": 1411905.904209586, "eta_seconds": 24763.861619782445, "data_wait_seconds": 2.1666288375854492e-05, "mix": {"fineweb_hq": 0.5095, "dclm": 0.2076, "cosmopedia_v2": 0.20052, "finemath": 0.08238}}
{"step": 5800, "total_steps": 72479, "tokens": 3040870400, "loss": 4.468764960765839, "l_mix": 2.7576844692230225, "l_a": 2.8765639066696167, "l_c": 2.821303129196167, "z_loss": 17.20434832572937, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.3169156014919281, "tokens_per_second": 1468136.9616108611, "eta_seconds": 23811.810795664787, "data_wait_seconds": 2.7298927307128906e-05, "mix": {"fineweb_hq": 0.50966, "dclm": 0.20759, "cosmopedia_v2": 0.20034, "finemath": 0.08241}, "gates": {"gate_a2b/L1_absmean": 0.12614146, "gate_a2b/L1_absmax": 0.59804112, "gate_a2b/L1_rms": 0.16834997, "gate_a2b/L3_absmean": 0.26359615, "gate_a2b/L3_absmax": 0.44072396, "gate_a2b/L3_rms": 0.27157897, "gate_a2b/L5_absmean": 0.43537274, "gate_a2b/L5_absmax": 0.63163245, "gate_a2b/L5_rms": 0.44546416, "gate_c2b/L1_absmean": 0.01679722, "gate_c2b/L1_absmax": 0.57291871, "gate_c2b/L1_rms": 0.05306049, "gate_c2b/L3_absmean": 0.24184434, "gate_c2b/L3_absmax": 0.52035028, "gate_c2b/L3_rms": 0.25014132, "gate_c2b/L5_absmean": 0.5473175, "gate_c2b/L5_absmax": 0.77030206, "gate_c2b/L5_rms": 0.56304127, "gate_b2a/L7_absmean": 0.0607909, "gate_b2a/L7_absmax": 0.25251573, "gate_b2a/L7_rms": 0.06817869, "gate_b2a/L11_absmean": 0.11272886, "gate_b2a/L11_absmax": 0.24585564, "gate_b2a/L11_rms": 0.11487441, "gate_b2a/L14_absmean": 0.27293366, "gate_b2a/L14_absmax": 0.43165857, "gate_b2a/L14_rms": 0.27402532, "gate_b2c/L3_absmean": 0.03947891, "gate_b2c/L3_absmax": 0.40030879, "gate_b2c/L3_rms": 0.05839708, "gate_b2c/L5_absmean": 0.08027489, "gate_b2c/L5_absmax": 0.42264345, "gate_b2c/L5_rms": 0.08406955, "gate_b2c/L6_absmean": 0.12753497, "gate_b2c/L6_absmax": 0.35339549, "gate_b2c/L6_rms": 0.1291324}, "streams": {"cos_a2b/round0": 0.364272, "cos_c2b/round0": 0.235424, "cos_b2a/round0": 0.189452, "cos_b2c/round0": 0.120954, "rms_a/round0": 1.327083, "rms_b/round0": 2.39679, "rms_c/round0": 1.337649, "cos_a2b/round1": 0.393926, "cos_c2b/round1": 0.289529, "cos_b2a/round1": 0.212073, "cos_b2c/round1": 0.191376, "rms_a/round1": 2.318354, "rms_b/round1": 4.104169, "rms_c/round1": 2.20583, "cos_a2b/round2": 0.481487, "cos_c2b/round2": 0.438772, "cos_b2a/round2": 0.085069, "cos_b2c/round2": 0.142826, "rms_a/round2": 4.422198, "rms_b/round2": 7.88683, "rms_c/round2": 3.370897}, "alpha_mean": 0.46072614492566544, "alpha_hist": [0.01372, 0.0596, 0.12413, 0.16167, 0.21706, 0.20863, 0.12508, 0.07045, 0.01913, 0.00053], "alpha_by_type": {"special": {"share": 0.00064, "mean": 0.56352, "std": 0.09667}, "whitespace": {"share": 0.0439, "mean": 0.55061, "std": 0.10962}, "digit": {"share": 0.02385, "mean": 0.53358, "std": 0.12965}, "word_start": {"share": 0.58215, "mean": 0.473, "std": 0.16028}, "word_cont": {"share": 0.24782, "mean": 0.35484, "std": 0.15779}, "punct": {"share": 0.10164, "mean": 0.59204, "std": 0.15867}}, "alpha_p05": 0.18952, "alpha_p50": 0.45689, "alpha_p95": 0.74503}
{"step": 5810, "total_steps": 72479, "tokens": 3046113280, "loss": 4.578062474727631, "l_mix": 2.8264442682266235, "l_a": 2.948948860168457, "l_c": 2.884236514568329, "z_loss": 16.62526273727417, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.27002084255218506, "tokens_per_second": 1387183.7909206217, "eta_seconds": 25197.639203095434, "data_wait_seconds": 3.337860107421875e-05, "mix": {"fineweb_hq": 0.50964, "dclm": 0.20757, "cosmopedia_v2": 0.20034, "finemath": 0.08244}}
{"step": 5820, "total_steps": 72479, "tokens": 3051356160, "loss": 4.406466722488403, "l_mix": 2.7193563580513, "l_a": 2.8409860730171204, "l_c": 2.7771309316158295, "z_loss": 16.75324583053589, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17303165793418884, "tokens_per_second": 1392257.9472472141, "eta_seconds": 25102.039360666276, "data_wait_seconds": 3.319978713989258e-05, "mix": {"fineweb_hq": 0.50962, "dclm": 0.20773, "cosmopedia_v2": 0.20034, "finemath": 0.0823}}
{"step": 5830, "total_steps": 72479, "tokens": 3056599040, "loss": 4.4266602993011475, "l_mix": 2.732800602912903, "l_a": 2.85027015209198, "l_c": 2.790432721376419, "z_loss": 16.488112092018127, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18016469478607178, "tokens_per_second": 1395275.766487493, "eta_seconds": 25043.988974285126, "data_wait_seconds": 3.504753112792969e-05, "mix": {"fineweb_hq": 0.50961, "dclm": 0.20772, "cosmopedia_v2": 0.20034, "finemath": 0.08233}}
{"step": 5840, "total_steps": 72479, "tokens": 3061841920, "loss": 4.452270686626434, "l_mix": 2.747802585363388, "l_a": 2.8684111833572388, "l_c": 2.8074182271957397, "z_loss": 17.192850828170776, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19660143554210663, "tokens_per_second": 1332726.9089335334, "eta_seconds": 26215.44428780079, "data_wait_seconds": 1.621246337890625e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20771, "cosmopedia_v2": 0.20034, "finemath": 0.08236}}
{"step": 5850, "total_steps": 72479, "tokens": 3067084800, "loss": 4.388526916503906, "l_mix": 2.707864433526993, "l_a": 2.8286458253860474, "l_c": 2.7679202258586884, "z_loss": 16.926867246627808, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22074374556541443, "tokens_per_second": 1315915.3554938557, "eta_seconds": 26546.37701897621, "data_wait_seconds": 2.002716064453125e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20769, "cosmopedia_v2": 0.20034, "finemath": 0.08239}}
{"step": 5860, "total_steps": 72479, "tokens": 3072327680, "loss": 3.7308992743492126, "l_mix": 2.2901708781719208, "l_a": 2.4315319657325745, "l_c": 2.365618646144867, "z_loss": 15.831690311431885, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.24912934005260468, "tokens_per_second": 1430096.345821864, "eta_seconds": 24423.20922925472, "data_wait_seconds": 1.6897916793823242e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.20051, "finemath": 0.08225}}
{"step": 5870, "total_steps": 72479, "tokens": 3077570560, "loss": 3.7236535847187042, "l_mix": 2.2863850593566895, "l_a": 2.42399001121521, "l_c": 2.361648440361023, "z_loss": 15.769344806671143, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2034134715795517, "tokens_per_second": 1462597.4135458765, "eta_seconds": 23876.904928565025, "data_wait_seconds": 2.0325183868408203e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20767, "cosmopedia_v2": 0.20051, "finemath": 0.08228}}
{"step": 5880, "total_steps": 72479, "tokens": 3082813440, "loss": 3.725268244743347, "l_mix": 2.28797647356987, "l_a": 2.42003133893013, "l_c": 2.365594297647476, "z_loss": 16.04000496864319, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21895503997802734, "tokens_per_second": 1362346.0550072223, "eta_seconds": 25630.09331121445, "data_wait_seconds": 2.8520822525024414e-05, "mix": {"fineweb_hq": 0.50952, "dclm": 0.20765, "cosmopedia_v2": 0.20051, "finemath": 0.08231}}
{"step": 5890, "total_steps": 72479, "tokens": 3088056320, "loss": 3.7704794704914093, "l_mix": 2.3158181607723236, "l_a": 2.451573431491852, "l_c": 2.3920111656188965, "z_loss": 15.857963919639587, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21894976496696472, "tokens_per_second": 1394313.667009334, "eta_seconds": 25038.708619189263, "data_wait_seconds": 4.0858983993530273e-05, "mix": {"fineweb_hq": 0.50951, "dclm": 0.20764, "cosmopedia_v2": 0.20051, "finemath": 0.08234}}
{"step": 5900, "total_steps": 72479, "tokens": 3093299200, "loss": 3.7390436232089996, "l_mix": 2.296259105205536, "l_a": 2.431056886911392, "l_c": 2.3732104897499084, "z_loss": 15.04310393333435, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.23082762956619263, "tokens_per_second": 1336124.434068273, "eta_seconds": 26125.23943276405, "data_wait_seconds": 2.4557113647460938e-05, "mix": {"fineweb_hq": 0.50949, "dclm": 0.20763, "cosmopedia_v2": 0.20051, "finemath": 0.08237}, "gates": {"gate_a2b/L1_absmean": 0.12664814, "gate_a2b/L1_absmax": 0.60669661, "gate_a2b/L1_rms": 0.16998243, "gate_a2b/L3_absmean": 0.2637715, "gate_a2b/L3_absmax": 0.45029733, "gate_a2b/L3_rms": 0.27159968, "gate_a2b/L5_absmean": 0.43446928, "gate_a2b/L5_absmax": 0.63710976, "gate_a2b/L5_rms": 0.44453263, "gate_c2b/L1_absmean": 0.01381047, "gate_c2b/L1_absmax": 0.58450884, "gate_c2b/L1_rms": 0.0533287, "gate_c2b/L3_absmean": 0.24269728, "gate_c2b/L3_absmax": 0.51850963, "gate_c2b/L3_rms": 0.25085044, "gate_c2b/L5_absmean": 0.54778063, "gate_c2b/L5_absmax": 0.77991873, "gate_c2b/L5_rms": 0.56361991, "gate_b2a/L7_absmean": 0.0610728, "gate_b2a/L7_absmax": 0.25498289, "gate_b2a/L7_rms": 0.06831165, "gate_b2a/L11_absmean": 0.11250161, "gate_b2a/L11_absmax": 0.25391099, "gate_b2a/L11_rms": 0.11463643, "gate_b2a/L14_absmean": 0.2740401, "gate_b2a/L14_absmax": 0.43564263, "gate_b2a/L14_rms": 0.27506217, "gate_b2c/L3_absmean": 0.03878672, "gate_b2c/L3_absmax": 0.40575001, "gate_b2c/L3_rms": 0.05841532, "gate_b2c/L5_absmean": 0.08050704, "gate_b2c/L5_absmax": 0.43120533, "gate_b2c/L5_rms": 0.08435205, "gate_b2c/L6_absmean": 0.12711638, "gate_b2c/L6_absmax": 0.34889731, "gate_b2c/L6_rms": 0.12867109}, "streams": {"cos_a2b/round0": 0.345515, "cos_c2b/round0": 0.210414, "cos_b2a/round0": 0.182812, "cos_b2c/round0": 0.080203, "rms_a/round0": 1.37046, "rms_b/round0": 2.375574, "rms_c/round0": 1.370935, "cos_a2b/round1": 0.377523, "cos_c2b/round1": 0.262449, "cos_b2a/round1": 0.199927, "cos_b2c/round1": 0.165511, "rms_a/round1": 2.362011, "rms_b/round1": 4.2079, "rms_c/round1": 2.25971, "cos_a2b/round2": 0.447466, "cos_c2b/round2": 0.430706, "cos_b2a/round2": 0.042944, "cos_b2c/round2": 0.111722, "rms_a/round2": 4.406601, "rms_b/round2": 8.155997, "rms_c/round2": 3.48982}, "alpha_mean": 0.45075367825693746, "alpha_hist": [0.03822, 0.11342, 0.15643, 0.15407, 0.13333, 0.11176, 0.10509, 0.11234, 0.06868, 0.00667], "alpha_by_type": {"special": {"share": 0.00096, "mean": 0.64901, "std": 0.10358}, "whitespace": {"share": 0.04097, "mean": 0.6306, "std": 0.1636}, "digit": {"share": 0.01178, "mean": 0.42465, "std": 0.17257}, "word_start": {"share": 0.5428, "mean": 0.46556, "std": 0.21418}, "word_cont": {"share": 0.27385, "mean": 0.30383, "std": 0.17211}, "punct": {"share": 0.12964, "mean": 0.64317, "std": 0.18767}}, "alpha_p05": 0.11799, "alpha_p50": 0.42537, "alpha_p95": 0.82671}
{"step": 5910, "total_steps": 72479, "tokens": 3098542080, "loss": 4.393085420131683, "l_mix": 2.7120453119277954, "l_a": 2.8276775181293488, "l_c": 2.770216554403305, "z_loss": 16.718012809753418, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21480746567249298, "tokens_per_second": 1368407.0302346926, "eta_seconds": 25505.077875852585, "data_wait_seconds": 1.8358230590820312e-05, "mix": {"fineweb_hq": 0.50964, "dclm": 0.20761, "cosmopedia_v2": 0.20034, "finemath": 0.0824}}
{"step": 5920, "total_steps": 72479, "tokens": 3103784960, "loss": 4.445023357868195, "l_mix": 2.744651675224304, "l_a": 2.8601986169815063, "l_c": 2.8020869195461273, "z_loss": 16.859074592590332, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20914772152900696, "tokens_per_second": 1347242.84402101, "eta_seconds": 25901.852176737782, "data_wait_seconds": 1.811981201171875e-05, "mix": {"fineweb_hq": 0.50963, "dclm": 0.2076, "cosmopedia_v2": 0.20034, "finemath": 0.08243}}
{"step": 5930, "total_steps": 72479, "tokens": 3109027840, "loss": 4.166511565446854, "l_mix": 2.5691685378551483, "l_a": 2.691771775484085, "l_c": 2.6272171437740326, "z_loss": 16.462712287902832, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19739370048046112, "tokens_per_second": 1374705.545157193, "eta_seconds": 25380.593127679826, "data_wait_seconds": 1.856684684753418e-05, "mix": {"fineweb_hq": 0.50961, "dclm": 0.20759, "cosmopedia_v2": 0.20051, "finemath": 0.08229}}
{"step": 5940, "total_steps": 72479, "tokens": 3114270720, "loss": 4.231288909912109, "l_mix": 2.6104190349578857, "l_a": 2.729874908924103, "l_c": 2.6673344373703003, "z_loss": 17.070201635360718, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19291311502456665, "tokens_per_second": 1357829.1952616153, "eta_seconds": 25692.185256981847, "data_wait_seconds": 2.22623348236084e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20758, "cosmopedia_v2": 0.20051, "finemath": 0.08232}}
{"step": 5950, "total_steps": 72479, "tokens": 3119513600, "loss": 5.136714577674866, "l_mix": 3.180008351802826, "l_a": 3.2798998653888702, "l_c": 3.2370992600917816, "z_loss": 16.062960982322693, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17960390448570251, "tokens_per_second": 1415166.9512857578, "eta_seconds": 24647.520435881615, "data_wait_seconds": 1.9878149032592773e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20773, "cosmopedia_v2": 0.20034, "finemath": 0.08235}}
{"step": 5960, "total_steps": 72479, "tokens": 3124756480, "loss": 5.154344797134399, "l_mix": 3.1918977797031403, "l_a": 3.2875075340270996, "l_c": 3.2487156093120575, "z_loss": 15.799985766410828, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1755983829498291, "tokens_per_second": 1389030.6118845693, "eta_seconds": 25107.519714546204, "data_wait_seconds": 2.6434659957885742e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20772, "cosmopedia_v2": 0.20034, "finemath": 0.08238}}
{"step": 5970, "total_steps": 72479, "tokens": 3129999360, "loss": 5.246030747890472, "l_mix": 3.2472030222415924, "l_a": 3.3472396433353424, "l_c": 3.3100039660930634, "z_loss": 16.545485258102417, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.24120409786701202, "tokens_per_second": 1459674.483123381, "eta_seconds": 23888.799177598954, "data_wait_seconds": 2.065300941467285e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20771, "cosmopedia_v2": 0.20034, "finemath": 0.08241}}
{"step": 5980, "total_steps": 72479, "tokens": 3135242240, "loss": 3.70234015583992, "l_mix": 2.2732027769088745, "l_a": 2.411374181509018, "l_c": 2.3471510112285614, "z_loss": 15.797813296318054, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1881462186574936, "tokens_per_second": 1373293.0460535416, "eta_seconds": 25387.609594464302, "data_wait_seconds": 3.555417060852051e-05, "mix": {"fineweb_hq": 0.50953, "dclm": 0.20769, "cosmopedia_v2": 0.2005, "finemath": 0.08227}}
{"step": 5990, "total_steps": 72479, "tokens": 3140485120, "loss": 3.7138992249965668, "l_mix": 2.2805231511592865, "l_a": 2.416565030813217, "l_c": 2.3561906814575195, "z_loss": 15.49324882030487, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20944705605506897, "tokens_per_second": 1397817.302158502, "eta_seconds": 24938.441367244723, "data_wait_seconds": 2.5898218154907227e-05, "mix": {"fineweb_hq": 0.50952, "dclm": 0.20768, "cosmopedia_v2": 0.2005, "finemath": 0.0823}}
{"step": 6000, "total_steps": 72479, "tokens": 3145728000, "loss": 3.7251210510730743, "l_mix": 2.287932872772217, "l_a": 2.423654228448868, "l_c": 2.3618458807468414, "z_loss": 15.38097333908081, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21733540296554565, "tokens_per_second": 1391273.7723574983, "eta_seconds": 25051.96507294178, "data_wait_seconds": 1.7762184143066406e-05, "mix": {"fineweb_hq": 0.5095, "dclm": 0.20767, "cosmopedia_v2": 0.2005, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.12711366, "gate_a2b/L1_absmax": 0.60663474, "gate_a2b/L1_rms": 0.17068444, "gate_a2b/L3_absmean": 0.2648575, "gate_a2b/L3_absmax": 0.44966674, "gate_a2b/L3_rms": 0.27267867, "gate_a2b/L5_absmean": 0.43663332, "gate_a2b/L5_absmax": 0.63682902, "gate_a2b/L5_rms": 0.44664901, "gate_c2b/L1_absmean": 0.01450948, "gate_c2b/L1_absmax": 0.58360642, "gate_c2b/L1_rms": 0.05431764, "gate_c2b/L3_absmean": 0.24127232, "gate_c2b/L3_absmax": 0.51151907, "gate_c2b/L3_rms": 0.24935415, "gate_c2b/L5_absmean": 0.54828364, "gate_c2b/L5_absmax": 0.78139764, "gate_c2b/L5_rms": 0.56383109, "gate_b2a/L7_absmean": 0.06183968, "gate_b2a/L7_absmax": 0.25390115, "gate_b2a/L7_rms": 0.06887369, "gate_b2a/L11_absmean": 0.11304649, "gate_b2a/L11_absmax": 0.25835034, "gate_b2a/L11_rms": 0.11521825, "gate_b2a/L14_absmean": 0.27468061, "gate_b2a/L14_absmax": 0.44376248, "gate_b2a/L14_rms": 0.27575454, "gate_b2c/L3_absmean": 0.03930098, "gate_b2c/L3_absmax": 0.40916675, "gate_b2c/L3_rms": 0.058751, "gate_b2c/L5_absmean": 0.08100016, "gate_b2c/L5_absmax": 0.43832651, "gate_b2c/L5_rms": 0.08488465, "gate_b2c/L6_absmean": 0.12762114, "gate_b2c/L6_absmax": 0.35708076, "gate_b2c/L6_rms": 0.12925169}, "streams": {"cos_a2b/round0": 0.342869, "cos_c2b/round0": 0.203959, "cos_b2a/round0": 0.187883, "cos_b2c/round0": 0.080972, "rms_a/round0": 1.34516, "rms_b/round0": 2.371866, "rms_c/round0": 1.371481, "cos_a2b/round1": 0.385007, "cos_c2b/round1": 0.272124, "cos_b2a/round1": 0.207139, "cos_b2c/round1": 0.175959, "rms_a/round1": 2.346679, "rms_b/round1": 4.1898, "rms_c/round1": 2.263435, "cos_a2b/round2": 0.460994, "cos_c2b/round2": 0.421322, "cos_b2a/round2": 0.03813, "cos_b2c/round2": 0.115805, "rms_a/round2": 4.427123, "rms_b/round2": 8.107166, "rms_c/round2": 3.522585}, "alpha_mean": 0.45014454885317434, "alpha_hist": [0.04233, 0.12218, 0.15388, 0.14549, 0.13211, 0.10835, 0.09996, 0.1111, 0.07423, 0.01036], "alpha_by_type": {"special": {"share": 0.00111, "mean": 0.71422, "std": 0.10286}, "whitespace": {"share": 0.03212, "mean": 0.59472, "std": 0.17829}, "digit": {"share": 0.00992, "mean": 0.36882, "std": 0.1646}, "word_start": {"share": 0.57942, "mean": 0.47538, "std": 0.21737}, "word_cont": {"share": 0.25931, "mean": 0.2895, "std": 0.17779}, "punct": {"share": 0.11812, "mean": 0.64405, "std": 0.2019}}, "alpha_p05": 0.11241, "alpha_p50": 0.44107, "alpha_p95": 0.83411}
{"step": 6010, "total_steps": 72479, "tokens": 3150970880, "loss": 3.728445589542389, "l_mix": 2.2906816601753235, "l_a": 2.4211763739585876, "l_c": 2.366040915250778, "z_loss": 15.986473560333252, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2122560441493988, "tokens_per_second": 1417581.1959888097, "eta_seconds": 24583.353088068965, "data_wait_seconds": 2.8640031814575195e-05, "mix": {"fineweb_hq": 0.50948, "dclm": 0.20765, "cosmopedia_v2": 0.2005, "finemath": 0.08236}}
{"step": 6020, "total_steps": 72479, "tokens": 3156213760, "loss": 4.417076945304871, "l_mix": 2.728492110967636, "l_a": 2.8377148509025574, "l_c": 2.785407543182373, "z_loss": 16.48048734664917, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.24371659755706787, "tokens_per_second": 1398122.6669921924, "eta_seconds": 24921.744718551636, "data_wait_seconds": 1.704692840576172e-05, "mix": {"fineweb_hq": 0.50963, "dclm": 0.20764, "cosmopedia_v2": 0.20033, "finemath": 0.08239}}
{"step": 6030, "total_steps": 72479, "tokens": 3161456640, "loss": 4.2312546372413635, "l_mix": 2.610820859670639, "l_a": 2.7266165912151337, "l_c": 2.6693769097328186, "z_loss": 16.357175588607788, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20834802091121674, "tokens_per_second": 1453789.13705867, "eta_seconds": 23963.8696038723, "data_wait_seconds": 2.0503997802734375e-05, "mix": {"fineweb_hq": 0.50962, "dclm": 0.20763, "cosmopedia_v2": 0.20033, "finemath": 0.08242}}
{"step": 6040, "total_steps": 72479, "tokens": 3166699520, "loss": 4.242478013038635, "l_mix": 2.6171959936618805, "l_a": 2.7381028831005096, "l_c": 2.6739063262939453, "z_loss": 16.793013095855713, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.23369359970092773, "tokens_per_second": 1419058.8464715884, "eta_seconds": 24546.670857667927, "data_wait_seconds": 1.952052116394043e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20762, "cosmopedia_v2": 0.2005, "finemath": 0.08228}}
{"step": 6050, "total_steps": 72479, "tokens": 3171942400, "loss": 4.293307185173035, "l_mix": 2.6481568217277527, "l_a": 2.7718003392219543, "l_c": 2.706401079893112, "z_loss": 16.900007247924805, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17359954118728638, "tokens_per_second": 1320076.79028914, "eta_seconds": 26383.258768129348, "data_wait_seconds": 1.710653305053711e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.2076, "cosmopedia_v2": 0.2005, "finemath": 0.08231}}
{"step": 6060, "total_steps": 72479, "tokens": 3177185280, "loss": 3.7407122254371643, "l_mix": 2.2975879311561584, "l_a": 2.432386130094528, "l_c": 2.37274831533432, "z_loss": 15.839002013206482, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21491865813732147, "tokens_per_second": 1343461.9490891346, "eta_seconds": 25920.112360167503, "data_wait_seconds": 1.773238182067871e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20759, "cosmopedia_v2": 0.2005, "finemath": 0.08234}}
{"step": 6070, "total_steps": 72479, "tokens": 3182428160, "loss": 3.7540346682071686, "l_mix": 2.305269479751587, "l_a": 2.4400643706321716, "l_c": 2.3838351666927338, "z_loss": 15.953058362007141, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2790178954601288, "tokens_per_second": 1455918.3218941118, "eta_seconds": 23914.419695401193, "data_wait_seconds": 1.6123056411743164e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20758, "cosmopedia_v2": 0.20049, "finemath": 0.08237}}
{"step": 6080, "total_steps": 72479, "tokens": 3187671040, "loss": 3.742765188217163, "l_mix": 2.2989769876003265, "l_a": 2.4339925944805145, "l_c": 2.3733420372009277, "z_loss": 15.878817915916443, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18964742124080658, "tokens_per_second": 1412739.6777150221, "eta_seconds": 24641.62326657772, "data_wait_seconds": 3.0338764190673828e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20757, "cosmopedia_v2": 0.20049, "finemath": 0.0824}}
{"step": 6090, "total_steps": 72479, "tokens": 3192913920, "loss": 5.156926333904266, "l_mix": 3.1926221549510956, "l_a": 3.2873500287532806, "l_c": 3.2550257742404938, "z_loss": 15.914817810058594, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19211281836032867, "tokens_per_second": 1458595.7244905648, "eta_seconds": 23863.33337440491, "data_wait_seconds": 3.483891487121582e-05, "mix": {"fineweb_hq": 0.50952, "dclm": 0.20772, "cosmopedia_v2": 0.20033, "finemath": 0.08243}}
{"step": 6100, "total_steps": 72479, "tokens": 3198156800, "loss": 3.70579731464386, "l_mix": 2.277192085981369, "l_a": 2.4075560569763184, "l_c": 2.349247097969055, "z_loss": 15.642859697341919, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.272952139377594, "tokens_per_second": 1401539.0989913358, "eta_seconds": 24831.0683426857, "data_wait_seconds": 2.574920654296875e-05, "mix": {"fineweb_hq": 0.50951, "dclm": 0.2077, "cosmopedia_v2": 0.20049, "finemath": 0.0823}, "gates": {"gate_a2b/L1_absmean": 0.12891035, "gate_a2b/L1_absmax": 0.61252099, "gate_a2b/L1_rms": 0.17269693, "gate_a2b/L3_absmean": 0.26539406, "gate_a2b/L3_absmax": 0.45511302, "gate_a2b/L3_rms": 0.27322328, "gate_a2b/L5_absmean": 0.43796739, "gate_a2b/L5_absmax": 0.63956791, "gate_a2b/L5_rms": 0.44788796, "gate_c2b/L1_absmean": 0.01464675, "gate_c2b/L1_absmax": 0.59556663, "gate_c2b/L1_rms": 0.05561214, "gate_c2b/L3_absmean": 0.24094422, "gate_c2b/L3_absmax": 0.51135272, "gate_c2b/L3_rms": 0.24900389, "gate_c2b/L5_absmean": 0.54876554, "gate_c2b/L5_absmax": 0.78151649, "gate_c2b/L5_rms": 0.56448972, "gate_b2a/L7_absmean": 0.06201748, "gate_b2a/L7_absmax": 0.25385556, "gate_b2a/L7_rms": 0.0688085, "gate_b2a/L11_absmean": 0.11290635, "gate_b2a/L11_absmax": 0.26330763, "gate_b2a/L11_rms": 0.11505682, "gate_b2a/L14_absmean": 0.27584538, "gate_b2a/L14_absmax": 0.44166112, "gate_b2a/L14_rms": 0.27686599, "gate_b2c/L3_absmean": 0.03991568, "gate_b2c/L3_absmax": 0.41243479, "gate_b2c/L3_rms": 0.0598083, "gate_b2c/L5_absmean": 0.08211102, "gate_b2c/L5_absmax": 0.44131532, "gate_b2c/L5_rms": 0.08600722, "gate_b2c/L6_absmean": 0.12866609, "gate_b2c/L6_absmax": 0.35257149, "gate_b2c/L6_rms": 0.13026223}, "streams": {"cos_a2b/round0": 0.349721, "cos_c2b/round0": 0.215085, "cos_b2a/round0": 0.188102, "cos_b2c/round0": 0.08659, "rms_a/round0": 1.329038, "rms_b/round0": 2.340854, "rms_c/round0": 1.359644, "cos_a2b/round1": 0.383354, "cos_c2b/round1": 0.268732, "cos_b2a/round1": 0.204779, "cos_b2c/round1": 0.174219, "rms_a/round1": 2.318645, "rms_b/round1": 4.130388, "rms_c/round1": 2.241741, "cos_a2b/round2": 0.466215, "cos_c2b/round2": 0.422073, "cos_b2a/round2": 0.046763, "cos_b2c/round2": 0.12286, "rms_a/round2": 4.392918, "rms_b/round2": 8.026253, "rms_c/round2": 3.487175}, "alpha_mean": 0.4728945654260741, "alpha_hist": [0.02728, 0.0874, 0.14587, 0.14914, 0.14395, 0.12703, 0.12256, 0.12595, 0.06525, 0.00557], "alpha_by_type": {"special": {"share": 0.00105, "mean": 0.73096, "std": 0.10309}, "whitespace": {"share": 0.03979, "mean": 0.62357, "std": 0.17545}, "digit": {"share": 0.0128, "mean": 0.41683, "std": 0.16582}, "word_start": {"share": 0.55486, "mean": 0.49589, "std": 0.2022}, "word_cont": {"share": 0.2628, "mean": 0.32372, "std": 0.17986}, "punct": {"share": 0.12869, "mean": 0.63527, "std": 0.17915}}, "alpha_p05": 0.11086, "alpha_p50": 0.44637, "alpha_p95": 0.81078}
{"step": 6110, "total_steps": 72479, "tokens": 3203399680, "loss": 3.705153226852417, "l_mix": 2.2762274742126465, "l_a": 2.4022263884544373, "l_c": 2.355540007352829, "z_loss": 15.958000421524048, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.24403269588947296, "tokens_per_second": 1283639.0610632726, "eta_seconds": 27107.674834370613, "data_wait_seconds": 2.8878450393676758e-05, "mix": {"fineweb_hq": 0.50949, "dclm": 0.20769, "cosmopedia_v2": 0.20049, "finemath": 0.08232}}
{"step": 6120, "total_steps": 72479, "tokens": 3208642560, "loss": 3.7477966845035553, "l_mix": 2.303792804479599, "l_a": 2.4325783252716064, "l_c": 2.375605434179306, "z_loss": 15.486539125442505, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.3182488977909088, "tokens_per_second": 1305038.174812733, "eta_seconds": 26659.16450834274, "data_wait_seconds": 2.962350845336914e-05, "mix": {"fineweb_hq": 0.50948, "dclm": 0.20768, "cosmopedia_v2": 0.20049, "finemath": 0.08235}}
{"step": 6130, "total_steps": 72479, "tokens": 3213885440, "loss": 4.371295511722565, "l_mix": 2.6977517306804657, "l_a": 2.817281186580658, "l_c": 2.7555729746818542, "z_loss": 16.875282764434814, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1901136040687561, "tokens_per_second": 1415951.2435977014, "eta_seconds": 24567.21915340424, "data_wait_seconds": 2.0772218704223633e-05, "mix": {"fineweb_hq": 0.50962, "dclm": 0.20767, "cosmopedia_v2": 0.20033, "finemath": 0.08238}}
{"step": 6140, "total_steps": 72479, "tokens": 3219128320, "loss": 4.135223716497421, "l_mix": 2.549353927373886, "l_a": 2.670221358537674, "l_c": 2.6105567514896393, "z_loss": 16.36226725578308, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.24509364366531372, "tokens_per_second": 1434195.6735828647, "eta_seconds": 24251.043475198745, "data_wait_seconds": 1.6629695892333984e-05, "mix": {"fineweb_hq": 0.50961, "dclm": 0.20765, "cosmopedia_v2": 0.20033, "finemath": 0.08241}}
{"step": 6150, "total_steps": 72479, "tokens": 3224371200, "loss": 4.36169159412384, "l_mix": 2.6907364428043365, "l_a": 2.812015265226364, "l_c": 2.752219080924988, "z_loss": 16.84842038154602, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20651523768901825, "tokens_per_second": 1383962.6639596082, "eta_seconds": 25127.483318448067, "data_wait_seconds": 2.777576446533203e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20764, "cosmopedia_v2": 0.20049, "finemath": 0.08228}}
{"step": 6160, "total_steps": 72479, "tokens": 3229614080, "loss": 4.352103233337402, "l_mix": 2.6870697140693665, "l_a": 2.8035617768764496, "l_c": 2.7409965991973877, "z_loss": 16.66083073616028, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17730069160461426, "tokens_per_second": 1404380.5195846425, "eta_seconds": 24758.429348111153, "data_wait_seconds": 3.230571746826172e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20763, "cosmopedia_v2": 0.20049, "finemath": 0.08231}}
{"step": 6170, "total_steps": 72479, "tokens": 3234856960, "loss": 3.681882232427597, "l_mix": 2.261618822813034, "l_a": 2.3933387398719788, "l_c": 2.33575177192688, "z_loss": 15.361989378929138, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1870664656162262, "tokens_per_second": 1463504.9098120478, "eta_seconds": 23754.626826953885, "data_wait_seconds": 2.1070241928100586e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20762, "cosmopedia_v2": 0.20049, "finemath": 0.08233}}
{"step": 6180, "total_steps": 72479, "tokens": 3240099840, "loss": 3.7144861817359924, "l_mix": 2.2816483080387115, "l_a": 2.414211720228195, "l_c": 2.3564790189266205, "z_loss": 16.306325674057007, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1949881613254547, "tokens_per_second": 1415709.8583558316, "eta_seconds": 24552.891192245486, "data_wait_seconds": 2.0176172256469727e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20761, "cosmopedia_v2": 0.20049, "finemath": 0.08236}}
{"step": 6190, "total_steps": 72479, "tokens": 3245342720, "loss": 3.6605941355228424, "l_mix": 2.247909665107727, "l_a": 2.3797853589057922, "l_c": 2.324004352092743, "z_loss": 15.474308729171753, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19032827019691467, "tokens_per_second": 1407755.0900480002, "eta_seconds": 24687.907348155975, "data_wait_seconds": 2.8014183044433594e-05, "mix": {"fineweb_hq": 0.50953, "dclm": 0.20759, "cosmopedia_v2": 0.20048, "finemath": 0.08239}}
{"step": 6200, "total_steps": 72479, "tokens": 3250585600, "loss": 3.655126065015793, "l_mix": 2.2438091933727264, "l_a": 2.3759531378746033, "l_c": 2.3230695724487305, "z_loss": 16.1002334356308, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2078043520450592, "tokens_per_second": 1425594.8525302873, "eta_seconds": 24375.287474083903, "data_wait_seconds": 1.9878149032592773e-05, "mix": {"fineweb_hq": 0.50952, "dclm": 0.20758, "cosmopedia_v2": 0.20048, "finemath": 0.08242}, "gates": {"gate_a2b/L1_absmean": 0.13110088, "gate_a2b/L1_absmax": 0.61065203, "gate_a2b/L1_rms": 0.17518955, "gate_a2b/L3_absmean": 0.26701567, "gate_a2b/L3_absmax": 0.45554635, "gate_a2b/L3_rms": 0.2746591, "gate_a2b/L5_absmean": 0.43964338, "gate_a2b/L5_absmax": 0.63666475, "gate_a2b/L5_rms": 0.4495067, "gate_c2b/L1_absmean": 0.01480578, "gate_c2b/L1_absmax": 0.59465969, "gate_c2b/L1_rms": 0.05676365, "gate_c2b/L3_absmean": 0.23953699, "gate_c2b/L3_absmax": 0.50162065, "gate_c2b/L3_rms": 0.24750543, "gate_c2b/L5_absmean": 0.54832774, "gate_c2b/L5_absmax": 0.78572273, "gate_c2b/L5_rms": 0.56389469, "gate_b2a/L7_absmean": 0.0623958, "gate_b2a/L7_absmax": 0.25436324, "gate_b2a/L7_rms": 0.06918895, "gate_b2a/L11_absmean": 0.11306883, "gate_b2a/L11_absmax": 0.26769412, "gate_b2a/L11_rms": 0.11520965, "gate_b2a/L14_absmean": 0.27559876, "gate_b2a/L14_absmax": 0.43955341, "gate_b2a/L14_rms": 0.27664173, "gate_b2c/L3_absmean": 0.04090331, "gate_b2c/L3_absmax": 0.41307968, "gate_b2c/L3_rms": 0.06080738, "gate_b2c/L5_absmean": 0.083381, "gate_b2c/L5_absmax": 0.44153985, "gate_b2c/L5_rms": 0.08729862, "gate_b2c/L6_absmean": 0.12985502, "gate_b2c/L6_absmax": 0.35368147, "gate_b2c/L6_rms": 0.13144003}, "streams": {"cos_a2b/round0": 0.351367, "cos_c2b/round0": 0.214143, "cos_b2a/round0": 0.186916, "cos_b2c/round0": 0.083458, "rms_a/round0": 1.312854, "rms_b/round0": 2.300906, "rms_c/round0": 1.347602, "cos_a2b/round1": 0.380872, "cos_c2b/round1": 0.259285, "cos_b2a/round1": 0.197464, "cos_b2c/round1": 0.16967, "rms_a/round1": 2.315038, "rms_b/round1": 4.130198, "rms_c/round1": 2.246939, "cos_a2b/round2": 0.464396, "cos_c2b/round2": 0.425528, "cos_b2a/round2": 0.035745, "cos_b2c/round2": 0.1082, "rms_a/round2": 4.366295, "rms_b/round2": 8.03838, "rms_c/round2": 3.481905}, "alpha_mean": 0.45437129093807016, "alpha_hist": [0.0388, 0.1088, 0.15489, 0.15775, 0.13509, 0.10814, 0.09872, 0.1116, 0.07722, 0.00899], "alpha_by_type": {"special": {"share": 0.00101, "mean": 0.72817, "std": 0.12786}, "whitespace": {"share": 0.0415, "mean": 0.63908, "std": 0.19103}, "digit": {"share": 0.01079, "mean": 0.41017, "std": 0.19524}, "word_start": {"share": 0.54881, "mean": 0.47231, "std": 0.21556}, "word_cont": {"share": 0.25568, "mean": 0.29997, "std": 0.17537}, "punct": {"share": 0.14221, "mean": 0.61025, "std": 0.2059}}, "alpha_p05": 0.12169, "alpha_p50": 0.4187, "alpha_p95": 0.83031}
{"step": 6210, "total_steps": 72479, "tokens": 3255828480, "loss": 5.1464842557907104, "l_mix": 3.1861500442028046, "l_a": 3.2812660932540894, "l_c": 3.2477757334709167, "z_loss": 16.215062618255615, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2147412896156311, "tokens_per_second": 1429502.205370295, "eta_seconds": 24304.993263721466, "data_wait_seconds": 1.7076730728149414e-05, "mix": {"fineweb_hq": 0.5095, "dclm": 0.20773, "cosmopedia_v2": 0.20048, "finemath": 0.08229}}
{"step": 6220, "total_steps": 72479, "tokens": 3261071360, "loss": 5.126329183578491, "l_mix": 3.174076199531555, "l_a": 3.2689013481140137, "l_c": 3.2332187592983246, "z_loss": 16.169410705566406, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16885630786418915, "tokens_per_second": 1423006.1852888472, "eta_seconds": 24412.26113500595, "data_wait_seconds": 1.7523765563964844e-05, "mix": {"fineweb_hq": 0.50949, "dclm": 0.20772, "cosmopedia_v2": 0.20048, "finemath": 0.08232}}
{"step": 6230, "total_steps": 72479, "tokens": 3266314240, "loss": 3.666737735271454, "l_mix": 2.2522329688072205, "l_a": 2.3855761289596558, "l_c": 2.3242008984088898, "z_loss": 15.71655786037445, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2361394464969635, "tokens_per_second": 1362764.4723882917, "eta_seconds": 25487.570607948302, "data_wait_seconds": 2.378225326538086e-05, "mix": {"fineweb_hq": 0.50947, "dclm": 0.2077, "cosmopedia_v2": 0.20048, "finemath": 0.08234}}
{"step": 6240, "total_steps": 72479, "tokens": 3271557120, "loss": 4.171649992465973, "l_mix": 2.5739955008029938, "l_a": 2.689960390329361, "l_c": 2.6298789978027344, "z_loss": 17.02631950378418, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.199284166097641, "tokens_per_second": 1436618.271338586, "eta_seconds": 24173.65386815071, "data_wait_seconds": 2.1159648895263672e-05, "mix": {"fineweb_hq": 0.50962, "dclm": 0.20769, "cosmopedia_v2": 0.20032, "finemath": 0.08237}}
{"step": 6250, "total_steps": 72479, "tokens": 3276800000, "loss": 4.305690973997116, "l_mix": 2.656393885612488, "l_a": 2.7747832238674164, "l_c": 2.7171761989593506, "z_loss": 17.092833757400513, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17472311854362488, "tokens_per_second": 1448589.9325395047, "eta_seconds": 23970.2549161911, "data_wait_seconds": 1.7195940017700195e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20768, "cosmopedia_v2": 0.20032, "finemath": 0.0824}}
{"step": 6260, "total_steps": 72479, "tokens": 3282042880, "loss": 4.207045167684555, "l_mix": 2.5962323248386383, "l_a": 2.7090511322021484, "l_c": 2.654954582452774, "z_loss": 16.11120116710663, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2090178281068802, "tokens_per_second": 1382538.1280380231, "eta_seconds": 25111.659756731988, "data_wait_seconds": 2.8312206268310547e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20048, "finemath": 0.08227}}
{"step": 6270, "total_steps": 72479, "tokens": 3287285760, "loss": 4.123301595449448, "l_mix": 2.542760282754898, "l_a": 2.6624662280082703, "l_c": 2.600364327430725, "z_loss": 16.92001485824585, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1876344531774521, "tokens_per_second": 1378890.145975831, "eta_seconds": 25174.292740654946, "data_wait_seconds": 3.49879264831543e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20048, "finemath": 0.0823}}
{"step": 6280, "total_steps": 72479, "tokens": 3292528640, "loss": 3.6869240403175354, "l_mix": 2.265575736761093, "l_a": 2.3949945867061615, "l_c": 2.3374518752098083, "z_loss": 16.14314913749695, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22050215303897858, "tokens_per_second": 1298910.4185701942, "eta_seconds": 26720.350238013267, "data_wait_seconds": 1.800060272216797e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20764, "cosmopedia_v2": 0.20048, "finemath": 0.08232}}
{"step": 6290, "total_steps": 72479, "tokens": 3297771520, "loss": 3.7059215903282166, "l_mix": 2.276608854532242, "l_a": 2.406783640384674, "l_c": 2.3523886501789093, "z_loss": 15.61038851737976, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21401192247867584, "tokens_per_second": 1350431.4460339465, "eta_seconds": 25697.045587849614, "data_wait_seconds": 2.6285648345947266e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20763, "cosmopedia_v2": 0.20048, "finemath": 0.08235}}
{"step": 6300, "total_steps": 72479, "tokens": 3303014400, "loss": 3.7094627618789673, "l_mix": 2.278406411409378, "l_a": 2.4126511812210083, "l_c": 2.3522654473781586, "z_loss": 15.813274264335632, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18726538121700287, "tokens_per_second": 1410710.9327670876, "eta_seconds": 24595.297836065292, "data_wait_seconds": 1.806020736694336e-05, "mix": {"fineweb_hq": 0.50952, "dclm": 0.20762, "cosmopedia_v2": 0.20048, "finemath": 0.08238}, "gates": {"gate_a2b/L1_absmean": 0.13267986, "gate_a2b/L1_absmax": 0.60787868, "gate_a2b/L1_rms": 0.17622948, "gate_a2b/L3_absmean": 0.26837847, "gate_a2b/L3_absmax": 0.45723042, "gate_a2b/L3_rms": 0.27599591, "gate_a2b/L5_absmean": 0.44197932, "gate_a2b/L5_absmax": 0.63751835, "gate_a2b/L5_rms": 0.45176324, "gate_c2b/L1_absmean": 0.01556095, "gate_c2b/L1_absmax": 0.59524012, "gate_c2b/L1_rms": 0.0581414, "gate_c2b/L3_absmean": 0.23877029, "gate_c2b/L3_absmax": 0.50648993, "gate_c2b/L3_rms": 0.24662088, "gate_c2b/L5_absmean": 0.54907888, "gate_c2b/L5_absmax": 0.78811753, "gate_c2b/L5_rms": 0.56453091, "gate_b2a/L7_absmean": 0.06284568, "gate_b2a/L7_absmax": 0.25303495, "gate_b2a/L7_rms": 0.06941883, "gate_b2a/L11_absmean": 0.11387248, "gate_b2a/L11_absmax": 0.2702722, "gate_b2a/L11_rms": 0.11602614, "gate_b2a/L14_absmean": 0.27648532, "gate_b2a/L14_absmax": 0.44326693, "gate_b2a/L14_rms": 0.27752748, "gate_b2c/L3_absmean": 0.04176464, "gate_b2c/L3_absmax": 0.41766515, "gate_b2c/L3_rms": 0.06181128, "gate_b2c/L5_absmean": 0.08439326, "gate_b2c/L5_absmax": 0.44737616, "gate_b2c/L5_rms": 0.08831178, "gate_b2c/L6_absmean": 0.13072938, "gate_b2c/L6_absmax": 0.3548274, "gate_b2c/L6_rms": 0.13239196}, "streams": {"cos_a2b/round0": 0.347204, "cos_c2b/round0": 0.20798, "cos_b2a/round0": 0.186586, "cos_b2c/round0": 0.080559, "rms_a/round0": 1.296803, "rms_b/round0": 2.291053, "rms_c/round0": 1.36031, "cos_a2b/round1": 0.38079, "cos_c2b/round1": 0.258474, "cos_b2a/round1": 0.203127, "cos_b2c/round1": 0.170898, "rms_a/round1": 2.277394, "rms_b/round1": 4.11562, "rms_c/round1": 2.225723, "cos_a2b/round2": 0.458862, "cos_c2b/round2": 0.420308, "cos_b2a/round2": 0.036066, "cos_b2c/round2": 0.113896, "rms_a/round2": 4.317659, "rms_b/round2": 8.035463, "rms_c/round2": 3.47427}, "alpha_mean": 0.4618092449339528, "alpha_hist": [0.04463, 0.11584, 0.15414, 0.1451, 0.11629, 0.10027, 0.09406, 0.12112, 0.09637, 0.01218], "alpha_by_type": {"special": {"share": 0.00104, "mean": 0.76127, "std": 0.10675}, "whitespace": {"share": 0.03432, "mean": 0.70418, "std": 0.17197}, "digit": {"share": 0.01233, "mean": 0.44078, "std": 0.17039}, "word_start": {"share": 0.55569, "mean": 0.48671, "std": 0.2252}, "word_cont": {"share": 0.27618, "mean": 0.29271, "std": 0.17936}, "punct": {"share": 0.12044, "mean": 0.6652, "std": 0.19114}}, "alpha_p05": 0.1101, "alpha_p50": 0.4371, "alpha_p95": 0.84922}
{"step": 6310, "total_steps": 72479, "tokens": 3308257280, "loss": 3.670340657234192, "l_mix": 2.25373438000679, "l_a": 2.3885720670223236, "l_c": 2.328240603208542, "z_loss": 15.624784588813782, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18532943725585938, "tokens_per_second": 1415930.5476038994, "eta_seconds": 24500.928192210198, "data_wait_seconds": 2.1278858184814453e-05, "mix": {"fineweb_hq": 0.50951, "dclm": 0.20761, "cosmopedia_v2": 0.20048, "finemath": 0.08241}}
{"step": 6320, "total_steps": 72479, "tokens": 3313500160, "loss": 4.336607038974762, "l_mix": 2.6768089830875397, "l_a": 2.795111119747162, "l_c": 2.7320151329040527, "z_loss": 16.60201930999756, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20029513537883759, "tokens_per_second": 1417224.0692964327, "eta_seconds": 24474.866426181794, "data_wait_seconds": 2.6881694793701172e-05, "mix": {"fineweb_hq": 0.50965, "dclm": 0.20759, "cosmopedia_v2": 0.20047, "finemath": 0.08228}}
{"step": 6330, "total_steps": 72479, "tokens": 3318743040, "loss": 4.313315063714981, "l_mix": 2.661646842956543, "l_a": 2.7794247567653656, "l_c": 2.7205251157283783, "z_loss": 16.83147096633911, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17546378076076508, "tokens_per_second": 1322889.5280727244, "eta_seconds": 26216.192793154718, "data_wait_seconds": 2.0325183868408203e-05, "mix": {"fineweb_hq": 0.50964, "dclm": 0.20758, "cosmopedia_v2": 0.20047, "finemath": 0.08231}}
{"step": 6340, "total_steps": 72479, "tokens": 3323985920, "loss": 4.168076574802399, "l_mix": 2.5701319575309753, "l_a": 2.688681483268738, "l_c": 2.6321796476840973, "z_loss": 16.861944675445557, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.23887991905212402, "tokens_per_second": 1416499.771102757, "eta_seconds": 24479.9785636425, "data_wait_seconds": 3.123283386230469e-05, "mix": {"fineweb_hq": 0.50962, "dclm": 0.20757, "cosmopedia_v2": 0.20047, "finemath": 0.08233}}
{"step": 6350, "total_steps": 72479, "tokens": 3329228800, "loss": 4.308302342891693, "l_mix": 2.6606553196907043, "l_a": 2.772867053747177, "l_c": 2.7139966785907745, "z_loss": 15.877593278884888, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22268307209014893, "tokens_per_second": 1421855.1473728546, "eta_seconds": 24384.088081026075, "data_wait_seconds": 1.7762184143066406e-05, "mix": {"fineweb_hq": 0.50961, "dclm": 0.20756, "cosmopedia_v2": 0.20047, "finemath": 0.08236}}
{"step": 6360, "total_steps": 72479, "tokens": 3334471680, "loss": 4.360971361398697, "l_mix": 2.69095242023468, "l_a": 2.809155374765396, "l_c": 2.75199893116951, "z_loss": 16.725100994110107, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20693907141685486, "tokens_per_second": 1354025.9992113605, "eta_seconds": 25601.722782421115, "data_wait_seconds": 1.71661376953125e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20755, "cosmopedia_v2": 0.20047, "finemath": 0.08239}}
{"step": 6370, "total_steps": 72479, "tokens": 3339714560, "loss": 4.291921615600586, "l_mix": 2.648020476102829, "l_a": 2.766249805688858, "l_c": 2.7079570591449738, "z_loss": 16.39042854309082, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.24147643148899078, "tokens_per_second": 1481505.233519877, "eta_seconds": 23395.229802632333, "data_wait_seconds": 2.8580427169799805e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20769, "cosmopedia_v2": 0.20047, "finemath": 0.08226}}
{"step": 6380, "total_steps": 72479, "tokens": 3344957440, "loss": 3.6572015583515167, "l_mix": 2.246447414159775, "l_a": 2.374813586473465, "l_c": 2.3225364685058594, "z_loss": 15.490212082862854, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2149820774793625, "tokens_per_second": 1367124.8044000189, "eta_seconds": 25348.75557847023, "data_wait_seconds": 1.7404556274414062e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.20047, "finemath": 0.08229}}
{"step": 6390, "total_steps": 72479, "tokens": 3350200320, "loss": 3.6816071569919586, "l_mix": 2.261954039335251, "l_a": 2.3938302993774414, "l_c": 2.3331433534622192, "z_loss": 15.609880447387695, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2136138677597046, "tokens_per_second": 1405202.1936463166, "eta_seconds": 24658.13801648617, "data_wait_seconds": 2.5331974029541016e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20047, "finemath": 0.08232}}
{"step": 6400, "total_steps": 72479, "tokens": 3355443200, "loss": 3.6922607719898224, "l_mix": 2.269016534090042, "l_a": 2.396612137556076, "l_c": 2.3423270285129547, "z_loss": 15.624594807624817, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19802457094192505, "tokens_per_second": 1406213.4637293166, "eta_seconds": 24636.67689549923, "data_wait_seconds": 2.5153160095214844e-05, "mix": {"fineweb_hq": 0.50953, "dclm": 0.20766, "cosmopedia_v2": 0.20047, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.13486576, "gate_a2b/L1_absmax": 0.60740256, "gate_a2b/L1_rms": 0.178289, "gate_a2b/L3_absmean": 0.26856655, "gate_a2b/L3_absmax": 0.45519733, "gate_a2b/L3_rms": 0.2760573, "gate_a2b/L5_absmean": 0.44409052, "gate_a2b/L5_absmax": 0.63026047, "gate_a2b/L5_rms": 0.45376879, "gate_c2b/L1_absmean": 0.01539849, "gate_c2b/L1_absmax": 0.5997746, "gate_c2b/L1_rms": 0.05949729, "gate_c2b/L3_absmean": 0.23808268, "gate_c2b/L3_absmax": 0.49943307, "gate_c2b/L3_rms": 0.24581763, "gate_c2b/L5_absmean": 0.54911667, "gate_c2b/L5_absmax": 0.78326529, "gate_c2b/L5_rms": 0.56454527, "gate_b2a/L7_absmean": 0.06371816, "gate_b2a/L7_absmax": 0.25242129, "gate_b2a/L7_rms": 0.07013083, "gate_b2a/L11_absmean": 0.11428748, "gate_b2a/L11_absmax": 0.26746592, "gate_b2a/L11_rms": 0.11641482, "gate_b2a/L14_absmean": 0.27738938, "gate_b2a/L14_absmax": 0.44489074, "gate_b2a/L14_rms": 0.27842012, "gate_b2c/L3_absmean": 0.04225773, "gate_b2c/L3_absmax": 0.42058235, "gate_b2c/L3_rms": 0.06203423, "gate_b2c/L5_absmean": 0.08437379, "gate_b2c/L5_absmax": 0.45517042, "gate_b2c/L5_rms": 0.08843078, "gate_b2c/L6_absmean": 0.13155907, "gate_b2c/L6_absmax": 0.36207387, "gate_b2c/L6_rms": 0.13324711}, "streams": {"cos_a2b/round0": 0.352429, "cos_c2b/round0": 0.210929, "cos_b2a/round0": 0.189434, "cos_b2c/round0": 0.081683, "rms_a/round0": 1.275555, "rms_b/round0": 2.275821, "rms_c/round0": 1.350742, "cos_a2b/round1": 0.383063, "cos_c2b/round1": 0.264914, "cos_b2a/round1": 0.207474, "cos_b2c/round1": 0.176027, "rms_a/round1": 2.248201, "rms_b/round1": 4.028914, "rms_c/round1": 2.234425, "cos_a2b/round2": 0.470131, "cos_c2b/round2": 0.420929, "cos_b2a/round2": 0.039854, "cos_b2c/round2": 0.123506, "rms_a/round2": 4.29489, "rms_b/round2": 7.950621, "rms_c/round2": 3.474958}, "alpha_mean": 0.46875327040662995, "alpha_hist": [0.03453, 0.09857, 0.14662, 0.14507, 0.13042, 0.12265, 0.11955, 0.12126, 0.07436, 0.00697], "alpha_by_type": {"special": {"share": 0.00117, "mean": 0.74552, "std": 0.09228}, "whitespace": {"share": 0.03555, "mean": 0.66894, "std": 0.1556}, "digit": {"share": 0.00934, "mean": 0.46462, "std": 0.1795}, "word_start": {"share": 0.57158, "mean": 0.48369, "std": 0.20893}, "word_cont": {"share": 0.2598, "mean": 0.31512, "std": 0.18053}, "punct": {"share": 0.12256, "mean": 0.66435, "std": 0.17944}}, "alpha_p05": 0.1188, "alpha_p50": 0.44059, "alpha_p95": 0.82671}
{"step": 6410, "total_steps": 72479, "tokens": 3360686080, "loss": 3.6806595623493195, "l_mix": 2.2610314190387726, "l_a": 2.3912473320961, "l_c": 2.3356176018714905, "z_loss": 15.685280084609985, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19402451813220978, "tokens_per_second": 1186650.4560553285, "eta_seconds": 29190.722251224517, "data_wait_seconds": 3.287196159362793e-05, "mix": {"fineweb_hq": 0.50952, "dclm": 0.20764, "cosmopedia_v2": 0.20047, "finemath": 0.08237}}
{"step": 6420, "total_steps": 72479, "tokens": 3365928960, "loss": 3.6806578934192657, "l_mix": 2.2612683475017548, "l_a": 2.394935578107834, "l_c": 2.331246703863144, "z_loss": 15.347784638404846, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2379457652568817, "tokens_per_second": 1407969.159175985, "eta_seconds": 24598.508260130882, "data_wait_seconds": 4.079937934875488e-05, "mix": {"fineweb_hq": 0.5095, "dclm": 0.20763, "cosmopedia_v2": 0.20047, "finemath": 0.0824}}
{"step": 6430, "total_steps": 72479, "tokens": 3371171840, "loss": 4.334753155708313, "l_mix": 2.6761189103126526, "l_a": 2.7886995673179626, "l_c": 2.734550654888153, "z_loss": 16.59071946144104, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20377032458782196, "tokens_per_second": 1436671.6762697473, "eta_seconds": 24103.41811840534, "data_wait_seconds": 2.1725893020629883e-05, "mix": {"fineweb_hq": 0.50964, "dclm": 0.20762, "cosmopedia_v2": 0.20047, "finemath": 0.08227}}
{"step": 6440, "total_steps": 72479, "tokens": 3376414720, "loss": 4.2681997418403625, "l_mix": 2.634921431541443, "l_a": 2.748309701681137, "l_c": 2.6906326711177826, "z_loss": 15.955175280570984, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2569795548915863, "tokens_per_second": 1432282.0157251048, "eta_seconds": 24173.629810237882, "data_wait_seconds": 6.851553916931152e-05, "mix": {"fineweb_hq": 0.50963, "dclm": 0.20761, "cosmopedia_v2": 0.20047, "finemath": 0.0823}}
{"step": 6450, "total_steps": 72479, "tokens": 3381657600, "loss": 4.509938657283783, "l_mix": 2.7852209508419037, "l_a": 2.89687842130661, "l_c": 2.846656858921051, "z_loss": 16.571858644485474, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.174505814909935, "tokens_per_second": 1440574.761339444, "eta_seconds": 24030.833581876755, "data_wait_seconds": 2.1457672119140625e-05, "mix": {"fineweb_hq": 0.50961, "dclm": 0.2076, "cosmopedia_v2": 0.20047, "finemath": 0.08233}}
{"step": 6460, "total_steps": 72479, "tokens": 3386900480, "loss": 4.421097546815872, "l_mix": 2.7291429936885834, "l_a": 2.8462026715278625, "l_c": 2.7882054448127747, "z_loss": 16.320468187332153, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17860689759254456, "tokens_per_second": 1226372.8471720805, "eta_seconds": 28223.855046868324, "data_wait_seconds": 2.9861927032470703e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20759, "cosmopedia_v2": 0.20046, "finemath": 0.08235}}
{"step": 6470, "total_steps": 72479, "tokens": 3392143360, "loss": 4.536481529474258, "l_mix": 2.8029361963272095, "l_a": 2.9061020016670227, "l_c": 2.866859018802643, "z_loss": 16.57016682624817, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20989058911800385, "tokens_per_second": 1299963.2275398702, "eta_seconds": 26622.08119339943, "data_wait_seconds": 2.8908252716064453e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20757, "cosmopedia_v2": 0.20046, "finemath": 0.08238}}
{"step": 6480, "total_steps": 72479, "tokens": 3397386240, "loss": 4.701497554779053, "l_mix": 2.905581295490265, "l_a": 3.0139946937561035, "l_c": 2.9668956995010376, "z_loss": 16.49071741104126, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19839173555374146, "tokens_per_second": 1261906.0671802075, "eta_seconds": 27420.807785892488, "data_wait_seconds": 1.874566078186035e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20756, "cosmopedia_v2": 0.20046, "finemath": 0.08241}}
{"step": 6490, "total_steps": 72479, "tokens": 3402629120, "loss": 5.05280464887619, "l_mix": 3.128175586462021, "l_a": 3.2241677343845367, "l_c": 3.1861594915390015, "z_loss": 15.309086084365845, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22532695531845093, "tokens_per_second": 1365229.103416532, "eta_seconds": 25341.710593056676, "data_wait_seconds": 7.709860801696777e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.2077, "cosmopedia_v2": 0.20046, "finemath": 0.08228}}
{"step": 6500, "total_steps": 72479, "tokens": 3407872000, "loss": 5.137602388858795, "l_mix": 3.1818796396255493, "l_a": 3.272048741579056, "l_c": 3.2415720224380493, "z_loss": 16.364641308784485, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19057390093803406, "tokens_per_second": 1263100.5116691964, "eta_seconds": 27386.57583653927, "data_wait_seconds": 1.659989356994629e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20769, "cosmopedia_v2": 0.20046, "finemath": 0.08231}, "gates": {"gate_a2b/L1_absmean": 0.13726097, "gate_a2b/L1_absmax": 0.60248929, "gate_a2b/L1_rms": 0.18047637, "gate_a2b/L3_absmean": 0.27021107, "gate_a2b/L3_absmax": 0.46579057, "gate_a2b/L3_rms": 0.2775656, "gate_a2b/L5_absmean": 0.44528005, "gate_a2b/L5_absmax": 0.63480949, "gate_a2b/L5_rms": 0.45487255, "gate_c2b/L1_absmean": 0.01570844, "gate_c2b/L1_absmax": 0.59529257, "gate_c2b/L1_rms": 0.06013085, "gate_c2b/L3_absmean": 0.23638868, "gate_c2b/L3_absmax": 0.49843624, "gate_c2b/L3_rms": 0.24397281, "gate_c2b/L5_absmean": 0.54903257, "gate_c2b/L5_absmax": 0.79669166, "gate_c2b/L5_rms": 0.56441808, "gate_b2a/L7_absmean": 0.06404151, "gate_b2a/L7_absmax": 0.25323918, "gate_b2a/L7_rms": 0.07050785, "gate_b2a/L11_absmean": 0.11436592, "gate_b2a/L11_absmax": 0.27130574, "gate_b2a/L11_rms": 0.11648688, "gate_b2a/L14_absmean": 0.27714008, "gate_b2a/L14_absmax": 0.44235319, "gate_b2a/L14_rms": 0.27815488, "gate_b2c/L3_absmean": 0.04262271, "gate_b2c/L3_absmax": 0.42304716, "gate_b2c/L3_rms": 0.06301551, "gate_b2c/L5_absmean": 0.08559569, "gate_b2c/L5_absmax": 0.46004093, "gate_b2c/L5_rms": 0.08968202, "gate_b2c/L6_absmean": 0.13257232, "gate_b2c/L6_absmax": 0.36176029, "gate_b2c/L6_rms": 0.1342693}, "streams": {"cos_a2b/round0": 0.387195, "cos_c2b/round0": 0.258674, "cos_b2a/round0": 0.201517, "cos_b2c/round0": 0.135656, "rms_a/round0": 1.221354, "rms_b/round0": 2.225279, "rms_c/round0": 1.28766, "cos_a2b/round1": 0.391783, "cos_c2b/round1": 0.279526, "cos_b2a/round1": 0.214468, "cos_b2c/round1": 0.187125, "rms_a/round1": 2.168782, "rms_b/round1": 3.85269, "rms_c/round1": 2.155751, "cos_a2b/round2": 0.495011, "cos_c2b/round2": 0.45289, "cos_b2a/round2": 0.107737, "cos_b2c/round2": 0.159012, "rms_a/round2": 4.252115, "rms_b/round2": 7.633737, "rms_c/round2": 3.341859}, "alpha_mean": 0.4842839376407255, "alpha_hist": [0.0067, 0.02711, 0.08241, 0.17091, 0.22479, 0.25623, 0.17145, 0.05415, 0.00618, 6e-05], "alpha_by_type": {"special": {"share": 0.00031, "mean": 0.56069, "std": 0.10245}, "whitespace": {"share": 0.07938, "mean": 0.54344, "std": 0.093}, "digit": {"share": 0.06258, "mean": 0.5578, "std": 0.09394}, "word_start": {"share": 0.54845, "mean": 0.49085, "std": 0.13705}, "word_cont": {"share": 0.20291, "mean": 0.36538, "std": 0.13558}, "punct": {"share": 0.10638, "mean": 0.58963, "std": 0.13096}}, "alpha_p05": 0.23936, "alpha_p50": 0.49948, "alpha_p95": 0.69946}
{"step": 6510, "total_steps": 72479, "tokens": 3413114880, "loss": 3.655789464712143, "l_mix": 2.246786743402481, "l_a": 2.3741454482078552, "l_c": 2.317342698574066, "z_loss": 15.561648488044739, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20163312554359436, "tokens_per_second": 1273428.7598857621, "eta_seconds": 27160.337634515763, "data_wait_seconds": 2.524256706237793e-05, "mix": {"fineweb_hq": 0.50952, "dclm": 0.20768, "cosmopedia_v2": 0.20046, "finemath": 0.08233}}
{"step": 6520, "total_steps": 72479, "tokens": 3418357760, "loss": 3.688696175813675, "l_mix": 2.2656417191028595, "l_a": 2.394134432077408, "l_c": 2.3440884053707123, "z_loss": 15.874633193016052, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22090883553028107, "tokens_per_second": 1453320.458829367, "eta_seconds": 23794.82926969528, "data_wait_seconds": 2.2619962692260742e-05, "mix": {"fineweb_hq": 0.50951, "dclm": 0.20767, "cosmopedia_v2": 0.20046, "finemath": 0.08236}}
{"step": 6530, "total_steps": 72479, "tokens": 3423600640, "loss": 3.696697384119034, "l_mix": 2.27145516872406, "l_a": 2.398675709962845, "l_c": 2.3469382524490356, "z_loss": 15.57939100265503, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.25040122866630554, "tokens_per_second": 1380617.6736229223, "eta_seconds": 25044.05815787315, "data_wait_seconds": 2.238154411315918e-05, "mix": {"fineweb_hq": 0.50949, "dclm": 0.20766, "cosmopedia_v2": 0.20046, "finemath": 0.08239}}
{"step": 6540, "total_steps": 72479, "tokens": 3428843520, "loss": 4.90972638130188, "l_mix": 3.0358946323394775, "l_a": 3.1428242623806, "l_c": 3.0978415310382843, "z_loss": 16.319408178329468, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2692035138607025, "tokens_per_second": 1450130.7712332443, "eta_seconds": 23839.9371407032, "data_wait_seconds": 1.5616416931152344e-05, "mix": {"fineweb_hq": 0.50963, "dclm": 0.20765, "cosmopedia_v2": 0.20046, "finemath": 0.08226}}
{"step": 6550, "total_steps": 72479, "tokens": 3434086400, "loss": 4.900688290596008, "l_mix": 3.031598299741745, "l_a": 3.1369582414627075, "l_c": 3.087833821773529, "z_loss": 16.523148775100708, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.211248978972435, "tokens_per_second": 1446663.5814169575, "eta_seconds": 23893.44972529411, "data_wait_seconds": 1.5974044799804688e-05, "mix": {"fineweb_hq": 0.50962, "dclm": 0.20763, "cosmopedia_v2": 0.20046, "finemath": 0.08229}}
{"step": 6560, "total_steps": 72479, "tokens": 3439329280, "loss": 4.84041041135788, "l_mix": 2.9941325783729553, "l_a": 3.0963845551013947, "l_c": 3.0522930324077606, "z_loss": 16.744444608688354, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.23628199100494385, "tokens_per_second": 1426883.786607147, "eta_seconds": 24220.991924071313, "data_wait_seconds": 1.5139579772949219e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20762, "cosmopedia_v2": 0.20046, "finemath": 0.08232}}
{"step": 6570, "total_steps": 72479, "tokens": 3444572160, "loss": 4.8971893191337585, "l_mix": 3.028600573539734, "l_a": 3.131340831518173, "l_c": 3.092056006193161, "z_loss": 15.69633138179779, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2580673098564148, "tokens_per_second": 1429929.1445102063, "eta_seconds": 24165.74130589962, "data_wait_seconds": 1.621246337890625e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20761, "cosmopedia_v2": 0.20046, "finemath": 0.08234}}
{"step": 6580, "total_steps": 72479, "tokens": 3449815040, "loss": 4.865640044212341, "l_mix": 3.0102338790893555, "l_a": 3.1143059134483337, "l_c": 3.064888060092926, "z_loss": 16.478140115737915, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2584923505783081, "tokens_per_second": 1327140.8326872718, "eta_seconds": 26033.450302362442, "data_wait_seconds": 1.7434358596801758e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.2076, "cosmopedia_v2": 0.20046, "finemath": 0.08237}}
{"step": 6590, "total_steps": 72479, "tokens": 3455057920, "loss": 4.880436360836029, "l_mix": 3.019043445587158, "l_a": 3.120692938566208, "l_c": 3.0784492790699005, "z_loss": 16.50174355506897, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2654811143875122, "tokens_per_second": 1401172.9557695347, "eta_seconds": 24654.209810256958, "data_wait_seconds": 1.7881393432617188e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20759, "cosmopedia_v2": 0.20046, "finemath": 0.0824}}
{"step": 6600, "total_steps": 72479, "tokens": 3460300800, "loss": 4.9946224093437195, "l_mix": 3.091255486011505, "l_a": 3.183382958173752, "l_c": 3.1559221744537354, "z_loss": 15.753806710243225, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.3352218270301819, "tokens_per_second": 1287726.732843448, "eta_seconds": 26822.126365065575, "data_wait_seconds": 1.6957521438598633e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20773, "cosmopedia_v2": 0.20045, "finemath": 0.08227}, "gates": {"gate_a2b/L1_absmean": 0.13900453, "gate_a2b/L1_absmax": 0.60191977, "gate_a2b/L1_rms": 0.18204822, "gate_a2b/L3_absmean": 0.27041891, "gate_a2b/L3_absmax": 0.46263117, "gate_a2b/L3_rms": 0.27781373, "gate_a2b/L5_absmean": 0.44664326, "gate_a2b/L5_absmax": 0.6388942, "gate_a2b/L5_rms": 0.4561049, "gate_c2b/L1_absmean": 0.01523423, "gate_c2b/L1_absmax": 0.59126556, "gate_c2b/L1_rms": 0.06047149, "gate_c2b/L3_absmean": 0.2348953, "gate_c2b/L3_absmax": 0.49883637, "gate_c2b/L3_rms": 0.242401, "gate_c2b/L5_absmean": 0.54889727, "gate_c2b/L5_absmax": 0.78765005, "gate_c2b/L5_rms": 0.56414372, "gate_b2a/L7_absmean": 0.0640319, "gate_b2a/L7_absmax": 0.25452331, "gate_b2a/L7_rms": 0.07039856, "gate_b2a/L11_absmean": 0.11469714, "gate_b2a/L11_absmax": 0.27691531, "gate_b2a/L11_rms": 0.1167643, "gate_b2a/L14_absmean": 0.27687693, "gate_b2a/L14_absmax": 0.44316402, "gate_b2a/L14_rms": 0.277899, "gate_b2c/L3_absmean": 0.04292522, "gate_b2c/L3_absmax": 0.42687809, "gate_b2c/L3_rms": 0.06345018, "gate_b2c/L5_absmean": 0.08558574, "gate_b2c/L5_absmax": 0.46482629, "gate_b2c/L5_rms": 0.08966676, "gate_b2c/L6_absmean": 0.13318995, "gate_b2c/L6_absmax": 0.36502722, "gate_b2c/L6_rms": 0.13492784}, "streams": {"cos_a2b/round0": 0.376796, "cos_c2b/round0": 0.243705, "cos_b2a/round0": 0.199779, "cos_b2c/round0": 0.124437, "rms_a/round0": 1.238597, "rms_b/round0": 2.243849, "rms_c/round0": 1.315336, "cos_a2b/round1": 0.404029, "cos_c2b/round1": 0.287247, "cos_b2a/round1": 0.227823, "cos_b2c/round1": 0.184817, "rms_a/round1": 2.183573, "rms_b/round1": 3.900548, "rms_c/round1": 2.176155, "cos_a2b/round2": 0.498906, "cos_c2b/round2": 0.443859, "cos_b2a/round2": 0.100519, "cos_b2c/round2": 0.162664, "rms_a/round2": 4.248679, "rms_b/round2": 7.662586, "rms_c/round2": 3.352}, "alpha_mean": 0.49990692440245255, "alpha_hist": [0.00433, 0.02881, 0.09616, 0.16501, 0.20135, 0.20984, 0.16373, 0.10181, 0.02846, 0.0005], "alpha_by_type": {"special": {"share": 0.00044, "mean": 0.66341, "std": 0.11942}, "whitespace": {"share": 0.05048, "mean": 0.59038, "std": 0.10888}, "digit": {"share": 0.02097, "mean": 0.51969, "std": 0.11907}, "word_start": {"share": 0.58887, "mean": 0.51443, "std": 0.1573}, "word_cont": {"share": 0.244, "mean": 0.38809, "std": 0.13956}, "punct": {"share": 0.09525, "mean": 0.6435, "std": 0.15019}}, "alpha_p05": 0.23687, "alpha_p50": 0.50346, "alpha_p95": 0.77662}
{"step": 6610, "total_steps": 72479, "tokens": 3465543680, "loss": 5.070447683334351, "l_mix": 3.138676643371582, "l_a": 3.238186776638031, "l_c": 3.1956022679805756, "z_loss": 16.34286618232727, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1865421086549759, "tokens_per_second": 1315538.9019416408, "eta_seconds": 26251.08708000183, "data_wait_seconds": 1.8835067749023438e-05, "mix": {"fineweb_hq": 0.50953, "dclm": 0.20772, "cosmopedia_v2": 0.20045, "finemath": 0.0823}}
{"step": 6620, "total_steps": 72479, "tokens": 3470786560, "loss": 5.137084543704987, "l_mix": 3.1805518865585327, "l_a": 3.277288317680359, "l_c": 3.239096909761429, "z_loss": 16.169909596443176, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17921051383018494, "tokens_per_second": 1352519.4640512248, "eta_seconds": 25529.453963327407, "data_wait_seconds": 5.5283308029174805e-05, "mix": {"fineweb_hq": 0.50952, "dclm": 0.2077, "cosmopedia_v2": 0.20045, "finemath": 0.08233}}
{"step": 6630, "total_steps": 72479, "tokens": 3476029440, "loss": 5.127735137939453, "l_mix": 3.174856662750244, "l_a": 3.2725782990455627, "l_c": 3.2316695153713226, "z_loss": 16.04101324081421, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17419806122779846, "tokens_per_second": 1342438.9015461814, "eta_seconds": 25717.25273473263, "data_wait_seconds": 3.784894943237305e-05, "mix": {"fineweb_hq": 0.5095, "dclm": 0.20769, "cosmopedia_v2": 0.20045, "finemath": 0.08235}}
{"step": 6640, "total_steps": 72479, "tokens": 3481272320, "loss": 5.155084908008575, "l_mix": 3.1918625831604004, "l_a": 3.2888310849666595, "l_c": 3.2498550415039062, "z_loss": 16.163888812065125, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16602995991706848, "tokens_per_second": 1407253.2068120062, "eta_seconds": 24529.059493279456, "data_wait_seconds": 2.384185791015625e-05, "mix": {"fineweb_hq": 0.50949, "dclm": 0.20768, "cosmopedia_v2": 0.20045, "finemath": 0.08238}}
{"step": 6650, "total_steps": 72479, "tokens": 3486515200, "loss": 4.923877000808716, "l_mix": 3.047147274017334, "l_a": 3.1447290778160095, "l_c": 3.10553315281868, "z_loss": 16.509865045547485, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2372845560312271, "tokens_per_second": 1425197.0986897398, "eta_seconds": 24216.548562812804, "data_wait_seconds": 2.333521842956543e-05, "mix": {"fineweb_hq": 0.50962, "dclm": 0.20767, "cosmopedia_v2": 0.20045, "finemath": 0.08226}}
{"step": 6660, "total_steps": 72479, "tokens": 3491758080, "loss": 4.953403949737549, "l_mix": 3.0650500059127808, "l_a": 3.1655631065368652, "l_c": 3.123384028673172, "z_loss": 16.698388814926147, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22388605773448944, "tokens_per_second": 1403658.9827429007, "eta_seconds": 24584.39855852127, "data_wait_seconds": 2.008676528930664e-05, "mix": {"fineweb_hq": 0.50961, "dclm": 0.20766, "cosmopedia_v2": 0.20045, "finemath": 0.08228}}
{"step": 6670, "total_steps": 72479, "tokens": 3497000960, "loss": 4.955086052417755, "l_mix": 3.066149741411209, "l_a": 3.1638863682746887, "l_c": 3.1272022426128387, "z_loss": 16.096206426620483, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21727509796619415, "tokens_per_second": 1384606.8951510433, "eta_seconds": 24918.891501140595, "data_wait_seconds": 1.8209218978881836e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20765, "cosmopedia_v2": 0.20045, "finemath": 0.08231}}
{"step": 6680, "total_steps": 72479, "tokens": 3502243840, "loss": 4.943222284317017, "l_mix": 3.0578876435756683, "l_a": 3.159076005220413, "l_c": 3.1199189722537994, "z_loss": 16.36149573326111, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.23225784301757812, "tokens_per_second": 1421275.090379601, "eta_seconds": 24272.30755362511, "data_wait_seconds": 2.0891427993774414e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20763, "cosmopedia_v2": 0.20045, "finemath": 0.08234}}
{"step": 6690, "total_steps": 72479, "tokens": 3507486720, "loss": 4.9541924595832825, "l_mix": 3.065073013305664, "l_a": 3.162982791662216, "l_c": 3.128477394580841, "z_loss": 16.813383102416992, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.251463919878006, "tokens_per_second": 1456650.599884635, "eta_seconds": 23679.242801761626, "data_wait_seconds": 1.6748905181884766e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20762, "cosmopedia_v2": 0.20045, "finemath": 0.08236}}
{"step": 6700, "total_steps": 72479, "tokens": 3512729600, "loss": 4.993231773376465, "l_mix": 3.088960111141205, "l_a": 3.1890400052070618, "l_c": 3.1529831290245056, "z_loss": 16.647295713424683, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.28326302766799927, "tokens_per_second": 1432881.2706231887, "eta_seconds": 24068.38658516407, "data_wait_seconds": 2.282857894897461e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20761, "cosmopedia_v2": 0.20045, "finemath": 0.08239}, "gates": {"gate_a2b/L1_absmean": 0.14067732, "gate_a2b/L1_absmax": 0.59875304, "gate_a2b/L1_rms": 0.18388109, "gate_a2b/L3_absmean": 0.2708073, "gate_a2b/L3_absmax": 0.45821437, "gate_a2b/L3_rms": 0.27809632, "gate_a2b/L5_absmean": 0.44814578, "gate_a2b/L5_absmax": 0.6403932, "gate_a2b/L5_rms": 0.4574824, "gate_c2b/L1_absmean": 0.01548846, "gate_c2b/L1_absmax": 0.58945024, "gate_c2b/L1_rms": 0.06124283, "gate_c2b/L3_absmean": 0.23366776, "gate_c2b/L3_absmax": 0.48645598, "gate_c2b/L3_rms": 0.24116701, "gate_c2b/L5_absmean": 0.54899931, "gate_c2b/L5_absmax": 0.79313415, "gate_c2b/L5_rms": 0.56411076, "gate_b2a/L7_absmean": 0.06470622, "gate_b2a/L7_absmax": 0.25304136, "gate_b2a/L7_rms": 0.07096985, "gate_b2a/L11_absmean": 0.11464642, "gate_b2a/L11_absmax": 0.27858233, "gate_b2a/L11_rms": 0.11674916, "gate_b2a/L14_absmean": 0.27727893, "gate_b2a/L14_absmax": 0.44229347, "gate_b2a/L14_rms": 0.278312, "gate_b2c/L3_absmean": 0.04398733, "gate_b2c/L3_absmax": 0.42873517, "gate_b2c/L3_rms": 0.06447891, "gate_b2c/L5_absmean": 0.08640881, "gate_b2c/L5_absmax": 0.4707678, "gate_b2c/L5_rms": 0.09057354, "gate_b2c/L6_absmean": 0.13419691, "gate_b2c/L6_absmax": 0.36550969, "gate_b2c/L6_rms": 0.13591725}, "streams": {"cos_a2b/round0": 0.370973, "cos_c2b/round0": 0.241292, "cos_b2a/round0": 0.196933, "cos_b2c/round0": 0.115084, "rms_a/round0": 1.243196, "rms_b/round0": 2.245598, "rms_c/round0": 1.344837, "cos_a2b/round1": 0.39312, "cos_c2b/round1": 0.270395, "cos_b2a/round1": 0.214338, "cos_b2c/round1": 0.170298, "rms_a/round1": 2.199044, "rms_b/round1": 3.953372, "rms_c/round1": 2.199951, "cos_a2b/round2": 0.483567, "cos_c2b/round2": 0.439834, "cos_b2a/round2": 0.081198, "cos_b2c/round2": 0.148481, "rms_a/round2": 4.254648, "rms_b/round2": 7.957761, "rms_c/round2": 3.430631}, "alpha_mean": 0.44909918576890817, "alpha_hist": [0.01521, 0.05275, 0.13463, 0.20721, 0.20029, 0.17984, 0.13327, 0.06682, 0.00984, 0.00014], "alpha_by_type": {"special": {"share": 0.0013, "mean": 0.59815, "std": 0.08112}, "whitespace": {"share": 0.02803, "mean": 0.54496, "std": 0.11961}, "digit": {"share": 0.03731, "mean": 0.46969, "std": 0.1346}, "word_start": {"share": 0.55013, "mean": 0.47385, "std": 0.15636}, "word_cont": {"share": 0.27362, "mean": 0.3365, "std": 0.14381}, "punct": {"share": 0.10962, "mean": 0.57266, "std": 0.15241}}, "alpha_p05": 0.17982, "alpha_p50": 0.44384, "alpha_p95": 0.72875}
{"step": 6710, "total_steps": 72479, "tokens": 3517972480, "loss": 3.291186898946762, "l_mix": 2.0323984920978546, "l_a": 2.111044630408287, "l_c": 2.0789085179567337, "z_loss": 18.023943662643433, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.36218130588531494, "tokens_per_second": 1476892.422555475, "eta_seconds": 23347.602672600748, "data_wait_seconds": 2.8371810913085938e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.2076, "cosmopedia_v2": 0.20045, "finemath": 0.08241}}
{"step": 6720, "total_steps": 72479, "tokens": 3523215360, "loss": 4.993541181087494, "l_mix": 3.0914231836795807, "l_a": 3.183985471725464, "l_c": 3.1509852409362793, "z_loss": 16.26676297187805, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1635296791791916, "tokens_per_second": 1420105.9401725188, "eta_seconds": 24277.522976779936, "data_wait_seconds": 2.4884939193725586e-05, "mix": {"fineweb_hq": 0.50952, "dclm": 0.20774, "cosmopedia_v2": 0.20045, "finemath": 0.08229}}
{"step": 6730, "total_steps": 72479, "tokens": 3528458240, "loss": 5.024451732635498, "l_mix": 3.1100109219551086, "l_a": 3.207657217979431, "l_c": 3.168525457382202, "z_loss": 15.859006524085999, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1893797218799591, "tokens_per_second": 1471623.9607239163, "eta_seconds": 23424.062554025648, "data_wait_seconds": 2.0205974578857422e-05, "mix": {"fineweb_hq": 0.50951, "dclm": 0.20773, "cosmopedia_v2": 0.20045, "finemath": 0.08232}}
{"step": 6740, "total_steps": 72479, "tokens": 3533701120, "loss": 5.1192702651023865, "l_mix": 3.1697234213352203, "l_a": 3.2690809071063995, "l_c": 3.2240031957626343, "z_loss": 16.214776873588562, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16697876155376434, "tokens_per_second": 1474688.4413626755, "eta_seconds": 23371.83086628914, "data_wait_seconds": 1.7940998077392578e-05, "mix": {"fineweb_hq": 0.5095, "dclm": 0.20772, "cosmopedia_v2": 0.20045, "finemath": 0.08234}}
{"step": 6750, "total_steps": 72479, "tokens": 3538944000, "loss": 5.122520446777344, "l_mix": 3.171101689338684, "l_a": 3.2682918310165405, "l_c": 3.2308489382267, "z_loss": 16.763872623443604, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20340105891227722, "tokens_per_second": 1452120.645286799, "eta_seconds": 23731.44825387001, "data_wait_seconds": 2.1666288375854492e-05, "mix": {"fineweb_hq": 0.50948, "dclm": 0.2077, "cosmopedia_v2": 0.20044, "finemath": 0.08237}}
{"step": 6760, "total_steps": 72479, "tokens": 3544186880, "loss": 5.070201098918915, "l_mix": 3.1390788555145264, "l_a": 3.2365468740463257, "l_c": 3.195119619369507, "z_loss": 16.222150921821594, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.23435477912425995, "tokens_per_second": 1461937.866603802, "eta_seconds": 23568.500316667556, "data_wait_seconds": 1.996755599975586e-05, "mix": {"fineweb_hq": 0.50947, "dclm": 0.20769, "cosmopedia_v2": 0.20044, "finemath": 0.0824}}
{"step": 6770, "total_steps": 72479, "tokens": 3549429760, "loss": 4.857304215431213, "l_mix": 3.003675699234009, "l_a": 3.1081772446632385, "l_c": 3.0651626586914062, "z_loss": 16.264631152153015, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18851017951965332, "tokens_per_second": 1466163.8794859373, "eta_seconds": 23496.991485071183, "data_wait_seconds": 1.8805265426635742e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20768, "cosmopedia_v2": 0.20044, "finemath": 0.08227}}
{"step": 6780, "total_steps": 72479, "tokens": 3554672640, "loss": 4.8906238079071045, "l_mix": 3.0252407789230347, "l_a": 3.127105623483658, "l_c": 3.0853698551654816, "z_loss": 16.402873992919922, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1977808177471161, "tokens_per_second": 1389613.3570017985, "eta_seconds": 24787.612423586845, "data_wait_seconds": 1.9311904907226562e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.0823}}
{"step": 6790, "total_steps": 72479, "tokens": 3559915520, "loss": 4.911247551441193, "l_mix": 3.037863701581955, "l_a": 3.140557289123535, "l_c": 3.098643034696579, "z_loss": 16.23628556728363, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19595685601234436, "tokens_per_second": 1394315.1699455024, "eta_seconds": 24700.265172719955, "data_wait_seconds": 2.0831823348999023e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 6800, "total_steps": 72479, "tokens": 3565158400, "loss": 5.074643492698669, "l_mix": 3.142233908176422, "l_a": 3.2348784804344177, "l_c": 3.201050817966461, "z_loss": 16.307605981826782, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.28997015953063965, "tokens_per_second": 1317344.642020621, "eta_seconds": 26139.485791039468, "data_wait_seconds": 4.0978193283081055e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.14212547, "gate_a2b/L1_absmax": 0.59783351, "gate_a2b/L1_rms": 0.18515421, "gate_a2b/L3_absmean": 0.2718536, "gate_a2b/L3_absmax": 0.46960184, "gate_a2b/L3_rms": 0.27914703, "gate_a2b/L5_absmean": 0.44950867, "gate_a2b/L5_absmax": 0.64504814, "gate_a2b/L5_rms": 0.4588441, "gate_c2b/L1_absmean": 0.01530963, "gate_c2b/L1_absmax": 0.58929485, "gate_c2b/L1_rms": 0.06173317, "gate_c2b/L3_absmean": 0.23256195, "gate_c2b/L3_absmax": 0.48592198, "gate_c2b/L3_rms": 0.23997474, "gate_c2b/L5_absmean": 0.54948968, "gate_c2b/L5_absmax": 0.78847629, "gate_c2b/L5_rms": 0.56463963, "gate_b2a/L7_absmean": 0.06546149, "gate_b2a/L7_absmax": 0.25287583, "gate_b2a/L7_rms": 0.07144422, "gate_b2a/L11_absmean": 0.11512218, "gate_b2a/L11_absmax": 0.28058973, "gate_b2a/L11_rms": 0.11718171, "gate_b2a/L14_absmean": 0.27747434, "gate_b2a/L14_absmax": 0.44119677, "gate_b2a/L14_rms": 0.27853003, "gate_b2c/L3_absmean": 0.04448158, "gate_b2c/L3_absmax": 0.43025982, "gate_b2c/L3_rms": 0.06505519, "gate_b2c/L5_absmean": 0.08720595, "gate_b2c/L5_absmax": 0.47874415, "gate_b2c/L5_rms": 0.09140025, "gate_b2c/L6_absmean": 0.13516246, "gate_b2c/L6_absmax": 0.36607134, "gate_b2c/L6_rms": 0.13690287}, "streams": {"cos_a2b/round0": 0.36418, "cos_c2b/round0": 0.238691, "cos_b2a/round0": 0.200113, "cos_b2c/round0": 0.11678, "rms_a/round0": 1.217757, "rms_b/round0": 2.223539, "rms_c/round0": 1.337794, "cos_a2b/round1": 0.405718, "cos_c2b/round1": 0.28768, "cos_b2a/round1": 0.218906, "cos_b2c/round1": 0.175872, "rms_a/round1": 2.154382, "rms_b/round1": 3.842325, "rms_c/round1": 2.17926, "cos_a2b/round2": 0.50051, "cos_c2b/round2": 0.451197, "cos_b2a/round2": 0.097352, "cos_b2c/round2": 0.155337, "rms_a/round2": 4.17186, "rms_b/round2": 7.566329, "rms_c/round2": 3.358917}, "alpha_mean": 0.4768631911320611, "alpha_hist": [0.00603, 0.03397, 0.10919, 0.18718, 0.21272, 0.20859, 0.14757, 0.08374, 0.01089, 0.00012], "alpha_by_type": {"special": {"share": 0.00076, "mean": 0.56643, "std": 0.10002}, "whitespace": {"share": 0.01839, "mean": 0.5436, "std": 0.10706}, "digit": {"share": 0.0182, "mean": 0.55643, "std": 0.11293}, "word_start": {"share": 0.61668, "mean": 0.48454, "std": 0.15379}, "word_cont": {"share": 0.24202, "mean": 0.39106, "std": 0.14072}, "punct": {"share": 0.10394, "mean": 0.60474, "std": 0.15053}}, "alpha_p05": 0.23371, "alpha_p50": 0.47975, "alpha_p95": 0.74354}
{"step": 6810, "total_steps": 72479, "tokens": 3570401280, "loss": 5.074070572853088, "l_mix": 3.1406008303165436, "l_a": 3.2370639741420746, "l_c": 3.202204167842865, "z_loss": 16.890647172927856, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2968430519104004, "tokens_per_second": 1402615.4167120594, "eta_seconds": 24546.620735645294, "data_wait_seconds": 2.98917293548584e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08238}}
{"step": 6820, "total_steps": 72479, "tokens": 3575644160, "loss": 3.464038699865341, "l_mix": 2.1391665041446686, "l_a": 2.2288345396518707, "l_c": 2.181349977850914, "z_loss": 18.167686223983765, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.5253872275352478, "tokens_per_second": 1447969.555660441, "eta_seconds": 23774.13645019531, "data_wait_seconds": 3.4868717193603516e-05, "mix": {"fineweb_hq": 0.50953, "dclm": 0.20762, "cosmopedia_v2": 0.20044, "finemath": 0.0824}}
{"step": 6830, "total_steps": 72479, "tokens": 3580887040, "loss": 3.436814457178116, "l_mix": 2.1221850514411926, "l_a": 2.2059097290039062, "l_c": 2.1701096296310425, "z_loss": 18.235594272613525, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.35923025012016296, "tokens_per_second": 1387821.9160903597, "eta_seconds": 24800.720116138455, "data_wait_seconds": 2.6494264602661133e-05, "mix": {"fineweb_hq": 0.50952, "dclm": 0.20761, "cosmopedia_v2": 0.20044, "finemath": 0.08243}}
{"step": 6840, "total_steps": 72479, "tokens": 3586129920, "loss": 4.923413157463074, "l_mix": 3.0450290739536285, "l_a": 3.149288296699524, "l_c": 3.106407880783081, "z_loss": 16.75263237953186, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1737169772386551, "tokens_per_second": 1449909.9044823435, "eta_seconds": 23735.088591098785, "data_wait_seconds": 2.7000904083251953e-05, "mix": {"fineweb_hq": 0.50965, "dclm": 0.2076, "cosmopedia_v2": 0.20044, "finemath": 0.08231}}
{"step": 6850, "total_steps": 72479, "tokens": 3591372800, "loss": 4.957089006900787, "l_mix": 3.0667306184768677, "l_a": 3.1690391302108765, "l_c": 3.1268596053123474, "z_loss": 15.88795030117035, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1851959526538849, "tokens_per_second": 1454395.080924094, "eta_seconds": 23658.28762989044, "data_wait_seconds": 1.8417835235595703e-05, "mix": {"fineweb_hq": 0.50964, "dclm": 0.20759, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 6860, "total_steps": 72479, "tokens": 3596615680, "loss": 4.9868486523628235, "l_mix": 3.086015671491623, "l_a": 3.18882018327713, "l_c": 3.141919046640396, "z_loss": 16.111796379089355, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18834401667118073, "tokens_per_second": 1477838.3086934348, "eta_seconds": 23279.444083714483, "data_wait_seconds": 2.110004425048828e-05, "mix": {"fineweb_hq": 0.50962, "dclm": 0.20758, "cosmopedia_v2": 0.20044, "finemath": 0.08236}}
{"step": 6870, "total_steps": 72479, "tokens": 3601858560, "loss": 4.88107043504715, "l_mix": 3.0189991295337677, "l_a": 3.1230127215385437, "l_c": 3.0784507989883423, "z_loss": 16.322186946868896, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1848929226398468, "tokens_per_second": 1455625.0579543144, "eta_seconds": 23631.093188476563, "data_wait_seconds": 1.901388168334961e-05, "mix": {"fineweb_hq": 0.50961, "dclm": 0.20757, "cosmopedia_v2": 0.20044, "finemath": 0.08239}}
{"step": 6880, "total_steps": 72479, "tokens": 3607101440, "loss": 4.926010727882385, "l_mix": 3.0477040708065033, "l_a": 3.1494077146053314, "l_c": 3.1061330139636993, "z_loss": 16.44403338432312, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15382638573646545, "tokens_per_second": 1452515.4378841876, "eta_seconds": 23678.074335718156, "data_wait_seconds": 1.6033649444580078e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.2077, "cosmopedia_v2": 0.20044, "finemath": 0.08227}}
{"step": 6890, "total_steps": 72479, "tokens": 3612344320, "loss": 4.858258664608002, "l_mix": 3.00545397400856, "l_a": 3.1064536571502686, "l_c": 3.0640613436698914, "z_loss": 16.50166368484497, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1524851769208908, "tokens_per_second": 1462958.3109536564, "eta_seconds": 23505.47201142311, "data_wait_seconds": 2.0295381546020508e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20769, "cosmopedia_v2": 0.20044, "finemath": 0.08229}}
{"step": 6900, "total_steps": 72479, "tokens": 3617587200, "loss": 4.97878623008728, "l_mix": 3.080360561609268, "l_a": 3.1814494132995605, "l_c": 3.141180157661438, "z_loss": 16.367075204849243, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18844357132911682, "tokens_per_second": 1388900.6822417693, "eta_seconds": 24755.033381152156, "data_wait_seconds": 2.1517276763916016e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20768, "cosmopedia_v2": 0.20043, "finemath": 0.08232}, "gates": {"gate_a2b/L1_absmean": 0.14410658, "gate_a2b/L1_absmax": 0.59669572, "gate_a2b/L1_rms": 0.18693103, "gate_a2b/L3_absmean": 0.27300748, "gate_a2b/L3_absmax": 0.4614757, "gate_a2b/L3_rms": 0.2801879, "gate_a2b/L5_absmean": 0.45153338, "gate_a2b/L5_absmax": 0.64714622, "gate_a2b/L5_rms": 0.46079043, "gate_c2b/L1_absmean": 0.01566907, "gate_c2b/L1_absmax": 0.58861792, "gate_c2b/L1_rms": 0.06234924, "gate_c2b/L3_absmean": 0.23146684, "gate_c2b/L3_absmax": 0.47989979, "gate_c2b/L3_rms": 0.23873371, "gate_c2b/L5_absmean": 0.54836529, "gate_c2b/L5_absmax": 0.78726387, "gate_c2b/L5_rms": 0.5634197, "gate_b2a/L7_absmean": 0.06605946, "gate_b2a/L7_absmax": 0.25321743, "gate_b2a/L7_rms": 0.07202177, "gate_b2a/L11_absmean": 0.11586218, "gate_b2a/L11_absmax": 0.28056335, "gate_b2a/L11_rms": 0.11797617, "gate_b2a/L14_absmean": 0.27831891, "gate_b2a/L14_absmax": 0.44181582, "gate_b2a/L14_rms": 0.27934328, "gate_b2c/L3_absmean": 0.04534892, "gate_b2c/L3_absmax": 0.42967507, "gate_b2c/L3_rms": 0.06576808, "gate_b2c/L5_absmean": 0.08772777, "gate_b2c/L5_absmax": 0.47914237, "gate_b2c/L5_rms": 0.09193061, "gate_b2c/L6_absmean": 0.13574718, "gate_b2c/L6_absmax": 0.36576125, "gate_b2c/L6_rms": 0.13748766}, "streams": {"cos_a2b/round0": 0.370834, "cos_c2b/round0": 0.242324, "cos_b2a/round0": 0.195438, "cos_b2c/round0": 0.118345, "rms_a/round0": 1.218593, "rms_b/round0": 2.211499, "rms_c/round0": 1.333897, "cos_a2b/round1": 0.401466, "cos_c2b/round1": 0.271861, "cos_b2a/round1": 0.213009, "cos_b2c/round1": 0.165103, "rms_a/round1": 2.151772, "rms_b/round1": 3.902568, "rms_c/round1": 2.185969, "cos_a2b/round2": 0.485192, "cos_c2b/round2": 0.453581, "cos_b2a/round2": 0.080053, "cos_b2c/round2": 0.132152, "rms_a/round2": 4.202011, "rms_b/round2": 7.899158, "rms_c/round2": 3.440122}, "alpha_mean": 0.43247983740738505, "alpha_hist": [0.01338, 0.0677, 0.1709, 0.20464, 0.19066, 0.15923, 0.12125, 0.06419, 0.00784, 0.00021], "alpha_by_type": {"special": {"share": 0.00119, "mean": 0.53203, "std": 0.12243}, "whitespace": {"share": 0.0251, "mean": 0.51845, "std": 0.13889}, "digit": {"share": 0.03694, "mean": 0.55977, "std": 0.13161}, "word_start": {"share": 0.55988, "mean": 0.44639, "std": 0.16349}, "word_cont": {"share": 0.26683, "mean": 0.3279, "std": 0.13591}, "punct": {"share": 0.11006, "mean": 0.55186, "std": 0.16536}}, "alpha_p05": 0.18922, "alpha_p50": 0.42131, "alpha_p95": 0.72584}
{"step": 6910, "total_steps": 72479, "tokens": 3622830080, "loss": 4.948133051395416, "l_mix": 3.060731142759323, "l_a": 3.167649745941162, "l_c": 3.118283212184906, "z_loss": 16.22046422958374, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18927116692066193, "tokens_per_second": 1400373.9994194782, "eta_seconds": 24548.470541620252, "data_wait_seconds": 2.0384788513183594e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 6920, "total_steps": 72479, "tokens": 3628072960, "loss": 4.899118006229401, "l_mix": 3.029398947954178, "l_a": 3.1335234344005585, "l_c": 3.0932594537734985, "z_loss": 16.840360641479492, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20955397188663483, "tokens_per_second": 1473065.5956420763, "eta_seconds": 23333.51419901848, "data_wait_seconds": 2.3812055587768555e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08237}}
{"step": 6930, "total_steps": 72479, "tokens": 3633315840, "loss": 3.7128865718841553, "l_mix": 2.2808956503868103, "l_a": 2.4149319231510162, "l_c": 2.3532543182373047, "z_loss": 15.350419402122498, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20368200540542603, "tokens_per_second": 1391474.493385567, "eta_seconds": 24697.940404486657, "data_wait_seconds": 2.8789043426513672e-05, "mix": {"fineweb_hq": 0.50952, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.0824}}
{"step": 6940, "total_steps": 72479, "tokens": 3638558720, "loss": 3.3139811158180237, "l_mix": 2.046523466706276, "l_a": 2.1298781633377075, "l_c": 2.089042514562607, "z_loss": 17.813421726226807, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2652691602706909, "tokens_per_second": 1407568.6550846135, "eta_seconds": 24411.81899573803, "data_wait_seconds": 2.0742416381835938e-05, "mix": {"fineweb_hq": 0.50951, "dclm": 0.20764, "cosmopedia_v2": 0.20043, "finemath": 0.08242}}
{"step": 6950, "total_steps": 72479, "tokens": 3643801600, "loss": 4.914965927600861, "l_mix": 3.0396891236305237, "l_a": 3.146028161048889, "l_c": 3.0995407700538635, "z_loss": 16.061267614364624, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19751279056072235, "tokens_per_second": 1481599.5605186932, "eta_seconds": 23188.497936630247, "data_wait_seconds": 1.4632940292358398e-05, "mix": {"fineweb_hq": 0.50964, "dclm": 0.20763, "cosmopedia_v2": 0.20043, "finemath": 0.0823}}
{"step": 6960, "total_steps": 72479, "tokens": 3649044480, "loss": 5.017641723155975, "l_mix": 3.1046562492847443, "l_a": 3.208768904209137, "l_c": 3.1624066531658173, "z_loss": 16.3274188041687, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1940666288137436, "tokens_per_second": 1273974.6165715868, "eta_seconds": 26963.508554387096, "data_wait_seconds": 2.530217170715332e-05, "mix": {"fineweb_hq": 0.50963, "dclm": 0.20761, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 6970, "total_steps": 72479, "tokens": 3654287360, "loss": 4.85617733001709, "l_mix": 3.003890335559845, "l_a": 3.0980779826641083, "l_c": 3.0707825124263763, "z_loss": 16.28829789161682, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1972644329071045, "tokens_per_second": 1463752.5422709824, "eta_seconds": 23464.063494443893, "data_wait_seconds": 1.5795230865478516e-05, "mix": {"fineweb_hq": 0.50961, "dclm": 0.2076, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 6980, "total_steps": 72479, "tokens": 3659530240, "loss": 4.836050450801849, "l_mix": 2.9912711083889008, "l_a": 3.0948410630226135, "l_c": 3.0489898025989532, "z_loss": 16.299692392349243, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1905229538679123, "tokens_per_second": 1456300.7157996625, "eta_seconds": 23580.527935910224, "data_wait_seconds": 1.5974044799804688e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20759, "cosmopedia_v2": 0.20043, "finemath": 0.08238}}
{"step": 6990, "total_steps": 72479, "tokens": 3664773120, "loss": 4.990507543087006, "l_mix": 3.087905377149582, "l_a": 3.190723329782486, "l_c": 3.145909070968628, "z_loss": 16.122488617897034, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.207240492105484, "tokens_per_second": 1456512.729152102, "eta_seconds": 23573.495888352394, "data_wait_seconds": 2.03549861907959e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20758, "cosmopedia_v2": 0.20043, "finemath": 0.0824}}
{"step": 7000, "total_steps": 72479, "tokens": 3670016000, "loss": 5.09334260225296, "l_mix": 3.154059946537018, "l_a": 3.2478200793266296, "l_c": 3.211064577102661, "z_loss": 16.17154335975647, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20665796101093292, "tokens_per_second": 1432980.3392172009, "eta_seconds": 23956.96089644432, "data_wait_seconds": 1.8388032913208008e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20771, "cosmopedia_v2": 0.20043, "finemath": 0.08229}, "gates": {"gate_a2b/L1_absmean": 0.14518254, "gate_a2b/L1_absmax": 0.59751683, "gate_a2b/L1_rms": 0.18756649, "gate_a2b/L3_absmean": 0.27212307, "gate_a2b/L3_absmax": 0.45816115, "gate_a2b/L3_rms": 0.2794047, "gate_a2b/L5_absmean": 0.45256129, "gate_a2b/L5_absmax": 0.64276272, "gate_a2b/L5_rms": 0.46175852, "gate_c2b/L1_absmean": 0.01574244, "gate_c2b/L1_absmax": 0.59004933, "gate_c2b/L1_rms": 0.06327923, "gate_c2b/L3_absmean": 0.23066311, "gate_c2b/L3_absmax": 0.47274357, "gate_c2b/L3_rms": 0.23780435, "gate_c2b/L5_absmean": 0.54806423, "gate_c2b/L5_absmax": 0.78879625, "gate_c2b/L5_rms": 0.56318939, "gate_b2a/L7_absmean": 0.06653431, "gate_b2a/L7_absmax": 0.24897115, "gate_b2a/L7_rms": 0.07243533, "gate_b2a/L11_absmean": 0.11576242, "gate_b2a/L11_absmax": 0.27140686, "gate_b2a/L11_rms": 0.11783942, "gate_b2a/L14_absmean": 0.2784043, "gate_b2a/L14_absmax": 0.44174874, "gate_b2a/L14_rms": 0.2794489, "gate_b2c/L3_absmean": 0.0455596, "gate_b2c/L3_absmax": 0.43110844, "gate_b2c/L3_rms": 0.06620803, "gate_b2c/L5_absmean": 0.0881571, "gate_b2c/L5_absmax": 0.4706119, "gate_b2c/L5_rms": 0.09226915, "gate_b2c/L6_absmean": 0.13642573, "gate_b2c/L6_absmax": 0.37618998, "gate_b2c/L6_rms": 0.13822913}, "streams": {"cos_a2b/round0": 0.377805, "cos_c2b/round0": 0.247727, "cos_b2a/round0": 0.197919, "cos_b2c/round0": 0.123979, "rms_a/round0": 1.19572, "rms_b/round0": 2.167295, "rms_c/round0": 1.288159, "cos_a2b/round1": 0.391291, "cos_c2b/round1": 0.274311, "cos_b2a/round1": 0.222261, "cos_b2c/round1": 0.181386, "rms_a/round1": 2.090207, "rms_b/round1": 3.755151, "rms_c/round1": 2.134646, "cos_a2b/round2": 0.504922, "cos_c2b/round2": 0.438196, "cos_b2a/round2": 0.106684, "cos_b2c/round2": 0.170476, "rms_a/round2": 4.129152, "rms_b/round2": 7.490121, "rms_c/round2": 3.238894}, "alpha_mean": 0.4990967232023138, "alpha_hist": [0.00609, 0.03127, 0.09241, 0.15323, 0.19133, 0.22774, 0.19159, 0.09042, 0.01553, 0.0004], "alpha_by_type": {"special": {"share": 0.00031, "mean": 0.6097, "std": 0.10125}, "whitespace": {"share": 0.08601, "mean": 0.55574, "std": 0.12091}, "digit": {"share": 0.02773, "mean": 0.49913, "std": 0.11187}, "word_start": {"share": 0.56215, "mean": 0.52134, "std": 0.14868}, "word_cont": {"share": 0.21785, "mean": 0.36739, "std": 0.14271}, "punct": {"share": 0.10596, "mean": 0.60558, "std": 0.15044}}, "alpha_p05": 0.21735, "alpha_p50": 0.51336, "alpha_p95": 0.76065}
{"step": 7010, "total_steps": 72479, "tokens": 3675258880, "loss": 5.113502502441406, "l_mix": 3.1660510301589966, "l_a": 3.2592644095420837, "l_c": 3.226820796728134, "z_loss": 16.25706696510315, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1929931938648224, "tokens_per_second": 1399621.9186697789, "eta_seconds": 24524.20229644775, "data_wait_seconds": 1.8805265426635742e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.2077, "cosmopedia_v2": 0.20043, "finemath": 0.08231}}
{"step": 7020, "total_steps": 72479, "tokens": 3680501760, "loss": 5.017448008060455, "l_mix": 3.1066857874393463, "l_a": 3.198084682226181, "l_c": 3.165624678134918, "z_loss": 16.493233919143677, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1933804452419281, "tokens_per_second": 1465573.877439598, "eta_seconds": 23417.016856193546, "data_wait_seconds": 1.6421079635620117e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20769, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 7030, "total_steps": 72479, "tokens": 3685744640, "loss": 5.037409782409668, "l_mix": 3.1188018321990967, "l_a": 3.2122384905815125, "l_c": 3.1776730120182037, "z_loss": 16.34479832649231, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20062962174415588, "tokens_per_second": 1400701.4488042665, "eta_seconds": 24497.81524913311, "data_wait_seconds": 1.6689300537109375e-05, "mix": {"fineweb_hq": 0.50953, "dclm": 0.20768, "cosmopedia_v2": 0.20043, "finemath": 0.08236}}
{"step": 7040, "total_steps": 72479, "tokens": 3690987520, "loss": 4.933774888515472, "l_mix": 3.0547634661197662, "l_a": 3.146305501461029, "l_c": 3.1115714609622955, "z_loss": 16.48281478881836, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1912158876657486, "tokens_per_second": 1361705.671919096, "eta_seconds": 25195.519956707954, "data_wait_seconds": 3.4302473068237305e-05, "mix": {"fineweb_hq": 0.50952, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08239}}
{"step": 7050, "total_steps": 72479, "tokens": 3696230400, "loss": 3.5413065552711487, "l_mix": 2.1885393112897873, "l_a": 2.273064211010933, "l_c": 2.229898363351822, "z_loss": 18.784794807434082, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.23987546563148499, "tokens_per_second": 1389356.5518443119, "eta_seconds": 24690.306823301315, "data_wait_seconds": 3.534555435180664e-05, "mix": {"fineweb_hq": 0.5095, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08241}}
{"step": 7060, "total_steps": 72479, "tokens": 3701473280, "loss": 5.028423845767975, "l_mix": 3.112180173397064, "l_a": 3.2102080285549164, "l_c": 3.171835482120514, "z_loss": 16.304754853248596, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16869239509105682, "tokens_per_second": 1462239.5138633668, "eta_seconds": 23456.072925686836, "data_wait_seconds": 2.5004148483276367e-05, "mix": {"fineweb_hq": 0.50963, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08229}}
{"step": 7070, "total_steps": 72479, "tokens": 3706716160, "loss": 4.886153399944305, "l_mix": 3.021859735250473, "l_a": 3.1273953020572662, "l_c": 3.081335186958313, "z_loss": 16.743651151657104, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2187211811542511, "tokens_per_second": 1251598.8482862527, "eta_seconds": 27399.476948189735, "data_wait_seconds": 2.3990869522094727e-05, "mix": {"fineweb_hq": 0.50962, "dclm": 0.20764, "cosmopedia_v2": 0.20042, "finemath": 0.08232}}
{"step": 7080, "total_steps": 72479, "tokens": 3711959040, "loss": 4.868889391422272, "l_mix": 3.0130134224891663, "l_a": 3.110288441181183, "l_c": 3.0705010890960693, "z_loss": 16.390911102294922, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.214521124958992, "tokens_per_second": 1470776.2101902245, "eta_seconds": 23312.799509835244, "data_wait_seconds": 1.6450881958007812e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20763, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 7090, "total_steps": 72479, "tokens": 3717201920, "loss": 4.969151616096497, "l_mix": 3.074327230453491, "l_a": 3.1756482124328613, "l_c": 3.1349379122257233, "z_loss": 16.484492540359497, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2090006172657013, "tokens_per_second": 1434811.135472991, "eta_seconds": 23893.505691742896, "data_wait_seconds": 2.1189451217651367e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20762, "cosmopedia_v2": 0.20042, "finemath": 0.08237}}
{"step": 7100, "total_steps": 72479, "tokens": 3722444800, "loss": 4.863537609577179, "l_mix": 3.007972836494446, "l_a": 3.1117528676986694, "l_c": 3.067987948656082, "z_loss": 16.424691081047058, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21951299905776978, "tokens_per_second": 1447153.4979626874, "eta_seconds": 23686.101854610442, "data_wait_seconds": 1.5050172805786133e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20761, "cosmopedia_v2": 0.20042, "finemath": 0.08239}, "gates": {"gate_a2b/L1_absmean": 0.14755654, "gate_a2b/L1_absmax": 0.59532326, "gate_a2b/L1_rms": 0.1895633, "gate_a2b/L3_absmean": 0.27359292, "gate_a2b/L3_absmax": 0.45945996, "gate_a2b/L3_rms": 0.28075483, "gate_a2b/L5_absmean": 0.4535329, "gate_a2b/L5_absmax": 0.6444971, "gate_a2b/L5_rms": 0.46274579, "gate_c2b/L1_absmean": 0.01527162, "gate_c2b/L1_absmax": 0.58559841, "gate_c2b/L1_rms": 0.06321887, "gate_c2b/L3_absmean": 0.22967754, "gate_c2b/L3_absmax": 0.47523308, "gate_c2b/L3_rms": 0.23673801, "gate_c2b/L5_absmean": 0.54829162, "gate_c2b/L5_absmax": 0.79164749, "gate_c2b/L5_rms": 0.56346315, "gate_b2a/L7_absmean": 0.06651959, "gate_b2a/L7_absmax": 0.24860071, "gate_b2a/L7_rms": 0.07223849, "gate_b2a/L11_absmean": 0.11541995, "gate_b2a/L11_absmax": 0.27530214, "gate_b2a/L11_rms": 0.11755302, "gate_b2a/L14_absmean": 0.27879429, "gate_b2a/L14_absmax": 0.43996358, "gate_b2a/L14_rms": 0.27981156, "gate_b2c/L3_absmean": 0.04599036, "gate_b2c/L3_absmax": 0.43684557, "gate_b2c/L3_rms": 0.06693381, "gate_b2c/L5_absmean": 0.08895596, "gate_b2c/L5_absmax": 0.47951764, "gate_b2c/L5_rms": 0.09311855, "gate_b2c/L6_absmean": 0.13718337, "gate_b2c/L6_absmax": 0.37388074, "gate_b2c/L6_rms": 0.13896804}, "streams": {"cos_a2b/round0": 0.361879, "cos_c2b/round0": 0.23942, "cos_b2a/round0": 0.187994, "cos_b2c/round0": 0.112631, "rms_a/round0": 1.205053, "rms_b/round0": 2.181317, "rms_c/round0": 1.332144, "cos_a2b/round1": 0.378953, "cos_c2b/round1": 0.265363, "cos_b2a/round1": 0.202055, "cos_b2c/round1": 0.162411, "rms_a/round1": 2.127296, "rms_b/round1": 3.874458, "rms_c/round1": 2.169438, "cos_a2b/round2": 0.487229, "cos_c2b/round2": 0.431787, "cos_b2a/round2": 0.084908, "cos_b2c/round2": 0.143082, "rms_a/round2": 4.200271, "rms_b/round2": 7.743659, "rms_c/round2": 3.305431}, "alpha_mean": 0.4457773393770168, "alpha_hist": [0.02321, 0.06563, 0.1376, 0.19707, 0.18646, 0.16838, 0.13087, 0.07481, 0.0154, 0.00056], "alpha_by_type": {"special": {"share": 0.00099, "mean": 0.55764, "std": 0.10688}, "whitespace": {"share": 0.02737, "mean": 0.54457, "std": 0.13371}, "digit": {"share": 0.05341, "mean": 0.56448, "std": 0.12454}, "word_start": {"share": 0.52759, "mean": 0.46945, "std": 0.1611}, "word_cont": {"share": 0.28021, "mean": 0.31604, "std": 0.14561}, "punct": {"share": 0.11043, "mean": 0.57901, "std": 0.17722}}, "alpha_p05": 0.14953, "alpha_p50": 0.4477, "alpha_p95": 0.71701}
{"step": 7110, "total_steps": 72479, "tokens": 3727687680, "loss": 5.133357286453247, "l_mix": 3.1801944971084595, "l_a": 3.270503878593445, "l_c": 3.234687715768814, "z_loss": 16.05219340324402, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17466682195663452, "tokens_per_second": 1469490.1621727962, "eta_seconds": 23322.49861497879, "data_wait_seconds": 1.6242265701293945e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20774, "cosmopedia_v2": 0.20042, "finemath": 0.08228}}
{"step": 7120, "total_steps": 72479, "tokens": 3732930560, "loss": 5.198326051235199, "l_mix": 3.221336305141449, "l_a": 3.306319683790207, "l_c": 3.278352051973343, "z_loss": 15.881640911102295, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22523809969425201, "tokens_per_second": 1463766.2804348173, "eta_seconds": 23410.116662764547, "data_wait_seconds": 2.1249055862426758e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20772, "cosmopedia_v2": 0.20042, "finemath": 0.0823}}
{"step": 7130, "total_steps": 72479, "tokens": 3738173440, "loss": 4.977451503276825, "l_mix": 3.082330346107483, "l_a": 3.1715966761112213, "l_c": 3.14009428024292, "z_loss": 16.138732194900513, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19691312313079834, "tokens_per_second": 1439887.6857261953, "eta_seconds": 23794.700692033766, "data_wait_seconds": 2.059340476989746e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20771, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 7140, "total_steps": 72479, "tokens": 3743416320, "loss": 5.455749332904816, "l_mix": 3.3798842430114746, "l_a": 3.461826503276825, "l_c": 3.4524331092834473, "z_loss": 15.871334552764893, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.5324720740318298, "tokens_per_second": 1480781.461560963, "eta_seconds": 23134.037345314024, "data_wait_seconds": 3.886222839355469e-05, "mix": {"fineweb_hq": 0.50952, "dclm": 0.2077, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 7150, "total_steps": 72479, "tokens": 3748659200, "loss": 4.9791587591171265, "l_mix": 3.0823561549186707, "l_a": 3.1796833276748657, "l_c": 3.137692630290985, "z_loss": 15.89767301082611, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1801975965499878, "tokens_per_second": 1448793.5021685883, "eta_seconds": 23641.19572646618, "data_wait_seconds": 2.917647361755371e-05, "mix": {"fineweb_hq": 0.50951, "dclm": 0.20769, "cosmopedia_v2": 0.20042, "finemath": 0.08238}}
{"step": 7160, "total_steps": 72479, "tokens": 3753902080, "loss": 3.258899539709091, "l_mix": 2.012496918439865, "l_a": 2.091329872608185, "l_c": 2.057327449321747, "z_loss": 18.05375027656555, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.26361119747161865, "tokens_per_second": 1466113.7333214658, "eta_seconds": 23358.329639554024, "data_wait_seconds": 3.787875175476074e-05, "mix": {"fineweb_hq": 0.5095, "dclm": 0.20768, "cosmopedia_v2": 0.20042, "finemath": 0.0824}}
{"step": 7170, "total_steps": 72479, "tokens": 3759144960, "loss": 5.0388840436935425, "l_mix": 3.1187582910060883, "l_a": 3.2121899724006653, "l_c": 3.182849258184433, "z_loss": 16.138814687728882, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1726653128862381, "tokens_per_second": 1402944.0047853985, "eta_seconds": 24406.33758382797, "data_wait_seconds": 1.823902130126953e-05, "mix": {"fineweb_hq": 0.50962, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08229}}
{"step": 7180, "total_steps": 72479, "tokens": 3764387840, "loss": 4.88306850194931, "l_mix": 3.0202039778232574, "l_a": 3.1251320242881775, "l_c": 3.0789481103420258, "z_loss": 16.4030202627182, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18046478927135468, "tokens_per_second": 1333777.4291411699, "eta_seconds": 25668.062274861335, "data_wait_seconds": 2.136826515197754e-05, "mix": {"fineweb_hq": 0.50961, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08231}}
{"step": 7190, "total_steps": 72479, "tokens": 3769630720, "loss": 5.0027236342430115, "l_mix": 3.096485674381256, "l_a": 3.1940200328826904, "l_c": 3.1546459794044495, "z_loss": 16.3807590007782, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.14751900732517242, "tokens_per_second": 1429755.010300109, "eta_seconds": 23941.331896305084, "data_wait_seconds": 2.3931264877319336e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 7200, "total_steps": 72479, "tokens": 3774873600, "loss": 4.7492382526397705, "l_mix": 2.9384123980998993, "l_a": 3.035043716430664, "l_c": 2.9955332279205322, "z_loss": 16.526344537734985, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19226963818073273, "tokens_per_second": 1433839.476718677, "eta_seconds": 23869.4755638361, "data_wait_seconds": 2.47955322265625e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20042, "finemath": 0.08236}, "gates": {"gate_a2b/L1_absmean": 0.14896642, "gate_a2b/L1_absmax": 0.5975526, "gate_a2b/L1_rms": 0.19108531, "gate_a2b/L3_absmean": 0.2733469, "gate_a2b/L3_absmax": 0.45462561, "gate_a2b/L3_rms": 0.28045404, "gate_a2b/L5_absmean": 0.45344368, "gate_a2b/L5_absmax": 0.64692158, "gate_a2b/L5_rms": 0.46253824, "gate_c2b/L1_absmean": 0.01602493, "gate_c2b/L1_absmax": 0.58626974, "gate_c2b/L1_rms": 0.0640583, "gate_c2b/L3_absmean": 0.22906022, "gate_c2b/L3_absmax": 0.47057739, "gate_c2b/L3_rms": 0.23611638, "gate_c2b/L5_absmean": 0.5485974, "gate_c2b/L5_absmax": 0.78638172, "gate_c2b/L5_rms": 0.56342703, "gate_b2a/L7_absmean": 0.0670719, "gate_b2a/L7_absmax": 0.25113043, "gate_b2a/L7_rms": 0.07277392, "gate_b2a/L11_absmean": 0.11559959, "gate_b2a/L11_absmax": 0.27846298, "gate_b2a/L11_rms": 0.11779995, "gate_b2a/L14_absmean": 0.2789306, "gate_b2a/L14_absmax": 0.44405413, "gate_b2a/L14_rms": 0.27995989, "gate_b2c/L3_absmean": 0.04653258, "gate_b2c/L3_absmax": 0.44115716, "gate_b2c/L3_rms": 0.06740166, "gate_b2c/L5_absmean": 0.08907195, "gate_b2c/L5_absmax": 0.49118096, "gate_b2c/L5_rms": 0.09341212, "gate_b2c/L6_absmean": 0.13767099, "gate_b2c/L6_absmax": 0.37228259, "gate_b2c/L6_rms": 0.13946488}, "streams": {"cos_a2b/round0": 0.373996, "cos_c2b/round0": 0.241409, "cos_b2a/round0": 0.200108, "cos_b2c/round0": 0.117958, "rms_a/round0": 1.19947, "rms_b/round0": 2.216648, "rms_c/round0": 1.349607, "cos_a2b/round1": 0.385806, "cos_c2b/round1": 0.262876, "cos_b2a/round1": 0.214055, "cos_b2c/round1": 0.161825, "rms_a/round1": 2.111011, "rms_b/round1": 3.877738, "rms_c/round1": 2.205902, "cos_a2b/round2": 0.48261, "cos_c2b/round2": 0.461138, "cos_b2a/round2": 0.092651, "cos_b2c/round2": 0.138267, "rms_a/round2": 4.147029, "rms_b/round2": 7.887185, "rms_c/round2": 3.450289}, "alpha_mean": 0.4369857334604035, "alpha_hist": [0.01254, 0.06555, 0.14449, 0.20204, 0.21741, 0.18362, 0.10806, 0.05557, 0.01044, 0.00027], "alpha_by_type": {"special": {"share": 0.00095, "mean": 0.58017, "std": 0.09013}, "whitespace": {"share": 0.02908, "mean": 0.52274, "std": 0.11874}, "digit": {"share": 0.0437, "mean": 0.51167, "std": 0.12246}, "word_start": {"share": 0.51939, "mean": 0.4465, "std": 0.16033}, "word_cont": {"share": 0.28105, "mean": 0.34319, "std": 0.14188}, "punct": {"share": 0.12582, "mean": 0.56038, "std": 0.14097}}, "alpha_p05": 0.18565, "alpha_p50": 0.4371, "alpha_p95": 0.71721}
{"step": 7210, "total_steps": 72479, "tokens": 3780116480, "loss": 4.875445008277893, "l_mix": 3.0163413286209106, "l_a": 3.1154890060424805, "l_c": 3.0760465264320374, "z_loss": 16.430445432662964, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19963790476322174, "tokens_per_second": 1457988.0160831814, "eta_seconds": 23470.531372356418, "data_wait_seconds": 2.1576881408691406e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20763, "cosmopedia_v2": 0.20042, "finemath": 0.08239}}
{"step": 7220, "total_steps": 72479, "tokens": 3785359360, "loss": 3.3941843807697296, "l_mix": 2.095693975687027, "l_a": 2.1901975274086, "l_c": 2.131916254758835, "z_loss": 18.562530040740967, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.318004846572876, "tokens_per_second": 1361712.080358864, "eta_seconds": 25126.09756901264, "data_wait_seconds": 2.2411346435546875e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20762, "cosmopedia_v2": 0.20042, "finemath": 0.08241}}
{"step": 7230, "total_steps": 72479, "tokens": 3790602240, "loss": 4.596542373299599, "l_mix": 2.8464338183403015, "l_a": 2.9287861064076424, "l_c": 2.899332419037819, "z_loss": 16.73008894920349, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1664152890443802, "tokens_per_second": 1405718.1603582082, "eta_seconds": 24335.794099211693, "data_wait_seconds": 2.777576446533203e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20775, "cosmopedia_v2": 0.20041, "finemath": 0.0823}}
{"step": 7240, "total_steps": 72479, "tokens": 3795845120, "loss": 5.065498232841492, "l_mix": 3.1362219154834747, "l_a": 3.228210061788559, "l_c": 3.197270154953003, "z_loss": 16.32170796394348, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18396928906440735, "tokens_per_second": 1429528.7828591496, "eta_seconds": 23926.782896661756, "data_wait_seconds": 2.03549861907959e-05, "mix": {"fineweb_hq": 0.50953, "dclm": 0.20773, "cosmopedia_v2": 0.20041, "finemath": 0.08232}}
{"step": 7250, "total_steps": 72479, "tokens": 3801088000, "loss": 5.03546416759491, "l_mix": 3.118186056613922, "l_a": 3.209666609764099, "l_c": 3.1759421825408936, "z_loss": 15.953751683235168, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18423572182655334, "tokens_per_second": 1481244.1750651784, "eta_seconds": 23087.87607586384, "data_wait_seconds": 2.4110078811645508e-05, "mix": {"fineweb_hq": 0.50952, "dclm": 0.20772, "cosmopedia_v2": 0.20041, "finemath": 0.08234}}
{"step": 7260, "total_steps": 72479, "tokens": 3806330880, "loss": 5.062960684299469, "l_mix": 3.135054051876068, "l_a": 3.2261369824409485, "l_c": 3.1948162615299225, "z_loss": 16.20644736289978, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2036457657814026, "tokens_per_second": 1437604.133216574, "eta_seconds": 23785.086785674095, "data_wait_seconds": 2.5928020477294922e-05, "mix": {"fineweb_hq": 0.5095, "dclm": 0.20771, "cosmopedia_v2": 0.20041, "finemath": 0.08237}}
{"step": 7270, "total_steps": 72479, "tokens": 3811573760, "loss": 5.1324151158332825, "l_mix": 3.1789333820343018, "l_a": 3.2689353227615356, "l_c": 3.2371288537979126, "z_loss": 16.623328685760498, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17953698337078094, "tokens_per_second": 1441287.5257929733, "eta_seconds": 23720.66335146427, "data_wait_seconds": 2.2023916244506836e-05, "mix": {"fineweb_hq": 0.50949, "dclm": 0.2077, "cosmopedia_v2": 0.20041, "finemath": 0.08239}}
{"step": 7280, "total_steps": 72479, "tokens": 3816816640, "loss": 4.859739363193512, "l_mix": 3.0061376988887787, "l_a": 3.110195517539978, "l_c": 3.063049167394638, "z_loss": 16.281445384025574, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1976463794708252, "tokens_per_second": 1471781.2558304241, "eta_seconds": 23225.63436419964, "data_wait_seconds": 1.6182661056518555e-05, "mix": {"fineweb_hq": 0.50962, "dclm": 0.20769, "cosmopedia_v2": 0.20041, "finemath": 0.08228}}
{"step": 7290, "total_steps": 72479, "tokens": 3822059520, "loss": 4.945735812187195, "l_mix": 3.0604008734226227, "l_a": 3.161084473133087, "l_c": 3.1180336475372314, "z_loss": 15.994296073913574, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15229268372058868, "tokens_per_second": 1459226.7915399452, "eta_seconds": 23421.86329784393, "data_wait_seconds": 1.582503318786621e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20768, "cosmopedia_v2": 0.20041, "finemath": 0.0823}}
{"step": 7300, "total_steps": 72479, "tokens": 3827302400, "loss": 4.884218215942383, "l_mix": 3.0212635695934296, "l_a": 3.124098777770996, "l_c": 3.080275774002075, "z_loss": 16.422638654708862, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18636460602283478, "tokens_per_second": 1417844.2454617696, "eta_seconds": 24101.77821815014, "data_wait_seconds": 2.446770668029785e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.15068428, "gate_a2b/L1_absmax": 0.59814012, "gate_a2b/L1_rms": 0.19267397, "gate_a2b/L3_absmean": 0.27391902, "gate_a2b/L3_absmax": 0.45434937, "gate_a2b/L3_rms": 0.28090811, "gate_a2b/L5_absmean": 0.45450112, "gate_a2b/L5_absmax": 0.64484495, "gate_a2b/L5_rms": 0.46348473, "gate_c2b/L1_absmean": 0.0159959, "gate_c2b/L1_absmax": 0.58478564, "gate_c2b/L1_rms": 0.06400391, "gate_c2b/L3_absmean": 0.2284832, "gate_c2b/L3_absmax": 0.4724609, "gate_c2b/L3_rms": 0.23540775, "gate_c2b/L5_absmean": 0.54873723, "gate_c2b/L5_absmax": 0.78933561, "gate_c2b/L5_rms": 0.5636878, "gate_b2a/L7_absmean": 0.06701916, "gate_b2a/L7_absmax": 0.25099924, "gate_b2a/L7_rms": 0.072754, "gate_b2a/L11_absmean": 0.11564431, "gate_b2a/L11_absmax": 0.28168741, "gate_b2a/L11_rms": 0.11778142, "gate_b2a/L14_absmean": 0.27944362, "gate_b2a/L14_absmax": 0.4473224, "gate_b2a/L14_rms": 0.28048342, "gate_b2c/L3_absmean": 0.04691328, "gate_b2c/L3_absmax": 0.44343823, "gate_b2c/L3_rms": 0.06784852, "gate_b2c/L5_absmean": 0.08941712, "gate_b2c/L5_absmax": 0.49400917, "gate_b2c/L5_rms": 0.09380241, "gate_b2c/L6_absmean": 0.13787583, "gate_b2c/L6_absmax": 0.37967676, "gate_b2c/L6_rms": 0.13976014}, "streams": {"cos_a2b/round0": 0.362965, "cos_c2b/round0": 0.240962, "cos_b2a/round0": 0.193981, "cos_b2c/round0": 0.11601, "rms_a/round0": 1.172701, "rms_b/round0": 2.161271, "rms_c/round0": 1.331912, "cos_a2b/round1": 0.392255, "cos_c2b/round1": 0.268265, "cos_b2a/round1": 0.209089, "cos_b2c/round1": 0.173761, "rms_a/round1": 2.107288, "rms_b/round1": 3.869095, "rms_c/round1": 2.175586, "cos_a2b/round2": 0.496844, "cos_c2b/round2": 0.438676, "cos_b2a/round2": 0.070226, "cos_b2c/round2": 0.137996, "rms_a/round2": 4.157634, "rms_b/round2": 7.745048, "rms_c/round2": 3.378953}, "alpha_mean": 0.4514409017742338, "alpha_hist": [0.02167, 0.06859, 0.14348, 0.18246, 0.17661, 0.16457, 0.13557, 0.08755, 0.0191, 0.0004], "alpha_by_type": {"special": {"share": 0.0011, "mean": 0.57031, "std": 0.12775}, "whitespace": {"share": 0.02441, "mean": 0.54447, "std": 0.1292}, "digit": {"share": 0.03342, "mean": 0.6262, "std": 0.11595}, "word_start": {"share": 0.58635, "mean": 0.46807, "std": 0.16788}, "word_cont": {"share": 0.25592, "mean": 0.32178, "std": 0.15724}, "punct": {"share": 0.0988, "mean": 0.6052, "std": 0.16507}}, "alpha_p05": 0.14658, "alpha_p50": 0.43111, "alpha_p95": 0.76524}
{"step": 7310, "total_steps": 72479, "tokens": 3832545280, "loss": 4.917839288711548, "l_mix": 3.04359969496727, "l_a": 3.1409787237644196, "l_c": 3.1010730266571045, "z_loss": 16.24124002456665, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17452684044837952, "tokens_per_second": 1364137.1911428173, "eta_seconds": 25046.839052438736, "data_wait_seconds": 1.7404556274414062e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 7320, "total_steps": 72479, "tokens": 3837788160, "loss": 4.901786983013153, "l_mix": 3.0322346687316895, "l_a": 3.137854218482971, "l_c": 3.0885491967201233, "z_loss": 16.312573671340942, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17049859464168549, "tokens_per_second": 1324722.9394779105, "eta_seconds": 25788.095588850974, "data_wait_seconds": 1.767277717590332e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20041, "finemath": 0.08238}}
{"step": 7330, "total_steps": 72479, "tokens": 3843031040, "loss": 3.3761148154735565, "l_mix": 2.084903836250305, "l_a": 2.1715138852596283, "l_c": 2.126406356692314, "z_loss": 18.34861731529236, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20588406920433044, "tokens_per_second": 1445239.2263300216, "eta_seconds": 23634.038081526756, "data_wait_seconds": 1.913309097290039e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20764, "cosmopedia_v2": 0.20041, "finemath": 0.0824}}
{"step": 7340, "total_steps": 72479, "tokens": 3848273920, "loss": 3.490414261817932, "l_mix": 2.15590463578701, "l_a": 2.2431236654520035, "l_c": 2.1990473866462708, "z_loss": 18.582101106643677, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22597534954547882, "tokens_per_second": 1241970.7668091538, "eta_seconds": 27497.906508493423, "data_wait_seconds": 2.5600194931030273e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20763, "cosmopedia_v2": 0.20041, "finemath": 0.08243}}
{"step": 7350, "total_steps": 72479, "tokens": 3853516800, "loss": 4.994097113609314, "l_mix": 3.0898862183094025, "l_a": 3.1888415813446045, "l_c": 3.1532071828842163, "z_loss": 15.961928129196167, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17842921614646912, "tokens_per_second": 1412779.2501812854, "eta_seconds": 24169.630993390085, "data_wait_seconds": 2.3365020751953125e-05, "mix": {"fineweb_hq": 0.50966, "dclm": 0.20762, "cosmopedia_v2": 0.20041, "finemath": 0.08231}}
{"step": 7360, "total_steps": 72479, "tokens": 3858759680, "loss": 4.863822817802429, "l_mix": 3.008386552333832, "l_a": 3.116737574338913, "l_c": 3.062590569257736, "z_loss": 16.378466963768005, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18137888610363007, "tokens_per_second": 1437452.836685448, "eta_seconds": 23751.116837143898, "data_wait_seconds": 2.1666288375854492e-05, "mix": {"fineweb_hq": 0.50965, "dclm": 0.20761, "cosmopedia_v2": 0.20041, "finemath": 0.08234}}
{"step": 7370, "total_steps": 72479, "tokens": 3864002560, "loss": 4.974593818187714, "l_mix": 3.0779468417167664, "l_a": 3.1750433444976807, "l_c": 3.1417572796344757, "z_loss": 16.067290544509888, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19046550989151, "tokens_per_second": 1468433.720352009, "eta_seconds": 23246.44750313759, "data_wait_seconds": 2.4050474166870117e-05, "mix": {"fineweb_hq": 0.50963, "dclm": 0.2076, "cosmopedia_v2": 0.20041, "finemath": 0.08236}}
{"step": 7380, "total_steps": 72479, "tokens": 3869245440, "loss": 4.928033113479614, "l_mix": 3.047816514968872, "l_a": 3.154395431280136, "l_c": 3.107538193464279, "z_loss": 16.364962577819824, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19479013979434967, "tokens_per_second": 1470281.5746997104, "eta_seconds": 23213.665395331383, "data_wait_seconds": 1.5616416931152344e-05, "mix": {"fineweb_hq": 0.50962, "dclm": 0.20759, "cosmopedia_v2": 0.20041, "finemath": 0.08238}}
{"step": 7390, "total_steps": 72479, "tokens": 3874488320, "loss": 4.967042684555054, "l_mix": 3.073834240436554, "l_a": 3.1793623864650726, "l_c": 3.1258115768432617, "z_loss": 16.560818910598755, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22968821227550507, "tokens_per_second": 1447347.3283902898, "eta_seconds": 23577.880003380775, "data_wait_seconds": 1.6748905181884766e-05, "mix": {"fineweb_hq": 0.50961, "dclm": 0.20758, "cosmopedia_v2": 0.20041, "finemath": 0.08241}}
{"step": 7400, "total_steps": 72479, "tokens": 3879731200, "loss": 4.8219969272613525, "l_mix": 2.9820388853549957, "l_a": 3.085462063550949, "l_c": 3.0422865748405457, "z_loss": 16.332714676856995, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17029452323913574, "tokens_per_second": 1457651.8858872817, "eta_seconds": 23407.604437208174, "data_wait_seconds": 2.0742416381835938e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.2077, "cosmopedia_v2": 0.20041, "finemath": 0.0823}, "gates": {"gate_a2b/L1_absmean": 0.1527548, "gate_a2b/L1_absmax": 0.59909189, "gate_a2b/L1_rms": 0.19440092, "gate_a2b/L3_absmean": 0.27496168, "gate_a2b/L3_absmax": 0.45487243, "gate_a2b/L3_rms": 0.28182352, "gate_a2b/L5_absmean": 0.45564207, "gate_a2b/L5_absmax": 0.64549077, "gate_a2b/L5_rms": 0.4646064, "gate_c2b/L1_absmean": 0.01578073, "gate_c2b/L1_absmax": 0.58466834, "gate_c2b/L1_rms": 0.06387775, "gate_c2b/L3_absmean": 0.22735572, "gate_c2b/L3_absmax": 0.46728671, "gate_c2b/L3_rms": 0.23423082, "gate_c2b/L5_absmean": 0.54788584, "gate_c2b/L5_absmax": 0.7873987, "gate_c2b/L5_rms": 0.5628069, "gate_b2a/L7_absmean": 0.06711877, "gate_b2a/L7_absmax": 0.25157386, "gate_b2a/L7_rms": 0.07277428, "gate_b2a/L11_absmean": 0.11588922, "gate_b2a/L11_absmax": 0.28693751, "gate_b2a/L11_rms": 0.11803488, "gate_b2a/L14_absmean": 0.27994531, "gate_b2a/L14_absmax": 0.44963598, "gate_b2a/L14_rms": 0.28104332, "gate_b2c/L3_absmean": 0.04710668, "gate_b2c/L3_absmax": 0.44702521, "gate_b2c/L3_rms": 0.06815873, "gate_b2c/L5_absmean": 0.0896477, "gate_b2c/L5_absmax": 0.49787685, "gate_b2c/L5_rms": 0.09408343, "gate_b2c/L6_absmean": 0.13829795, "gate_b2c/L6_absmax": 0.37735713, "gate_b2c/L6_rms": 0.14017127}, "streams": {"cos_a2b/round0": 0.369465, "cos_c2b/round0": 0.244051, "cos_b2a/round0": 0.195555, "cos_b2c/round0": 0.116065, "rms_a/round0": 1.176434, "rms_b/round0": 2.15848, "rms_c/round0": 1.321676, "cos_a2b/round1": 0.393367, "cos_c2b/round1": 0.266025, "cos_b2a/round1": 0.209731, "cos_b2c/round1": 0.172959, "rms_a/round1": 2.125933, "rms_b/round1": 3.843707, "rms_c/round1": 2.197612, "cos_a2b/round2": 0.501538, "cos_c2b/round2": 0.430366, "cos_b2a/round2": 0.072185, "cos_b2c/round2": 0.152555, "rms_a/round2": 4.167153, "rms_b/round2": 7.712365, "rms_c/round2": 3.419951}, "alpha_mean": 0.49124097370756203, "alpha_hist": [0.01436, 0.05078, 0.10834, 0.15921, 0.18199, 0.17096, 0.15846, 0.11458, 0.04015, 0.00117], "alpha_by_type": {"special": {"share": 0.00156, "mean": 0.55014, "std": 0.12438}, "whitespace": {"share": 0.02602, "mean": 0.56057, "std": 0.13459}, "digit": {"share": 0.02921, "mean": 0.49892, "std": 0.11851}, "word_start": {"share": 0.59459, "mean": 0.51804, "std": 0.17544}, "word_cont": {"share": 0.25058, "mean": 0.36251, "std": 0.162}, "punct": {"share": 0.09805, "mean": 0.63609, "std": 0.16644}}, "alpha_p05": 0.17724, "alpha_p50": 0.48066, "alpha_p95": 0.78399}
{"step": 7410, "total_steps": 72479, "tokens": 3884974080, "loss": 4.774506032466888, "l_mix": 2.953143209218979, "l_a": 3.0545714497566223, "l_c": 3.0111367106437683, "z_loss": 16.503443241119385, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15062351524829865, "tokens_per_second": 1457830.7558941583, "eta_seconds": 23401.136060595512, "data_wait_seconds": 2.3365020751953125e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20769, "cosmopedia_v2": 0.2004, "finemath": 0.08232}}
{"step": 7420, "total_steps": 72479, "tokens": 3890216960, "loss": 4.99140989780426, "l_mix": 3.087671399116516, "l_a": 3.189506232738495, "l_c": 3.150998115539551, "z_loss": 15.871287822723389, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1809600293636322, "tokens_per_second": 1439053.399213666, "eta_seconds": 23702.84036064148, "data_wait_seconds": 1.850724220275879e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20768, "cosmopedia_v2": 0.2004, "finemath": 0.08235}}
{"step": 7430, "total_steps": 72479, "tokens": 3895459840, "loss": 4.889812469482422, "l_mix": 3.0254511535167694, "l_a": 3.129861891269684, "l_c": 3.0792235732078552, "z_loss": 16.35548448562622, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1556498259305954, "tokens_per_second": 1460153.2803115386, "eta_seconds": 23356.73286623955, "data_wait_seconds": 1.9311904907226562e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.2004, "finemath": 0.08237}}
{"step": 7440, "total_steps": 72479, "tokens": 3900702720, "loss": 3.3901666700839996, "l_mix": 2.095089912414551, "l_a": 2.178567588329315, "l_c": 2.1323049068450928, "z_loss": 18.14978313446045, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2145213782787323, "tokens_per_second": 1359465.5765207508, "eta_seconds": 25082.77356994152, "data_wait_seconds": 2.6851892471313477e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20766, "cosmopedia_v2": 0.2004, "finemath": 0.08239}}
{"step": 7450, "total_steps": 72479, "tokens": 3905945600, "loss": 3.371831029653549, "l_mix": 2.082957372069359, "l_a": 2.1643009781837463, "l_c": 2.126000627875328, "z_loss": 17.831421613693237, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.23879474401474, "tokens_per_second": 1303914.349536546, "eta_seconds": 26147.36494319439, "data_wait_seconds": 3.662705421447754e-05, "mix": {"fineweb_hq": 0.50953, "dclm": 0.20765, "cosmopedia_v2": 0.2004, "finemath": 0.08242}}
{"step": 7460, "total_steps": 72479, "tokens": 3911188480, "loss": 4.885359048843384, "l_mix": 3.021620273590088, "l_a": 3.128022253513336, "l_c": 3.07899409532547, "z_loss": 16.337586879730225, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1603448987007141, "tokens_per_second": 1347033.2262550232, "eta_seconds": 25306.48896224499, "data_wait_seconds": 1.6987323760986328e-05, "mix": {"fineweb_hq": 0.50965, "dclm": 0.20764, "cosmopedia_v2": 0.2004, "finemath": 0.08231}}
{"step": 7470, "total_steps": 72479, "tokens": 3916431360, "loss": 4.9384366273880005, "l_mix": 3.0561546981334686, "l_a": 3.1553133726119995, "l_c": 3.1135916113853455, "z_loss": 16.103079438209534, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18052762746810913, "tokens_per_second": 1294328.5966854119, "eta_seconds": 26332.91011207104, "data_wait_seconds": 2.7388334274291992e-05, "mix": {"fineweb_hq": 0.50964, "dclm": 0.20763, "cosmopedia_v2": 0.2004, "finemath": 0.08233}}
{"step": 7480, "total_steps": 72479, "tokens": 3921674240, "loss": 4.933493614196777, "l_mix": 3.052568346261978, "l_a": 3.1546434462070465, "l_c": 3.1095617413520813, "z_loss": 16.638395309448242, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16634996235370636, "tokens_per_second": 1465414.2930640322, "eta_seconds": 23254.99066939354, "data_wait_seconds": 2.378225326538086e-05, "mix": {"fineweb_hq": 0.50963, "dclm": 0.20762, "cosmopedia_v2": 0.2004, "finemath": 0.08235}}
{"step": 7490, "total_steps": 72479, "tokens": 3926917120, "loss": 4.792550504207611, "l_mix": 2.9636717438697815, "l_a": 3.070274740457535, "l_c": 3.020534574985504, "z_loss": 16.359350442886353, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1819053739309311, "tokens_per_second": 1417472.367027479, "eta_seconds": 24037.825092458726, "data_wait_seconds": 2.390146255493164e-05, "mix": {"fineweb_hq": 0.50961, "dclm": 0.20761, "cosmopedia_v2": 0.2004, "finemath": 0.08238}}
{"step": 7500, "total_steps": 72479, "tokens": 3932160000, "loss": 4.911129832267761, "l_mix": 3.0387798845767975, "l_a": 3.1380951702594757, "l_c": 3.097688525915146, "z_loss": 16.148595929145813, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17660684883594513, "tokens_per_second": 1379695.3255563427, "eta_seconds": 24692.19785046577, "data_wait_seconds": 1.582503318786621e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.2076, "cosmopedia_v2": 0.2004, "finemath": 0.0824}, "gates": {"gate_a2b/L1_absmean": 0.15444672, "gate_a2b/L1_absmax": 0.59876406, "gate_a2b/L1_rms": 0.19613551, "gate_a2b/L3_absmean": 0.27547881, "gate_a2b/L3_absmax": 0.45714098, "gate_a2b/L3_rms": 0.28233904, "gate_a2b/L5_absmean": 0.45659897, "gate_a2b/L5_absmax": 0.64973396, "gate_a2b/L5_rms": 0.46551597, "gate_c2b/L1_absmean": 0.01574392, "gate_c2b/L1_absmax": 0.58102798, "gate_c2b/L1_rms": 0.06417419, "gate_c2b/L3_absmean": 0.22748439, "gate_c2b/L3_absmax": 0.46350157, "gate_c2b/L3_rms": 0.23417199, "gate_c2b/L5_absmean": 0.5479151, "gate_c2b/L5_absmax": 0.78416467, "gate_c2b/L5_rms": 0.56276262, "gate_b2a/L7_absmean": 0.06760553, "gate_b2a/L7_absmax": 0.25153109, "gate_b2a/L7_rms": 0.07315858, "gate_b2a/L11_absmean": 0.11639827, "gate_b2a/L11_absmax": 0.28690314, "gate_b2a/L11_rms": 0.11851139, "gate_b2a/L14_absmean": 0.28095764, "gate_b2a/L14_absmax": 0.44555029, "gate_b2a/L14_rms": 0.28201655, "gate_b2c/L3_absmean": 0.04824772, "gate_b2c/L3_absmax": 0.44750181, "gate_b2c/L3_rms": 0.0693774, "gate_b2c/L5_absmean": 0.09051128, "gate_b2c/L5_absmax": 0.49968567, "gate_b2c/L5_rms": 0.09486818, "gate_b2c/L6_absmean": 0.13969474, "gate_b2c/L6_absmax": 0.37522018, "gate_b2c/L6_rms": 0.14155146}, "streams": {"cos_a2b/round0": 0.373025, "cos_c2b/round0": 0.251031, "cos_b2a/round0": 0.195367, "cos_b2c/round0": 0.121398, "rms_a/round0": 1.151613, "rms_b/round0": 2.119286, "rms_c/round0": 1.30763, "cos_a2b/round1": 0.392285, "cos_c2b/round1": 0.272414, "cos_b2a/round1": 0.21304, "cos_b2c/round1": 0.17133, "rms_a/round1": 2.059789, "rms_b/round1": 3.74736, "rms_c/round1": 2.149892, "cos_a2b/round2": 0.505052, "cos_c2b/round2": 0.437336, "cos_b2a/round2": 0.086863, "cos_b2c/round2": 0.151547, "rms_a/round2": 4.08399, "rms_b/round2": 7.517837, "rms_c/round2": 3.324786}, "alpha_mean": 0.4934927525574153, "alpha_hist": [0.01064, 0.0446, 0.10307, 0.14949, 0.18466, 0.20796, 0.16844, 0.10301, 0.02744, 0.00069], "alpha_by_type": {"special": {"share": 0.00111, "mean": 0.60456, "std": 0.11028}, "whitespace": {"share": 0.03383, "mean": 0.57639, "std": 0.11972}, "digit": {"share": 0.03238, "mean": 0.5289, "std": 0.12438}, "word_start": {"share": 0.56967, "mean": 0.51056, "std": 0.16273}, "word_cont": {"share": 0.25328, "mean": 0.37964, "std": 0.16382}, "punct": {"share": 0.10973, "mean": 0.63058, "std": 0.15325}}, "alpha_p05": 0.19042, "alpha_p50": 0.49428, "alpha_p95": 0.76804}
{"step": 7510, "total_steps": 72479, "tokens": 3937402880, "loss": 5.06463086605072, "l_mix": 3.13669216632843, "l_a": 3.227651923894882, "l_c": 3.1933723986148834, "z_loss": 16.312973618507385, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1385592222213745, "tokens_per_second": 1474320.7456135734, "eta_seconds": 23103.837596631052, "data_wait_seconds": 2.092123031616211e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20772, "cosmopedia_v2": 0.2004, "finemath": 0.08229}}
{"step": 7520, "total_steps": 72479, "tokens": 3942645760, "loss": 5.0666340589523315, "l_mix": 3.137611895799637, "l_a": 3.230061024427414, "l_c": 3.194597363471985, "z_loss": 16.24518322944641, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17655815184116364, "tokens_per_second": 1465224.9659913487, "eta_seconds": 23243.68269889355, "data_wait_seconds": 2.8014183044433594e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20771, "cosmopedia_v2": 0.2004, "finemath": 0.08231}}
{"step": 7530, "total_steps": 72479, "tokens": 3947888640, "loss": 4.969284355640411, "l_mix": 3.0763156712055206, "l_a": 3.169547379016876, "l_c": 3.134870082139969, "z_loss": 16.433139324188232, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16689245402812958, "tokens_per_second": 1462285.311357747, "eta_seconds": 23286.824429893495, "data_wait_seconds": 2.22623348236084e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.2077, "cosmopedia_v2": 0.2004, "finemath": 0.08234}}
{"step": 7540, "total_steps": 72479, "tokens": 3953131520, "loss": 5.051052272319794, "l_mix": 3.1280530095100403, "l_a": 3.216728299856186, "l_c": 3.1878578066825867, "z_loss": 16.234413146972656, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17227961122989655, "tokens_per_second": 1397675.418678174, "eta_seconds": 24359.545840907096, "data_wait_seconds": 2.5659799575805664e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20769, "cosmopedia_v2": 0.2004, "finemath": 0.08236}}
{"step": 7550, "total_steps": 72479, "tokens": 3958374400, "loss": 4.959276497364044, "l_mix": 3.0707268118858337, "l_a": 3.162534683942795, "l_c": 3.127206653356552, "z_loss": 16.27262771129608, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18086890876293182, "tokens_per_second": 1470751.4213281046, "eta_seconds": 23145.648583674432, "data_wait_seconds": 2.2619962692260742e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20768, "cosmopedia_v2": 0.2004, "finemath": 0.08238}}
{"step": 7560, "total_steps": 72479, "tokens": 3963617280, "loss": 3.3884057104587555, "l_mix": 2.0940456688404083, "l_a": 2.1733026802539825, "l_c": 2.1350392252206802, "z_loss": 18.573670864105225, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.3541290760040283, "tokens_per_second": 1462040.1191480285, "eta_seconds": 23279.971750593184, "data_wait_seconds": 2.2351741790771484e-05, "mix": {"fineweb_hq": 0.50952, "dclm": 0.20767, "cosmopedia_v2": 0.2004, "finemath": 0.08241}}
{"step": 7570, "total_steps": 72479, "tokens": 3968860160, "loss": 4.97817075252533, "l_mix": 3.0806576907634735, "l_a": 3.1828266382217407, "l_c": 3.1367730796337128, "z_loss": 16.330159068107605, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15388120710849762, "tokens_per_second": 1380090.4295173807, "eta_seconds": 24658.536182951928, "data_wait_seconds": 2.580881118774414e-05, "mix": {"fineweb_hq": 0.50964, "dclm": 0.20766, "cosmopedia_v2": 0.2004, "finemath": 0.0823}}
{"step": 7580, "total_steps": 72479, "tokens": 3974103040, "loss": 4.876677632331848, "l_mix": 3.0164418816566467, "l_a": 3.1190118193626404, "l_c": 3.0763759315013885, "z_loss": 16.194168090820312, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15461768209934235, "tokens_per_second": 1381108.975161404, "eta_seconds": 24636.554771518706, "data_wait_seconds": 2.148747444152832e-05, "mix": {"fineweb_hq": 0.50963, "dclm": 0.20765, "cosmopedia_v2": 0.2004, "finemath": 0.08232}}
{"step": 7590, "total_steps": 72479, "tokens": 3979345920, "loss": 4.840454518795013, "l_mix": 2.994694411754608, "l_a": 3.0946733951568604, "l_c": 3.052397906780243, "z_loss": 16.387885570526123, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1555292010307312, "tokens_per_second": 1441980.4769790154, "eta_seconds": 23592.91583702564, "data_wait_seconds": 2.0056962966918945e-05, "mix": {"fineweb_hq": 0.50962, "dclm": 0.20764, "cosmopedia_v2": 0.2004, "finemath": 0.08235}}
{"step": 7600, "total_steps": 72479, "tokens": 3984588800, "loss": 4.809406340122223, "l_mix": 2.9738102853298187, "l_a": 3.0789601802825928, "l_c": 3.0341895520687103, "z_loss": 16.510914087295532, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15767212212085724, "tokens_per_second": 1374388.5898278444, "eta_seconds": 24749.391404843333, "data_wait_seconds": 2.3245811462402344e-05, "mix": {"fineweb_hq": 0.50961, "dclm": 0.20763, "cosmopedia_v2": 0.20039, "finemath": 0.08237}, "gates": {"gate_a2b/L1_absmean": 0.15606503, "gate_a2b/L1_absmax": 0.59753293, "gate_a2b/L1_rms": 0.19761898, "gate_a2b/L3_absmean": 0.27654576, "gate_a2b/L3_absmax": 0.45875356, "gate_a2b/L3_rms": 0.2833873, "gate_a2b/L5_absmean": 0.45789161, "gate_a2b/L5_absmax": 0.65001726, "gate_a2b/L5_rms": 0.46671686, "gate_c2b/L1_absmean": 0.01626098, "gate_c2b/L1_absmax": 0.57858878, "gate_c2b/L1_rms": 0.06427854, "gate_c2b/L3_absmean": 0.22595783, "gate_c2b/L3_absmax": 0.45981637, "gate_c2b/L3_rms": 0.23273343, "gate_c2b/L5_absmean": 0.5467099, "gate_c2b/L5_absmax": 0.77396691, "gate_c2b/L5_rms": 0.56130165, "gate_b2a/L7_absmean": 0.06808586, "gate_b2a/L7_absmax": 0.25030467, "gate_b2a/L7_rms": 0.07348568, "gate_b2a/L11_absmean": 0.11643127, "gate_b2a/L11_absmax": 0.28463256, "gate_b2a/L11_rms": 0.11850852, "gate_b2a/L14_absmean": 0.28001136, "gate_b2a/L14_absmax": 0.44261774, "gate_b2a/L14_rms": 0.28106117, "gate_b2c/L3_absmean": 0.04871654, "gate_b2c/L3_absmax": 0.44860712, "gate_b2c/L3_rms": 0.06955313, "gate_b2c/L5_absmean": 0.09083216, "gate_b2c/L5_absmax": 0.5051142, "gate_b2c/L5_rms": 0.09534726, "gate_b2c/L6_absmean": 0.14042521, "gate_b2c/L6_absmax": 0.37263444, "gate_b2c/L6_rms": 0.14232485}, "streams": {"cos_a2b/round0": 0.369465, "cos_c2b/round0": 0.246037, "cos_b2a/round0": 0.193118, "cos_b2c/round0": 0.115597, "rms_a/round0": 1.152452, "rms_b/round0": 2.126093, "rms_c/round0": 1.328005, "cos_a2b/round1": 0.390171, "cos_c2b/round1": 0.269472, "cos_b2a/round1": 0.21188, "cos_b2c/round1": 0.162591, "rms_a/round1": 2.048011, "rms_b/round1": 3.793416, "rms_c/round1": 2.170563, "cos_a2b/round2": 0.497122, "cos_c2b/round2": 0.440394, "cos_b2a/round2": 0.08102, "cos_b2c/round2": 0.139585, "rms_a/round2": 4.092865, "rms_b/round2": 7.65683, "rms_c/round2": 3.359731}, "alpha_mean": 0.4500996806467583, "alpha_hist": [0.01685, 0.06575, 0.1459, 0.18427, 0.19119, 0.16808, 0.1293, 0.08209, 0.01624, 0.00034], "alpha_by_type": {"special": {"share": 0.00089, "mean": 0.57036, "std": 0.101}, "whitespace": {"share": 0.02788, "mean": 0.54059, "std": 0.13113}, "digit": {"share": 0.03752, "mean": 0.4966, "std": 0.11745}, "word_start": {"share": 0.55083, "mean": 0.47167, "std": 0.16929}, "word_cont": {"share": 0.28131, "mean": 0.33494, "std": 0.14664}, "punct": {"share": 0.10158, "mean": 0.60898, "std": 0.1593}}, "alpha_p05": 0.14854, "alpha_p50": 0.44324, "alpha_p95": 0.74129}
{"step": 7610, "total_steps": 72479, "tokens": 3989831680, "loss": 4.9469146728515625, "l_mix": 3.0603830218315125, "l_a": 3.1631220281124115, "l_c": 3.1198278665542603, "z_loss": 16.466187000274658, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21383555233478546, "tokens_per_second": 1345530.4298688688, "eta_seconds": 25276.30554986, "data_wait_seconds": 1.850724220275879e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20762, "cosmopedia_v2": 0.20039, "finemath": 0.08239}}
{"step": 7620, "total_steps": 72479, "tokens": 3995074560, "loss": 3.4435860216617584, "l_mix": 2.126499205827713, "l_a": 2.211063012480736, "l_c": 2.1733036637306213, "z_loss": 17.766736268997192, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2497444748878479, "tokens_per_second": 1442546.9005164185, "eta_seconds": 23572.748573946952, "data_wait_seconds": 2.0176172256469727e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20761, "cosmopedia_v2": 0.20039, "finemath": 0.08241}}
{"step": 7630, "total_steps": 72479, "tokens": 4000317440, "loss": 5.120587468147278, "l_mix": 3.1707204282283783, "l_a": 3.2663747370243073, "l_c": 3.2276657223701477, "z_loss": 16.54921293258667, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18168054521083832, "tokens_per_second": 1350869.9599934171, "eta_seconds": 25168.634671664237, "data_wait_seconds": 5.128979682922363e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20773, "cosmopedia_v2": 0.20039, "finemath": 0.08231}}
{"step": 7640, "total_steps": 72479, "tokens": 4005560320, "loss": 4.870664447546005, "l_mix": 3.0174883902072906, "l_a": 3.10051092505455, "l_c": 3.071293443441391, "z_loss": 16.347912073135376, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18773338198661804, "tokens_per_second": 1262045.8422267693, "eta_seconds": 26935.875460767747, "data_wait_seconds": 1.8358230590820312e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20772, "cosmopedia_v2": 0.20039, "finemath": 0.08233}}
{"step": 7650, "total_steps": 72479, "tokens": 4010803200, "loss": 5.110387325286865, "l_mix": 3.1659113466739655, "l_a": 3.2534011006355286, "l_c": 3.222778230905533, "z_loss": 16.220285296440125, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16152536869049072, "tokens_per_second": 1457274.9627779198, "eta_seconds": 23323.715578842162, "data_wait_seconds": 1.564621925354004e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20771, "cosmopedia_v2": 0.20039, "finemath": 0.08235}}
{"step": 7660, "total_steps": 72479, "tokens": 4016046080, "loss": 5.108389317989349, "l_mix": 3.1636611223220825, "l_a": 3.2560549080371857, "l_c": 3.220921128988266, "z_loss": 16.352632880210876, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1805766075849533, "tokens_per_second": 1475294.7140407257, "eta_seconds": 23035.278001451494, "data_wait_seconds": 2.613663673400879e-05, "mix": {"fineweb_hq": 0.50953, "dclm": 0.2077, "cosmopedia_v2": 0.20039, "finemath": 0.08238}}
{"step": 7670, "total_steps": 72479, "tokens": 4021288960, "loss": 5.195079565048218, "l_mix": 3.2172583043575287, "l_a": 3.3078412413597107, "l_c": 3.2793973684310913, "z_loss": 16.495895862579346, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17090673744678497, "tokens_per_second": 1469980.92424257, "eta_seconds": 23114.98090324402, "data_wait_seconds": 4.3898820877075195e-05, "mix": {"fineweb_hq": 0.50952, "dclm": 0.20769, "cosmopedia_v2": 0.20039, "finemath": 0.0824}}
{"step": 7680, "total_steps": 72479, "tokens": 4026531840, "loss": 4.797891736030579, "l_mix": 2.9671072959899902, "l_a": 3.072355479001999, "l_c": 3.0247534811496735, "z_loss": 16.517601490020752, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19944648444652557, "tokens_per_second": 1425238.9424774072, "eta_seconds": 23836.942072987556, "data_wait_seconds": 2.6047229766845703e-05, "mix": {"fineweb_hq": 0.50964, "dclm": 0.20768, "cosmopedia_v2": 0.20039, "finemath": 0.08229}}
{"step": 7690, "total_steps": 72479, "tokens": 4031774720, "loss": 4.801662027835846, "l_mix": 2.970087379217148, "l_a": 3.074277549982071, "l_c": 3.025719106197357, "z_loss": 15.756680727005005, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.3292226791381836, "tokens_per_second": 1475416.265981662, "eta_seconds": 23022.719767427443, "data_wait_seconds": 1.6033649444580078e-05, "mix": {"fineweb_hq": 0.50962, "dclm": 0.20767, "cosmopedia_v2": 0.20039, "finemath": 0.08231}}
{"step": 7700, "total_steps": 72479, "tokens": 4037017600, "loss": 4.811152935028076, "l_mix": 2.9762901961803436, "l_a": 3.0781024396419525, "l_c": 3.0326998233795166, "z_loss": 16.220354557037354, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18604867160320282, "tokens_per_second": 1367973.6527900612, "eta_seconds": 24827.124618029597, "data_wait_seconds": 1.659989356994629e-05, "mix": {"fineweb_hq": 0.50961, "dclm": 0.20766, "cosmopedia_v2": 0.20039, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.15823172, "gate_a2b/L1_absmax": 0.59787524, "gate_a2b/L1_rms": 0.19965316, "gate_a2b/L3_absmean": 0.27673754, "gate_a2b/L3_absmax": 0.46074784, "gate_a2b/L3_rms": 0.28350523, "gate_a2b/L5_absmean": 0.45893192, "gate_a2b/L5_absmax": 0.65154767, "gate_a2b/L5_rms": 0.46780306, "gate_c2b/L1_absmean": 0.0159425, "gate_c2b/L1_absmax": 0.57447684, "gate_c2b/L1_rms": 0.06413154, "gate_c2b/L3_absmean": 0.22557293, "gate_c2b/L3_absmax": 0.46142653, "gate_c2b/L3_rms": 0.23233415, "gate_c2b/L5_absmean": 0.54732573, "gate_c2b/L5_absmax": 0.77609915, "gate_c2b/L5_rms": 0.56200761, "gate_b2a/L7_absmean": 0.06843893, "gate_b2a/L7_absmax": 0.25118205, "gate_b2a/L7_rms": 0.0737576, "gate_b2a/L11_absmean": 0.11704666, "gate_b2a/L11_absmax": 0.2903873, "gate_b2a/L11_rms": 0.11916935, "gate_b2a/L14_absmean": 0.28113991, "gate_b2a/L14_absmax": 0.44521472, "gate_b2a/L14_rms": 0.28218964, "gate_b2c/L3_absmean": 0.04925601, "gate_b2c/L3_absmax": 0.44989324, "gate_b2c/L3_rms": 0.07034106, "gate_b2c/L5_absmean": 0.09125762, "gate_b2c/L5_absmax": 0.50881547, "gate_b2c/L5_rms": 0.09578724, "gate_b2c/L6_absmean": 0.14071424, "gate_b2c/L6_absmax": 0.37558991, "gate_b2c/L6_rms": 0.14261717}, "streams": {"cos_a2b/round0": 0.370553, "cos_c2b/round0": 0.247991, "cos_b2a/round0": 0.193913, "cos_b2c/round0": 0.118012, "rms_a/round0": 1.151116, "rms_b/round0": 2.125183, "rms_c/round0": 1.320436, "cos_a2b/round1": 0.388826, "cos_c2b/round1": 0.271281, "cos_b2a/round1": 0.205912, "cos_b2c/round1": 0.16293, "rms_a/round1": 2.054329, "rms_b/round1": 3.752058, "rms_c/round1": 2.164195, "cos_a2b/round2": 0.507208, "cos_c2b/round2": 0.434863, "cos_b2a/round2": 0.089471, "cos_b2c/round2": 0.147236, "rms_a/round2": 4.075251, "rms_b/round2": 7.522016, "rms_c/round2": 3.336609}, "alpha_mean": 0.4830936550691547, "alpha_hist": [0.01155, 0.04211, 0.10144, 0.17595, 0.20181, 0.19261, 0.15674, 0.09572, 0.02155, 0.00052], "alpha_by_type": {"special": {"share": 0.00111, "mean": 0.6083, "std": 0.11986}, "whitespace": {"share": 0.02994, "mean": 0.57522, "std": 0.11287}, "digit": {"share": 0.04108, "mean": 0.49217, "std": 0.11487}, "word_start": {"share": 0.53662, "mean": 0.51463, "std": 0.1567}, "word_cont": {"share": 0.26407, "mean": 0.36356, "std": 0.14472}, "punct": {"share": 0.12718, "mean": 0.57248, "std": 0.19083}}, "alpha_p05": 0.19993, "alpha_p50": 0.4694, "alpha_p95": 0.75672}
{"step": 7710, "total_steps": 72479, "tokens": 4042260480, "loss": 4.8919901847839355, "l_mix": 3.0264824628829956, "l_a": 3.1278975307941437, "l_c": 3.085039585828781, "z_loss": 16.265233874320984, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16848114132881165, "tokens_per_second": 1327824.0723275454, "eta_seconds": 25573.87697639465, "data_wait_seconds": 2.8967857360839844e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20765, "cosmopedia_v2": 0.20039, "finemath": 0.08236}}
{"step": 7720, "total_steps": 72479, "tokens": 4047503360, "loss": 4.826503813266754, "l_mix": 2.986364722251892, "l_a": 3.084577888250351, "l_c": 3.0438379645347595, "z_loss": 16.14243769645691, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22656400501728058, "tokens_per_second": 1257327.977911514, "eta_seconds": 27003.587916970253, "data_wait_seconds": 2.288818359375e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20764, "cosmopedia_v2": 0.20039, "finemath": 0.08238}}
{"step": 7730, "total_steps": 72479, "tokens": 4052746240, "loss": 3.2995974123477936, "l_mix": 2.0376532673835754, "l_a": 2.121397614479065, "l_c": 2.0793616324663162, "z_loss": 17.16361379623413, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.255073606967926, "tokens_per_second": 1243060.7843805836, "eta_seconds": 27309.303083610535, "data_wait_seconds": 3.489851951599121e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20763, "cosmopedia_v2": 0.20039, "finemath": 0.08241}}
{"step": 7740, "total_steps": 72479, "tokens": 4057989120, "loss": 3.6565022468566895, "l_mix": 2.245021790266037, "l_a": 2.3861473500728607, "l_c": 2.3136242628097534, "z_loss": 15.489442467689514, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2563784420490265, "tokens_per_second": 1225547.9604690003, "eta_seconds": 27695.26932182312, "data_wait_seconds": 3.9011240005493164e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20762, "cosmopedia_v2": 0.20052, "finemath": 0.0823}}
{"step": 7750, "total_steps": 72479, "tokens": 4063232000, "loss": 5.041460692882538, "l_mix": 3.1214252710342407, "l_a": 3.2121328115463257, "l_c": 3.1825983226299286, "z_loss": 16.15862727165222, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16266091167926788, "tokens_per_second": 1457409.1142393197, "eta_seconds": 23285.59470393658, "data_wait_seconds": 4.178285598754883e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20774, "cosmopedia_v2": 0.20039, "finemath": 0.08232}}
{"step": 7760, "total_steps": 72479, "tokens": 4068474880, "loss": 4.990147769451141, "l_mix": 3.090150862932205, "l_a": 3.1868561804294586, "l_c": 3.1411540508270264, "z_loss": 15.938855767250061, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1918928027153015, "tokens_per_second": 1445675.143566383, "eta_seconds": 23470.968026947976, "data_wait_seconds": 2.199411392211914e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20773, "cosmopedia_v2": 0.20039, "finemath": 0.08235}}
{"step": 7770, "total_steps": 72479, "tokens": 4073717760, "loss": 5.135913610458374, "l_mix": 3.1817888021469116, "l_a": 3.269933730363846, "l_c": 3.2383619844913483, "z_loss": 16.360262393951416, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1839287430047989, "tokens_per_second": 1311279.6920177066, "eta_seconds": 25872.552132487297, "data_wait_seconds": 3.11732292175293e-05, "mix": {"fineweb_hq": 0.50952, "dclm": 0.20772, "cosmopedia_v2": 0.20039, "finemath": 0.08237}}
{"step": 7780, "total_steps": 72479, "tokens": 4078960640, "loss": 5.12224268913269, "l_mix": 3.1717068552970886, "l_a": 3.262965202331543, "l_c": 3.2335087954998016, "z_loss": 15.935320496559143, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22294625639915466, "tokens_per_second": 1395817.2487368274, "eta_seconds": 24301.827006864547, "data_wait_seconds": 3.5256147384643555e-05, "mix": {"fineweb_hq": 0.50951, "dclm": 0.20771, "cosmopedia_v2": 0.20039, "finemath": 0.08239}}
{"step": 7790, "total_steps": 72479, "tokens": 4084203520, "loss": 4.859314322471619, "l_mix": 3.0068823993206024, "l_a": 3.1095166206359863, "l_c": 3.0598475337028503, "z_loss": 16.225987315177917, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17771469056606293, "tokens_per_second": 1465381.9702295375, "eta_seconds": 23144.591049313545, "data_wait_seconds": 3.445148468017578e-05, "mix": {"fineweb_hq": 0.50963, "dclm": 0.2077, "cosmopedia_v2": 0.20039, "finemath": 0.08228}}
{"step": 7800, "total_steps": 72479, "tokens": 4089446400, "loss": 4.818958342075348, "l_mix": 2.9813036620616913, "l_a": 3.0841937959194183, "l_c": 3.0358650386333466, "z_loss": 16.370410919189453, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1586901694536209, "tokens_per_second": 1460630.8394241016, "eta_seconds": 23216.286166715625, "data_wait_seconds": 1.8209218978881836e-05, "mix": {"fineweb_hq": 0.50962, "dclm": 0.20769, "cosmopedia_v2": 0.20038, "finemath": 0.08231}, "gates": {"gate_a2b/L1_absmean": 0.15918091, "gate_a2b/L1_absmax": 0.60148132, "gate_a2b/L1_rms": 0.20016885, "gate_a2b/L3_absmean": 0.27699992, "gate_a2b/L3_absmax": 0.46055809, "gate_a2b/L3_rms": 0.28363982, "gate_a2b/L5_absmean": 0.45934525, "gate_a2b/L5_absmax": 0.65555531, "gate_a2b/L5_rms": 0.46815068, "gate_c2b/L1_absmean": 0.01722392, "gate_c2b/L1_absmax": 0.57385647, "gate_c2b/L1_rms": 0.06439052, "gate_c2b/L3_absmean": 0.22504579, "gate_c2b/L3_absmax": 0.46380532, "gate_c2b/L3_rms": 0.23174925, "gate_c2b/L5_absmean": 0.54720432, "gate_c2b/L5_absmax": 0.7831828, "gate_c2b/L5_rms": 0.5619086, "gate_b2a/L7_absmean": 0.06789072, "gate_b2a/L7_absmax": 0.25182277, "gate_b2a/L7_rms": 0.07332679, "gate_b2a/L11_absmean": 0.11696469, "gate_b2a/L11_absmax": 0.28719404, "gate_b2a/L11_rms": 0.11904973, "gate_b2a/L14_absmean": 0.2821413, "gate_b2a/L14_absmax": 0.4556084, "gate_b2a/L14_rms": 0.28325143, "gate_b2c/L3_absmean": 0.04894653, "gate_b2c/L3_absmax": 0.456348, "gate_b2c/L3_rms": 0.07015315, "gate_b2c/L5_absmean": 0.09035396, "gate_b2c/L5_absmax": 0.5070256, "gate_b2c/L5_rms": 0.09495421, "gate_b2c/L6_absmean": 0.14029197, "gate_b2c/L6_absmax": 0.38170272, "gate_b2c/L6_rms": 0.1423035}, "streams": {"cos_a2b/round0": 0.371216, "cos_c2b/round0": 0.245981, "cos_b2a/round0": 0.191193, "cos_b2c/round0": 0.117478, "rms_a/round0": 1.153183, "rms_b/round0": 2.110762, "rms_c/round0": 1.319832, "cos_a2b/round1": 0.39951, "cos_c2b/round1": 0.280322, "cos_b2a/round1": 0.214087, "cos_b2c/round1": 0.168991, "rms_a/round1": 2.064493, "rms_b/round1": 3.778487, "rms_c/round1": 2.162041, "cos_a2b/round2": 0.519026, "cos_c2b/round2": 0.425179, "cos_b2a/round2": 0.077258, "cos_b2c/round2": 0.14621, "rms_a/round2": 4.170494, "rms_b/round2": 7.613499, "rms_c/round2": 3.346073}, "alpha_mean": 0.4954225046184746, "alpha_hist": [0.0083, 0.03587, 0.097, 0.17032, 0.19098, 0.19574, 0.16618, 0.11221, 0.02299, 0.0004], "alpha_by_type": {"special": {"share": 0.00063, "mean": 0.57304, "std": 0.10193}, "whitespace": {"share": 0.02052, "mean": 0.57143, "std": 0.12588}, "digit": {"share": 0.03743, "mean": 0.51589, "std": 0.14532}, "word_start": {"share": 0.58838, "mean": 0.51726, "std": 0.15596}, "word_cont": {"share": 0.23944, "mean": 0.36795, "std": 0.14886}, "punct": {"share": 0.1136, "mean": 0.63007, "std": 0.15269}}, "alpha_p05": 0.2053, "alpha_p50": 0.4957, "alpha_p95": 0.74909}
{"step": 7810, "total_steps": 72479, "tokens": 4094689280, "loss": 4.894348382949829, "l_mix": 3.0281134843826294, "l_a": 3.131677597761154, "l_c": 3.0837262570858, "z_loss": 16.136154413223267, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16178308427333832, "tokens_per_second": 1454991.517966871, "eta_seconds": 23302.665516138077, "data_wait_seconds": 2.065300941467285e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20768, "cosmopedia_v2": 0.20038, "finemath": 0.08233}}
{"step": 7820, "total_steps": 72479, "tokens": 4099932160, "loss": 4.7507142424583435, "l_mix": 2.9386237263679504, "l_a": 3.043918788433075, "l_c": 2.9910440742969513, "z_loss": 16.016358971595764, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15618814527988434, "tokens_per_second": 1419247.14708008, "eta_seconds": 23885.859388017656, "data_wait_seconds": 2.2172927856445312e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20767, "cosmopedia_v2": 0.20038, "finemath": 0.08235}}
{"step": 7830, "total_steps": 72479, "tokens": 4105175040, "loss": 4.836669683456421, "l_mix": 2.992159217596054, "l_a": 3.0973226130008698, "l_c": 3.045641303062439, "z_loss": 16.211050152778625, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17444482445716858, "tokens_per_second": 1343322.104618334, "eta_seconds": 25231.993723225594, "data_wait_seconds": 1.862645149230957e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20038, "finemath": 0.08238}}
{"step": 7840, "total_steps": 72479, "tokens": 4110417920, "loss": 3.441381722688675, "l_mix": 2.1268904209136963, "l_a": 2.2046961337327957, "l_c": 2.170821890234947, "z_loss": 18.35765242576599, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.44425898790359497, "tokens_per_second": 1419981.3186624576, "eta_seconds": 23866.125269818305, "data_wait_seconds": 3.221631050109863e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20038, "finemath": 0.0824}}
{"step": 7850, "total_steps": 72479, "tokens": 4115660800, "loss": 3.676979750394821, "l_mix": 2.2587465047836304, "l_a": 2.3914476335048676, "l_c": 2.330906569957733, "z_loss": 15.269424557685852, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21921642124652863, "tokens_per_second": 1377399.4106947575, "eta_seconds": 24600.13333017826, "data_wait_seconds": 3.49581241607666e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20764, "cosmopedia_v2": 0.20051, "finemath": 0.08229}}
{"step": 7860, "total_steps": 72479, "tokens": 4120903680, "loss": 3.6699717044830322, "l_mix": 2.254549950361252, "l_a": 2.3902081847190857, "l_c": 2.3227758705615997, "z_loss": 15.265118837356567, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.203366219997406, "tokens_per_second": 1451931.1903821502, "eta_seconds": 23333.727174139025, "data_wait_seconds": 2.002716064453125e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20763, "cosmopedia_v2": 0.20051, "finemath": 0.08232}}
{"step": 7870, "total_steps": 72479, "tokens": 4126146560, "loss": 3.645614415407181, "l_mix": 2.2396745085716248, "l_a": 2.36867892742157, "l_c": 2.3126431703567505, "z_loss": 15.432056188583374, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1889466643333435, "tokens_per_second": 1340666.657004702, "eta_seconds": 25266.327923512457, "data_wait_seconds": 2.422928810119629e-05, "mix": {"fineweb_hq": 0.50953, "dclm": 0.20762, "cosmopedia_v2": 0.20051, "finemath": 0.08234}}
{"step": 7880, "total_steps": 72479, "tokens": 4131389440, "loss": 4.850272059440613, "l_mix": 3.0004480183124542, "l_a": 3.106608361005783, "l_c": 3.053989499807358, "z_loss": 16.445374250411987, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16134808957576752, "tokens_per_second": 1470938.3420820574, "eta_seconds": 23025.08510591984, "data_wait_seconds": 1.4930963516235352e-05, "mix": {"fineweb_hq": 0.50964, "dclm": 0.20761, "cosmopedia_v2": 0.20038, "finemath": 0.08236}}
{"step": 7890, "total_steps": 72479, "tokens": 4136632320, "loss": 4.878204107284546, "l_mix": 3.0175799429416656, "l_a": 3.1240043342113495, "l_c": 3.0725907683372498, "z_loss": 16.455424070358276, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16961267590522766, "tokens_per_second": 1365846.2517805037, "eta_seconds": 24792.861998820306, "data_wait_seconds": 2.4110078811645508e-05, "mix": {"fineweb_hq": 0.50963, "dclm": 0.2076, "cosmopedia_v2": 0.20038, "finemath": 0.08238}}
{"step": 7900, "total_steps": 72479, "tokens": 4141875200, "loss": 4.922515690326691, "l_mix": 3.0458612740039825, "l_a": 3.1521223187446594, "l_c": 3.0980902314186096, "z_loss": 15.905679702758789, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1769145131111145, "tokens_per_second": 1408460.9985338454, "eta_seconds": 24039.000573849677, "data_wait_seconds": 2.0295381546020508e-05, "mix": {"fineweb_hq": 0.50962, "dclm": 0.20759, "cosmopedia_v2": 0.20038, "finemath": 0.08241}, "gates": {"gate_a2b/L1_absmean": 0.16061735, "gate_a2b/L1_absmax": 0.60134858, "gate_a2b/L1_rms": 0.20120612, "gate_a2b/L3_absmean": 0.27723497, "gate_a2b/L3_absmax": 0.45936862, "gate_a2b/L3_rms": 0.28393126, "gate_a2b/L5_absmean": 0.46007743, "gate_a2b/L5_absmax": 0.65221548, "gate_a2b/L5_rms": 0.46876627, "gate_c2b/L1_absmean": 0.01587723, "gate_c2b/L1_absmax": 0.57186246, "gate_c2b/L1_rms": 0.06428067, "gate_c2b/L3_absmean": 0.22485809, "gate_c2b/L3_absmax": 0.45703986, "gate_c2b/L3_rms": 0.23145077, "gate_c2b/L5_absmean": 0.54743707, "gate_c2b/L5_absmax": 0.78348863, "gate_c2b/L5_rms": 0.562105, "gate_b2a/L7_absmean": 0.06918584, "gate_b2a/L7_absmax": 0.25148395, "gate_b2a/L7_rms": 0.07443101, "gate_b2a/L11_absmean": 0.11679026, "gate_b2a/L11_absmax": 0.29300675, "gate_b2a/L11_rms": 0.11890284, "gate_b2a/L14_absmean": 0.28337273, "gate_b2a/L14_absmax": 0.45665592, "gate_b2a/L14_rms": 0.28446463, "gate_b2c/L3_absmean": 0.04957771, "gate_b2c/L3_absmax": 0.45653489, "gate_b2c/L3_rms": 0.07074559, "gate_b2c/L5_absmean": 0.09120231, "gate_b2c/L5_absmax": 0.51161915, "gate_b2c/L5_rms": 0.09571206, "gate_b2c/L6_absmean": 0.14086366, "gate_b2c/L6_absmax": 0.38417169, "gate_b2c/L6_rms": 0.14283052}, "streams": {"cos_a2b/round0": 0.376906, "cos_c2b/round0": 0.25607, "cos_b2a/round0": 0.192652, "cos_b2c/round0": 0.124317, "rms_a/round0": 1.132971, "rms_b/round0": 2.087185, "rms_c/round0": 1.316062, "cos_a2b/round1": 0.392891, "cos_c2b/round1": 0.269811, "cos_b2a/round1": 0.211515, "cos_b2c/round1": 0.167511, "rms_a/round1": 2.036294, "rms_b/round1": 3.749539, "rms_c/round1": 2.171173, "cos_a2b/round2": 0.497434, "cos_c2b/round2": 0.44545, "cos_b2a/round2": 0.079456, "cos_b2c/round2": 0.136501, "rms_a/round2": 4.076709, "rms_b/round2": 7.746032, "rms_c/round2": 3.404709}, "alpha_mean": 0.47353138828358965, "alpha_hist": [0.0098, 0.04446, 0.12155, 0.17947, 0.19307, 0.19191, 0.15848, 0.08398, 0.01704, 0.00023], "alpha_by_type": {"special": {"share": 0.00089, "mean": 0.62363, "std": 0.09258}, "whitespace": {"share": 0.03159, "mean": 0.57731, "std": 0.11738}, "digit": {"share": 0.05885, "mean": 0.5277, "std": 0.13707}, "word_start": {"share": 0.55383, "mean": 0.49527, "std": 0.15833}, "word_cont": {"share": 0.25113, "mean": 0.34471, "std": 0.13887}, "punct": {"share": 0.10371, "mean": 0.60577, "std": 0.14954}}, "alpha_p05": 0.20025, "alpha_p50": 0.47561, "alpha_p95": 0.73259}
{"step": 7910, "total_steps": 72479, "tokens": 4147118080, "loss": 4.803565144538879, "l_mix": 2.9703630208969116, "l_a": 3.078247755765915, "l_c": 3.026949554681778, "z_loss": 16.42789602279663, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17974762618541718, "tokens_per_second": 1417941.9770299697, "eta_seconds": 23874.567803478243, "data_wait_seconds": 1.7702579498291016e-05, "mix": {"fineweb_hq": 0.50961, "dclm": 0.20771, "cosmopedia_v2": 0.20038, "finemath": 0.0823}}
{"step": 7920, "total_steps": 72479, "tokens": 4152360960, "loss": 4.749809265136719, "l_mix": 2.937688112258911, "l_a": 3.041474461555481, "l_c": 2.9934716522693634, "z_loss": 16.37220537662506, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1702677607536316, "tokens_per_second": 1469725.0898280593, "eta_seconds": 23029.823214054108, "data_wait_seconds": 1.5854835510253906e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.2077, "cosmopedia_v2": 0.20038, "finemath": 0.08232}}
{"step": 7930, "total_steps": 72479, "tokens": 4157603840, "loss": 4.74520868062973, "l_mix": 2.934428036212921, "l_a": 3.0403075516223907, "l_c": 2.99021777510643, "z_loss": 16.22957742214203, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16268910467624664, "tokens_per_second": 1433504.3887012627, "eta_seconds": 23608.065924835206, "data_wait_seconds": 1.6897916793823242e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20769, "cosmopedia_v2": 0.20038, "finemath": 0.08235}}
{"step": 7940, "total_steps": 72479, "tokens": 4162846720, "loss": 4.809476971626282, "l_mix": 2.9754492044448853, "l_a": 3.0744870603084564, "l_c": 3.0335097312927246, "z_loss": 16.286154985427856, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20715078711509705, "tokens_per_second": 1236894.2793665212, "eta_seconds": 27356.439266037938, "data_wait_seconds": 1.7821788787841797e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20768, "cosmopedia_v2": 0.20038, "finemath": 0.08237}}
{"step": 7950, "total_steps": 72479, "tokens": 4168089600, "loss": 3.451200693845749, "l_mix": 2.132962226867676, "l_a": 2.2111413776874542, "l_c": 2.177197501063347, "z_loss": 17.368120431900024, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.3125539720058441, "tokens_per_second": 1292783.4671450993, "eta_seconds": 26169.719223523138, "data_wait_seconds": 4.202127456665039e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20038, "finemath": 0.08239}}
{"step": 7960, "total_steps": 72479, "tokens": 4173332480, "loss": 3.2343634366989136, "l_mix": 1.9980032742023468, "l_a": 2.081153228878975, "l_c": 2.0339338332414627, "z_loss": 18.34049081802368, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.34278103709220886, "tokens_per_second": 1320246.4743529975, "eta_seconds": 25621.38065059185, "data_wait_seconds": 2.6881694793701172e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20766, "cosmopedia_v2": 0.20038, "finemath": 0.08241}}
{"step": 7970, "total_steps": 72479, "tokens": 4178575360, "loss": 3.603060245513916, "l_mix": 2.2132753133773804, "l_a": 2.3457236886024475, "l_c": 2.2817244827747345, "z_loss": 15.504552125930786, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19496068358421326, "tokens_per_second": 1412413.5615908245, "eta_seconds": 23945.74472501278, "data_wait_seconds": 1.677870750427246e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20765, "cosmopedia_v2": 0.2005, "finemath": 0.08231}}
{"step": 7980, "total_steps": 72479, "tokens": 4183818240, "loss": 3.651292771100998, "l_mix": 2.2431447505950928, "l_a": 2.376616358757019, "l_c": 2.3121010661125183, "z_loss": 15.327515840530396, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21200324594974518, "tokens_per_second": 1306301.3526057971, "eta_seconds": 25886.86878762245, "data_wait_seconds": 4.1037797927856445e-05, "mix": {"fineweb_hq": 0.50952, "dclm": 0.20764, "cosmopedia_v2": 0.2005, "finemath": 0.08233}}
{"step": 7990, "total_steps": 72479, "tokens": 4189061120, "loss": 4.786806166172028, "l_mix": 2.960696667432785, "l_a": 3.065498799085617, "l_c": 3.0161028802394867, "z_loss": 16.290168046951294, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17021775245666504, "tokens_per_second": 1460435.7628258807, "eta_seconds": 23151.17836239338, "data_wait_seconds": 2.4080276489257812e-05, "mix": {"fineweb_hq": 0.50964, "dclm": 0.20763, "cosmopedia_v2": 0.20038, "finemath": 0.08235}}
{"step": 8000, "total_steps": 72479, "tokens": 4194304000, "loss": 4.897289395332336, "l_mix": 3.0313381254673004, "l_a": 3.1306534707546234, "l_c": 3.083834618330002, "z_loss": 16.048408150672913, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16084152460098267, "tokens_per_second": 1458559.0580845934, "eta_seconds": 23177.37205402851, "data_wait_seconds": 2.1398067474365234e-05, "mix": {"fineweb_hq": 0.50962, "dclm": 0.20763, "cosmopedia_v2": 0.20037, "finemath": 0.08238}, "gates": {"gate_a2b/L1_absmean": 0.16270542, "gate_a2b/L1_absmax": 0.60026914, "gate_a2b/L1_rms": 0.20304883, "gate_a2b/L3_absmean": 0.27753392, "gate_a2b/L3_absmax": 0.46280691, "gate_a2b/L3_rms": 0.28423083, "gate_a2b/L5_absmean": 0.46126539, "gate_a2b/L5_absmax": 0.65247381, "gate_a2b/L5_rms": 0.46997103, "gate_c2b/L1_absmean": 0.01612684, "gate_c2b/L1_absmax": 0.57056922, "gate_c2b/L1_rms": 0.06388152, "gate_c2b/L3_absmean": 0.22437787, "gate_c2b/L3_absmax": 0.45795846, "gate_c2b/L3_rms": 0.23095964, "gate_c2b/L5_absmean": 0.54773241, "gate_c2b/L5_absmax": 0.77743745, "gate_c2b/L5_rms": 0.56233954, "gate_b2a/L7_absmean": 0.06918308, "gate_b2a/L7_absmax": 0.24996039, "gate_b2a/L7_rms": 0.07434063, "gate_b2a/L11_absmean": 0.11716397, "gate_b2a/L11_absmax": 0.29247662, "gate_b2a/L11_rms": 0.11932492, "gate_b2a/L14_absmean": 0.28352895, "gate_b2a/L14_absmax": 0.45490119, "gate_b2a/L14_rms": 0.28464767, "gate_b2c/L3_absmean": 0.0501819, "gate_b2c/L3_absmax": 0.45852527, "gate_b2c/L3_rms": 0.07133406, "gate_b2c/L5_absmean": 0.09205931, "gate_b2c/L5_absmax": 0.51602072, "gate_b2c/L5_rms": 0.0966055, "gate_b2c/L6_absmean": 0.14140768, "gate_b2c/L6_absmax": 0.3880761, "gate_b2c/L6_rms": 0.14342858}, "streams": {"cos_a2b/round0": 0.37095, "cos_c2b/round0": 0.245986, "cos_b2a/round0": 0.191093, "cos_b2c/round0": 0.122266, "rms_a/round0": 1.137685, "rms_b/round0": 2.090367, "rms_c/round0": 1.316029, "cos_a2b/round1": 0.397551, "cos_c2b/round1": 0.280609, "cos_b2a/round1": 0.206367, "cos_b2c/round1": 0.162623, "rms_a/round1": 2.042825, "rms_b/round1": 3.714264, "rms_c/round1": 2.162566, "cos_a2b/round2": 0.503081, "cos_c2b/round2": 0.455079, "cos_b2a/round2": 0.09305, "cos_b2c/round2": 0.153045, "rms_a/round2": 4.07694, "rms_b/round2": 7.613042, "rms_c/round2": 3.368796}, "alpha_mean": 0.47189823328933755, "alpha_hist": [0.00629, 0.03894, 0.11633, 0.17317, 0.20692, 0.2319, 0.15436, 0.06282, 0.00905, 0.00021], "alpha_by_type": {"special": {"share": 0.00047, "mean": 0.51854, "std": 0.10113}, "whitespace": {"share": 0.02629, "mean": 0.53909, "std": 0.11905}, "digit": {"share": 0.04185, "mean": 0.53829, "std": 0.13857}, "word_start": {"share": 0.56831, "mean": 0.49173, "std": 0.13998}, "word_cont": {"share": 0.24783, "mean": 0.34878, "std": 0.13359}, "punct": {"share": 0.11523, "mean": 0.59925, "std": 0.14084}}, "alpha_p05": 0.20025, "alpha_p50": 0.47548, "alpha_p95": 0.72584}
{"step": 8010, "total_steps": 72479, "tokens": 4199546880, "loss": 4.754779160022736, "l_mix": 2.941577762365341, "l_a": 3.042694479227066, "l_c": 2.9958847761154175, "z_loss": 16.275436520576477, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16597296297550201, "tokens_per_second": 1426690.955116349, "eta_seconds": 23691.411900234223, "data_wait_seconds": 1.6003847122192383e-05, "mix": {"fineweb_hq": 0.50961, "dclm": 0.20762, "cosmopedia_v2": 0.20037, "finemath": 0.0824}}
{"step": 8020, "total_steps": 72479, "tokens": 4204789760, "loss": 4.899365961551666, "l_mix": 3.031455546617508, "l_a": 3.1349276304244995, "l_c": 3.0860825181007385, "z_loss": 16.072633743286133, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1894526183605194, "tokens_per_second": 1328799.6639979165, "eta_seconds": 25432.78803241253, "data_wait_seconds": 1.5407800674438477e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20761, "cosmopedia_v2": 0.20037, "finemath": 0.08242}}
{"step": 8030, "total_steps": 72479, "tokens": 4210032640, "loss": 5.109929323196411, "l_mix": 3.1643496453762054, "l_a": 3.2581754326820374, "l_c": 3.221615821123123, "z_loss": 16.422362804412842, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.14287036657333374, "tokens_per_second": 1360259.2466816737, "eta_seconds": 24840.733407568932, "data_wait_seconds": 2.086162567138672e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20772, "cosmopedia_v2": 0.20037, "finemath": 0.08232}}
{"step": 8040, "total_steps": 72479, "tokens": 4215275520, "loss": 5.137504816055298, "l_mix": 3.1814508736133575, "l_a": 3.277758449316025, "l_c": 3.2368913292884827, "z_loss": 16.589925527572632, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1553346961736679, "tokens_per_second": 1444819.61466891, "eta_seconds": 23383.26119675636, "data_wait_seconds": 3.507733345031738e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20771, "cosmopedia_v2": 0.20037, "finemath": 0.08234}}
{"step": 8050, "total_steps": 72479, "tokens": 4220518400, "loss": 5.099085092544556, "l_mix": 3.1578889787197113, "l_a": 3.251235693693161, "l_c": 3.2140989899635315, "z_loss": 15.957387208938599, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20006787776947021, "tokens_per_second": 1457824.4739238466, "eta_seconds": 23171.069052696228, "data_wait_seconds": 3.102421760559082e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.2077, "cosmopedia_v2": 0.20037, "finemath": 0.08236}}
{"step": 8060, "total_steps": 72479, "tokens": 4225761280, "loss": 4.982289791107178, "l_mix": 3.08625528216362, "l_a": 3.1764550507068634, "l_c": 3.138231933116913, "z_loss": 16.284554600715637, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18866828083992004, "tokens_per_second": 1406477.438429367, "eta_seconds": 24013.260184049606, "data_wait_seconds": 2.4080276489257812e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20769, "cosmopedia_v2": 0.20037, "finemath": 0.08238}}
{"step": 8070, "total_steps": 72479, "tokens": 4231004160, "loss": 3.469005763530731, "l_mix": 2.1441486179828644, "l_a": 2.2272576987743378, "l_c": 2.1828213334083557, "z_loss": 18.334068536758423, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.24464593827724457, "tokens_per_second": 1437807.6349798748, "eta_seconds": 23486.35865497589, "data_wait_seconds": 2.518296241760254e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20768, "cosmopedia_v2": 0.20037, "finemath": 0.0824}}
{"step": 8080, "total_steps": 72479, "tokens": 4236247040, "loss": 3.574563592672348, "l_mix": 2.1951301395893097, "l_a": 2.3312176764011383, "l_c": 2.261683613061905, "z_loss": 15.62979280948639, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2354474514722824, "tokens_per_second": 1378202.3314700117, "eta_seconds": 24498.306337928774, "data_wait_seconds": 2.5093555450439453e-05, "mix": {"fineweb_hq": 0.50953, "dclm": 0.20767, "cosmopedia_v2": 0.2005, "finemath": 0.0823}}
{"step": 8090, "total_steps": 72479, "tokens": 4241489920, "loss": 3.618069738149643, "l_mix": 2.2234800159931183, "l_a": 2.3551548421382904, "l_c": 2.2881233990192413, "z_loss": 16.061869621276855, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22660429775714874, "tokens_per_second": 1407530.9055390973, "eta_seconds": 23984.11281709671, "data_wait_seconds": 2.5004148483276367e-05, "mix": {"fineweb_hq": 0.50952, "dclm": 0.20766, "cosmopedia_v2": 0.20049, "finemath": 0.08232}}
{"step": 8100, "total_steps": 72479, "tokens": 4246732800, "loss": 4.7926037311553955, "l_mix": 2.9636122286319733, "l_a": 3.0735231935977936, "l_c": 3.0176485776901245, "z_loss": 16.398420333862305, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16630354523658752, "tokens_per_second": 1227895.7403870572, "eta_seconds": 27488.601875400545, "data_wait_seconds": 3.084540367126465e-05, "mix": {"fineweb_hq": 0.50963, "dclm": 0.20765, "cosmopedia_v2": 0.20037, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.16540442, "gate_a2b/L1_absmax": 0.59958625, "gate_a2b/L1_rms": 0.20488355, "gate_a2b/L3_absmean": 0.27881029, "gate_a2b/L3_absmax": 0.46427259, "gate_a2b/L3_rms": 0.28543219, "gate_a2b/L5_absmean": 0.46232757, "gate_a2b/L5_absmax": 0.65265137, "gate_a2b/L5_rms": 0.47096846, "gate_c2b/L1_absmean": 0.01570708, "gate_c2b/L1_absmax": 0.57249504, "gate_c2b/L1_rms": 0.0641142, "gate_c2b/L3_absmean": 0.22409944, "gate_c2b/L3_absmax": 0.45788547, "gate_c2b/L3_rms": 0.23051594, "gate_c2b/L5_absmean": 0.54766142, "gate_c2b/L5_absmax": 0.78198349, "gate_c2b/L5_rms": 0.56218326, "gate_b2a/L7_absmean": 0.0697293, "gate_b2a/L7_absmax": 0.24937776, "gate_b2a/L7_rms": 0.07479792, "gate_b2a/L11_absmean": 0.11750188, "gate_b2a/L11_absmax": 0.29415274, "gate_b2a/L11_rms": 0.119608, "gate_b2a/L14_absmean": 0.28356606, "gate_b2a/L14_absmax": 0.45248082, "gate_b2a/L14_rms": 0.28464377, "gate_b2c/L3_absmean": 0.0509386, "gate_b2c/L3_absmax": 0.45886752, "gate_b2c/L3_rms": 0.07226705, "gate_b2c/L5_absmean": 0.0930989, "gate_b2c/L5_absmax": 0.5203982, "gate_b2c/L5_rms": 0.0975888, "gate_b2c/L6_absmean": 0.14266676, "gate_b2c/L6_absmax": 0.38826632, "gate_b2c/L6_rms": 0.14465393}, "streams": {"cos_a2b/round0": 0.3697, "cos_c2b/round0": 0.247323, "cos_b2a/round0": 0.190924, "cos_b2c/round0": 0.118957, "rms_a/round0": 1.125238, "rms_b/round0": 2.095003, "rms_c/round0": 1.328061, "cos_a2b/round1": 0.390957, "cos_c2b/round1": 0.270464, "cos_b2a/round1": 0.21497, "cos_b2c/round1": 0.163126, "rms_a/round1": 2.014481, "rms_b/round1": 3.735794, "rms_c/round1": 2.171118, "cos_a2b/round2": 0.502585, "cos_c2b/round2": 0.442123, "cos_b2a/round2": 0.073266, "cos_b2c/round2": 0.144168, "rms_a/round2": 4.028965, "rms_b/round2": 7.558203, "rms_c/round2": 3.378972}, "alpha_mean": 0.466213408172635, "alpha_hist": [0.0159, 0.0536, 0.12129, 0.17952, 0.1981, 0.18724, 0.13559, 0.08665, 0.02161, 0.00049], "alpha_by_type": {"special": {"share": 0.00133, "mean": 0.5781, "std": 0.12099}, "whitespace": {"share": 0.02533, "mean": 0.55437, "std": 0.11882}, "digit": {"share": 0.02811, "mean": 0.51384, "std": 0.1155}, "word_start": {"share": 0.59129, "mean": 0.48939, "std": 0.16368}, "word_cont": {"share": 0.25606, "mean": 0.3409, "std": 0.14768}, "punct": {"share": 0.09789, "mean": 0.61604, "std": 0.1634}}, "alpha_p05": 0.17724, "alpha_p50": 0.47615, "alpha_p95": 0.75815}
{"step": 8110, "total_steps": 72479, "tokens": 4251975680, "loss": 4.884961783885956, "l_mix": 3.0226244628429413, "l_a": 3.1257196068763733, "l_c": 3.0767533481121063, "z_loss": 15.954462766647339, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17085489630699158, "tokens_per_second": 1348686.519902422, "eta_seconds": 25022.786076664925, "data_wait_seconds": 2.154707908630371e-05, "mix": {"fineweb_hq": 0.50962, "dclm": 0.20764, "cosmopedia_v2": 0.20037, "finemath": 0.08237}}
{"step": 8120, "total_steps": 72479, "tokens": 4257218560, "loss": 4.746740400791168, "l_mix": 2.936182379722595, "l_a": 3.040748804807663, "l_c": 2.989040791988373, "z_loss": 16.210899829864502, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1591414362192154, "tokens_per_second": 1395189.0125186278, "eta_seconds": 24185.003672790524, "data_wait_seconds": 2.2083520889282227e-05, "mix": {"fineweb_hq": 0.50961, "dclm": 0.20764, "cosmopedia_v2": 0.20037, "finemath": 0.08239}}
{"step": 8130, "total_steps": 72479, "tokens": 4262461440, "loss": 4.851492822170258, "l_mix": 3.00162672996521, "l_a": 3.105674237012863, "l_c": 3.055055409669876, "z_loss": 16.47114658355713, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21854306757450104, "tokens_per_second": 1431981.4105851024, "eta_seconds": 23559.94865758419, "data_wait_seconds": 1.6808509826660156e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20763, "cosmopedia_v2": 0.20037, "finemath": 0.08241}}
{"step": 8140, "total_steps": 72479, "tokens": 4267704320, "loss": 3.631751835346222, "l_mix": 2.2309209406375885, "l_a": 2.3648601472377777, "l_c": 2.299274116754532, "z_loss": 15.906499981880188, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19315209984779358, "tokens_per_second": 1468159.8000279341, "eta_seconds": 22975.81341714859, "data_wait_seconds": 2.136826515197754e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20762, "cosmopedia_v2": 0.20049, "finemath": 0.08231}}
{"step": 8150, "total_steps": 72479, "tokens": 4272947200, "loss": 3.654584765434265, "l_mix": 2.2462675273418427, "l_a": 2.373024731874466, "l_c": 2.3160134851932526, "z_loss": 16.05698001384735, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2731773555278778, "tokens_per_second": 1435368.3792878077, "eta_seconds": 23497.050122237204, "data_wait_seconds": 1.6748905181884766e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20761, "cosmopedia_v2": 0.20049, "finemath": 0.08233}}
{"step": 8160, "total_steps": 72479, "tokens": 4278190080, "loss": 3.6105798184871674, "l_mix": 2.2179428935050964, "l_a": 2.3522503674030304, "l_c": 2.284671127796173, "z_loss": 15.60443902015686, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17151996493339539, "tokens_per_second": 1316109.3344034464, "eta_seconds": 25622.248084187508, "data_wait_seconds": 2.7865171432495117e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.2076, "cosmopedia_v2": 0.20049, "finemath": 0.08235}}
{"step": 8170, "total_steps": 72479, "tokens": 4283432960, "loss": 5.147309243679047, "l_mix": 3.1881116032600403, "l_a": 3.279403418302536, "l_c": 3.245704174041748, "z_loss": 16.654931783676147, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16568054258823395, "tokens_per_second": 1429231.4686741643, "eta_seconds": 23590.60637202263, "data_wait_seconds": 2.4139881134033203e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20771, "cosmopedia_v2": 0.20037, "finemath": 0.08237}}
{"step": 8180, "total_steps": 72479, "tokens": 4288675840, "loss": 5.080366432666779, "l_mix": 3.1471490561962128, "l_a": 3.2343567609786987, "l_c": 3.204130232334137, "z_loss": 16.711438179016113, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17946043610572815, "tokens_per_second": 1346078.3933025626, "eta_seconds": 25044.005074095727, "data_wait_seconds": 2.2709369659423828e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.2077, "cosmopedia_v2": 0.20037, "finemath": 0.0824}}
{"step": 8190, "total_steps": 72479, "tokens": 4293918720, "loss": 3.677814394235611, "l_mix": 2.2604490518569946, "l_a": 2.3897128105163574, "l_c": 2.3295819759368896, "z_loss": 15.76893937587738, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.23017942905426025, "tokens_per_second": 1416503.9683283626, "eta_seconds": 23795.168941020966, "data_wait_seconds": 2.5391578674316406e-05, "mix": {"fineweb_hq": 0.50952, "dclm": 0.20769, "cosmopedia_v2": 0.20049, "finemath": 0.0823}}
{"step": 8200, "total_steps": 72479, "tokens": 4299161600, "loss": 3.6099480986595154, "l_mix": 2.217668890953064, "l_a": 2.351161777973175, "l_c": 2.2845743596553802, "z_loss": 15.583490252494812, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20376864075660706, "tokens_per_second": 1468502.4615700843, "eta_seconds": 22949.030889582635, "data_wait_seconds": 1.8864870071411133e-05, "mix": {"fineweb_hq": 0.50951, "dclm": 0.20768, "cosmopedia_v2": 0.20049, "finemath": 0.08232}, "gates": {"gate_a2b/L1_absmean": 0.16697331, "gate_a2b/L1_absmax": 0.59662235, "gate_a2b/L1_rms": 0.20667286, "gate_a2b/L3_absmean": 0.27973062, "gate_a2b/L3_absmax": 0.46540129, "gate_a2b/L3_rms": 0.28628987, "gate_a2b/L5_absmean": 0.46431446, "gate_a2b/L5_absmax": 0.65768427, "gate_a2b/L5_rms": 0.4728674, "gate_c2b/L1_absmean": 0.01595279, "gate_c2b/L1_absmax": 0.56570613, "gate_c2b/L1_rms": 0.06416629, "gate_c2b/L3_absmean": 0.22378008, "gate_c2b/L3_absmax": 0.45463744, "gate_c2b/L3_rms": 0.23012018, "gate_c2b/L5_absmean": 0.54802305, "gate_c2b/L5_absmax": 0.7796157, "gate_c2b/L5_rms": 0.56238383, "gate_b2a/L7_absmean": 0.07007428, "gate_b2a/L7_absmax": 0.24909106, "gate_b2a/L7_rms": 0.07499956, "gate_b2a/L11_absmean": 0.11823799, "gate_b2a/L11_absmax": 0.29936886, "gate_b2a/L11_rms": 0.12033045, "gate_b2a/L14_absmean": 0.28410929, "gate_b2a/L14_absmax": 0.44807351, "gate_b2a/L14_rms": 0.28515932, "gate_b2c/L3_absmean": 0.05203838, "gate_b2c/L3_absmax": 0.45883343, "gate_b2c/L3_rms": 0.07319897, "gate_b2c/L5_absmean": 0.09337471, "gate_b2c/L5_absmax": 0.52272749, "gate_b2c/L5_rms": 0.09797014, "gate_b2c/L6_absmean": 0.14367488, "gate_b2c/L6_absmax": 0.38677612, "gate_b2c/L6_rms": 0.14564288}, "streams": {"cos_a2b/round0": 0.347222, "cos_c2b/round0": 0.210094, "cos_b2a/round0": 0.184168, "cos_b2c/round0": 0.070344, "rms_a/round0": 1.1446, "rms_b/round0": 2.067793, "rms_c/round0": 1.33929, "cos_a2b/round1": 0.38236, "cos_c2b/round1": 0.268613, "cos_b2a/round1": 0.205597, "cos_b2c/round1": 0.160366, "rms_a/round1": 2.021563, "rms_b/round1": 3.763882, "rms_c/round1": 2.207831, "cos_a2b/round2": 0.47394, "cos_c2b/round2": 0.432061, "cos_b2a/round2": 0.034753, "cos_b2c/round2": 0.113405, "rms_a/round2": 3.981351, "rms_b/round2": 7.640898, "rms_c/round2": 3.465294}, "alpha_mean": 0.4645189000158183, "alpha_hist": [0.03458, 0.11047, 0.14954, 0.1489, 0.13025, 0.10893, 0.10533, 0.11786, 0.08264, 0.01151], "alpha_by_type": {"special": {"share": 0.0011, "mean": 0.7253, "std": 0.08709}, "whitespace": {"share": 0.03024, "mean": 0.71065, "std": 0.16945}, "digit": {"share": 0.00867, "mean": 0.47413, "std": 0.16837}, "word_start": {"share": 0.57954, "mean": 0.48158, "std": 0.21601}, "word_cont": {"share": 0.26572, "mean": 0.30548, "std": 0.17258}, "punct": {"share": 0.11473, "mean": 0.67856, "std": 0.18037}}, "alpha_p05": 0.13006, "alpha_p50": 0.43158, "alpha_p95": 0.82559}
{"step": 8210, "total_steps": 72479, "tokens": 4304404480, "loss": 4.926933288574219, "l_mix": 3.0485197603702545, "l_a": 3.151299148797989, "l_c": 3.104640245437622, "z_loss": 16.31672978401184, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18475396931171417, "tokens_per_second": 1461290.2754665846, "eta_seconds": 23058.707799339296, "data_wait_seconds": 1.817941665649414e-05, "mix": {"fineweb_hq": 0.50962, "dclm": 0.20767, "cosmopedia_v2": 0.20037, "finemath": 0.08234}}
{"step": 8220, "total_steps": 72479, "tokens": 4309647360, "loss": 4.885313153266907, "l_mix": 3.022656172513962, "l_a": 3.1289592683315277, "l_c": 3.074528932571411, "z_loss": 16.104307651519775, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.14992685616016388, "tokens_per_second": 1403088.5714393272, "eta_seconds": 24011.472459959983, "data_wait_seconds": 2.1398067474365234e-05, "mix": {"fineweb_hq": 0.50961, "dclm": 0.20766, "cosmopedia_v2": 0.20036, "finemath": 0.08236}}
{"step": 8230, "total_steps": 72479, "tokens": 4314890240, "loss": 4.7900601625442505, "l_mix": 2.9638327062129974, "l_a": 3.064953774213791, "l_c": 3.017083376646042, "z_loss": 16.16176128387451, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18807263672351837, "tokens_per_second": 1477351.7158677585, "eta_seconds": 22800.92096567154, "data_wait_seconds": 1.6957521438598633e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20765, "cosmopedia_v2": 0.20036, "finemath": 0.08238}}
{"step": 8240, "total_steps": 72479, "tokens": 4320133120, "loss": 4.794640839099884, "l_mix": 2.967087000608444, "l_a": 3.06802037358284, "l_c": 3.0185151398181915, "z_loss": 15.9310302734375, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18758320808410645, "tokens_per_second": 1453969.0790872795, "eta_seconds": 23163.997994470596, "data_wait_seconds": 1.6987323760986328e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20765, "cosmopedia_v2": 0.20036, "finemath": 0.0824}}
{"step": 8250, "total_steps": 72479, "tokens": 4325376000, "loss": 3.654418021440506, "l_mix": 2.2455686032772064, "l_a": 2.3807564675807953, "l_c": 2.31026491522789, "z_loss": 15.430138230323792, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21696439385414124, "tokens_per_second": 1381747.9470527829, "eta_seconds": 24370.938291478156, "data_wait_seconds": 2.7924776077270508e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20048, "finemath": 0.0823}}
{"step": 8260, "total_steps": 72479, "tokens": 4330618880, "loss": 3.6524338722229004, "l_mix": 2.2441790401935577, "l_a": 2.375206381082535, "l_c": 2.3136503994464874, "z_loss": 15.977957367897034, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19798073172569275, "tokens_per_second": 1421275.274099636, "eta_seconds": 23689.465148353574, "data_wait_seconds": 1.767277717590332e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20763, "cosmopedia_v2": 0.20048, "finemath": 0.08232}}
{"step": 8270, "total_steps": 72479, "tokens": 4335861760, "loss": 3.61347895860672, "l_mix": 2.220512330532074, "l_a": 2.3501163125038147, "l_c": 2.287834644317627, "z_loss": 15.813614010810852, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22267086803913116, "tokens_per_second": 1458587.984726311, "eta_seconds": 23079.861170196535, "data_wait_seconds": 2.0116567611694336e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20762, "cosmopedia_v2": 0.20048, "finemath": 0.08235}}
{"step": 8280, "total_steps": 72479, "tokens": 4341104640, "loss": 3.6328342854976654, "l_mix": 2.2315705120563507, "l_a": 2.3641116321086884, "l_c": 2.3017662167549133, "z_loss": 15.004065036773682, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.23510867357254028, "tokens_per_second": 1418339.990593519, "eta_seconds": 23731.09799852371, "data_wait_seconds": 2.1755695343017578e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20761, "cosmopedia_v2": 0.20048, "finemath": 0.08237}}
{"step": 8290, "total_steps": 72479, "tokens": 4346347520, "loss": 3.6184147894382477, "l_mix": 2.2234504222869873, "l_a": 2.354538232088089, "l_c": 2.290140748023987, "z_loss": 15.606301307678223, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20735850930213928, "tokens_per_second": 1401173.1343293472, "eta_seconds": 24018.104263830188, "data_wait_seconds": 4.050135612487793e-05, "mix": {"fineweb_hq": 0.50953, "dclm": 0.2076, "cosmopedia_v2": 0.20048, "finemath": 0.08239}}
{"step": 8300, "total_steps": 72479, "tokens": 4351590400, "loss": 3.6333151161670685, "l_mix": 2.2325500547885895, "l_a": 2.366947114467621, "l_c": 2.297073930501938, "z_loss": 15.58702290058136, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20577295124530792, "tokens_per_second": 1393565.69547255, "eta_seconds": 24145.45626468658, "data_wait_seconds": 2.428889274597168e-05, "mix": {"fineweb_hq": 0.50952, "dclm": 0.20771, "cosmopedia_v2": 0.20048, "finemath": 0.08229}, "gates": {"gate_a2b/L1_absmean": 0.16864149, "gate_a2b/L1_absmax": 0.59853292, "gate_a2b/L1_rms": 0.20846422, "gate_a2b/L3_absmean": 0.27996454, "gate_a2b/L3_absmax": 0.46770403, "gate_a2b/L3_rms": 0.28649789, "gate_a2b/L5_absmean": 0.46507794, "gate_a2b/L5_absmax": 0.65422755, "gate_a2b/L5_rms": 0.47361112, "gate_c2b/L1_absmean": 0.01606889, "gate_c2b/L1_absmax": 0.56280619, "gate_c2b/L1_rms": 0.06396251, "gate_c2b/L3_absmean": 0.222928, "gate_c2b/L3_absmax": 0.45980069, "gate_c2b/L3_rms": 0.22928111, "gate_c2b/L5_absmean": 0.54753596, "gate_c2b/L5_absmax": 0.78408474, "gate_c2b/L5_rms": 0.56184763, "gate_b2a/L7_absmean": 0.07006836, "gate_b2a/L7_absmax": 0.25101665, "gate_b2a/L7_rms": 0.07505707, "gate_b2a/L11_absmean": 0.11789812, "gate_b2a/L11_absmax": 0.30262539, "gate_b2a/L11_rms": 0.12004898, "gate_b2a/L14_absmean": 0.28378654, "gate_b2a/L14_absmax": 0.45157483, "gate_b2a/L14_rms": 0.2848525, "gate_b2c/L3_absmean": 0.05208096, "gate_b2c/L3_absmax": 0.46349877, "gate_b2c/L3_rms": 0.07335798, "gate_b2c/L5_absmean": 0.09424321, "gate_b2c/L5_absmax": 0.53005213, "gate_b2c/L5_rms": 0.0989309, "gate_b2c/L6_absmean": 0.14363888, "gate_b2c/L6_absmax": 0.38949293, "gate_b2c/L6_rms": 0.14566067}, "streams": {"cos_a2b/round0": 0.353752, "cos_c2b/round0": 0.215612, "cos_b2a/round0": 0.182499, "cos_b2c/round0": 0.072824, "rms_a/round0": 1.137889, "rms_b/round0": 2.062499, "rms_c/round0": 1.33081, "cos_a2b/round1": 0.378035, "cos_c2b/round1": 0.26215, "cos_b2a/round1": 0.20745, "cos_b2c/round1": 0.1554, "rms_a/round1": 2.00424, "rms_b/round1": 3.784174, "rms_c/round1": 2.213865, "cos_a2b/round2": 0.468836, "cos_c2b/round2": 0.439018, "cos_b2a/round2": 0.038856, "cos_b2c/round2": 0.110292, "rms_a/round2": 3.991904, "rms_b/round2": 7.808086, "rms_c/round2": 3.511207}, "alpha_mean": 0.4493657971286069, "alpha_hist": [0.04033, 0.12602, 0.16122, 0.14868, 0.12097, 0.10127, 0.09993, 0.11369, 0.07771, 0.01016], "alpha_by_type": {"special": {"share": 0.00102, "mean": 0.73119, "std": 0.08594}, "whitespace": {"share": 0.03622, "mean": 0.65637, "std": 0.18204}, "digit": {"share": 0.01431, "mean": 0.43864, "std": 0.1742}, "word_start": {"share": 0.54451, "mean": 0.46546, "std": 0.22149}, "word_cont": {"share": 0.27179, "mean": 0.28984, "std": 0.16727}, "punct": {"share": 0.13214, "mean": 0.65338, "std": 0.19248}}, "alpha_p05": 0.09912, "alpha_p50": 0.44119, "alpha_p95": 0.84897}
{"step": 8310, "total_steps": 72479, "tokens": 4356833280, "loss": 3.6101342141628265, "l_mix": 2.218327611684799, "l_a": 2.348348170518875, "l_c": 2.285737156867981, "z_loss": 15.809770345687866, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.23890544474124908, "tokens_per_second": 1320446.0932244298, "eta_seconds": 25478.538536810876, "data_wait_seconds": 1.7523765563964844e-05, "mix": {"fineweb_hq": 0.50951, "dclm": 0.2077, "cosmopedia_v2": 0.20048, "finemath": 0.08231}}
{"step": 8320, "total_steps": 72479, "tokens": 4362076160, "loss": 4.723054885864258, "l_mix": 2.9203543066978455, "l_a": 3.031353920698166, "l_c": 2.9720906913280487, "z_loss": 16.66990041732788, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1742831915616989, "tokens_per_second": 1470585.1018076837, "eta_seconds": 22873.748517274857, "data_wait_seconds": 1.946091651916504e-05, "mix": {"fineweb_hq": 0.50962, "dclm": 0.20769, "cosmopedia_v2": 0.20036, "finemath": 0.08233}}
{"step": 8330, "total_steps": 72479, "tokens": 4367319040, "loss": 4.715358197689056, "l_mix": 2.9177014231681824, "l_a": 3.0182437896728516, "l_c": 2.968503087759018, "z_loss": 16.32625699043274, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.14104090631008148, "tokens_per_second": 1345784.5477122394, "eta_seconds": 24991.03661813736, "data_wait_seconds": 1.7255544662475586e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20768, "cosmopedia_v2": 0.20036, "finemath": 0.08235}}
{"step": 8340, "total_steps": 72479, "tokens": 4372561920, "loss": 4.738034784793854, "l_mix": 2.9304319322109222, "l_a": 3.033561944961548, "l_c": 2.986310839653015, "z_loss": 16.4101402759552, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17051595449447632, "tokens_per_second": 1439643.6317732995, "eta_seconds": 23358.07785332203, "data_wait_seconds": 1.755356788635254e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20767, "cosmopedia_v2": 0.20036, "finemath": 0.08237}}
{"step": 8350, "total_steps": 72479, "tokens": 4377804800, "loss": 4.804512023925781, "l_mix": 2.972517669200897, "l_a": 3.0766645669937134, "l_c": 3.0244703590869904, "z_loss": 16.53883671760559, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2002570480108261, "tokens_per_second": 1426275.7529083493, "eta_seconds": 23573.32730605602, "data_wait_seconds": 3.501772880554199e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20036, "finemath": 0.0824}}
{"step": 8360, "total_steps": 72479, "tokens": 4383047680, "loss": 3.6244903802871704, "l_mix": 2.226913660764694, "l_a": 2.358110338449478, "l_c": 2.2952955961227417, "z_loss": 15.548680543899536, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2063254863023758, "tokens_per_second": 1357870.2789742465, "eta_seconds": 24757.020455145834, "data_wait_seconds": 2.3484230041503906e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20048, "finemath": 0.0823}}
{"step": 8370, "total_steps": 72479, "tokens": 4388290560, "loss": 3.6361456215381622, "l_mix": 2.2349810004234314, "l_a": 2.3620761334896088, "l_c": 2.303206741809845, "z_loss": 15.797057747840881, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21738895773887634, "tokens_per_second": 1416484.0773491326, "eta_seconds": 23728.87908129692, "data_wait_seconds": 1.6689300537109375e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20048, "finemath": 0.08232}}
{"step": 8380, "total_steps": 72479, "tokens": 4393533440, "loss": 3.613871932029724, "l_mix": 2.2201068997383118, "l_a": 2.352985769510269, "l_c": 2.287663161754608, "z_loss": 15.702330112457275, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22184957563877106, "tokens_per_second": 1439240.2616706162, "eta_seconds": 23350.05308494568, "data_wait_seconds": 1.9282102584838867e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20764, "cosmopedia_v2": 0.20048, "finemath": 0.08234}}
{"step": 8390, "total_steps": 72479, "tokens": 4398776320, "loss": 3.677072197198868, "l_mix": 2.2594954669475555, "l_a": 2.3940528333187103, "l_c": 2.326078027486801, "z_loss": 15.374261856079102, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.23829130828380585, "tokens_per_second": 1451044.688702737, "eta_seconds": 23156.48435475826, "data_wait_seconds": 2.1696090698242188e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20763, "cosmopedia_v2": 0.20048, "finemath": 0.08236}}
{"step": 8400, "total_steps": 72479, "tokens": 4404019200, "loss": 3.626739114522934, "l_mix": 2.227558434009552, "l_a": 2.361564725637436, "l_c": 2.297284871339798, "z_loss": 15.257510423660278, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20460768043994904, "tokens_per_second": 1335041.0358691802, "eta_seconds": 25164.657751607894, "data_wait_seconds": 2.911686897277832e-05, "mix": {"fineweb_hq": 0.50952, "dclm": 0.20762, "cosmopedia_v2": 0.20048, "finemath": 0.08238}, "gates": {"gate_a2b/L1_absmean": 0.17079595, "gate_a2b/L1_absmax": 0.59653407, "gate_a2b/L1_rms": 0.20996004, "gate_a2b/L3_absmean": 0.28132114, "gate_a2b/L3_absmax": 0.47120422, "gate_a2b/L3_rms": 0.28773934, "gate_a2b/L5_absmean": 0.46648484, "gate_a2b/L5_absmax": 0.65518844, "gate_a2b/L5_rms": 0.4749718, "gate_c2b/L1_absmean": 0.01568863, "gate_c2b/L1_absmax": 0.55974919, "gate_c2b/L1_rms": 0.06347276, "gate_c2b/L3_absmean": 0.22235756, "gate_c2b/L3_absmax": 0.45478934, "gate_c2b/L3_rms": 0.22859266, "gate_c2b/L5_absmean": 0.54800904, "gate_c2b/L5_absmax": 0.78343385, "gate_c2b/L5_rms": 0.562325, "gate_b2a/L7_absmean": 0.07022142, "gate_b2a/L7_absmax": 0.2490532, "gate_b2a/L7_rms": 0.07497159, "gate_b2a/L11_absmean": 0.11798952, "gate_b2a/L11_absmax": 0.30229646, "gate_b2a/L11_rms": 0.12011873, "gate_b2a/L14_absmean": 0.2846275, "gate_b2a/L14_absmax": 0.44986719, "gate_b2a/L14_rms": 0.28566733, "gate_b2c/L3_absmean": 0.05259509, "gate_b2c/L3_absmax": 0.46575677, "gate_b2c/L3_rms": 0.07413294, "gate_b2c/L5_absmean": 0.09410151, "gate_b2c/L5_absmax": 0.53724378, "gate_b2c/L5_rms": 0.09891914, "gate_b2c/L6_absmean": 0.14444333, "gate_b2c/L6_absmax": 0.39098954, "gate_b2c/L6_rms": 0.1464754}, "streams": {"cos_a2b/round0": 0.348696, "cos_c2b/round0": 0.208782, "cos_b2a/round0": 0.181417, "cos_b2c/round0": 0.072833, "rms_a/round0": 1.12943, "rms_b/round0": 2.047413, "rms_c/round0": 1.334654, "cos_a2b/round1": 0.37951, "cos_c2b/round1": 0.271363, "cos_b2a/round1": 0.207059, "cos_b2c/round1": 0.160436, "rms_a/round1": 2.002527, "rms_b/round1": 3.730006, "rms_c/round1": 2.233652, "cos_a2b/round2": 0.47743, "cos_c2b/round2": 0.432222, "cos_b2a/round2": 0.033322, "cos_b2c/round2": 0.112908, "rms_a/round2": 3.984753, "rms_b/round2": 7.665279, "rms_c/round2": 3.518956}, "alpha_mean": 0.4542085811415717, "alpha_hist": [0.04094, 0.11858, 0.157, 0.14403, 0.1216, 0.10568, 0.11159, 0.12105, 0.06894, 0.0106], "alpha_by_type": {"special": {"share": 0.00111, "mean": 0.72661, "std": 0.07508}, "whitespace": {"share": 0.02942, "mean": 0.70876, "std": 0.17448}, "digit": {"share": 0.00862, "mean": 0.51722, "std": 0.21134}, "word_start": {"share": 0.59035, "mean": 0.46946, "std": 0.21151}, "word_cont": {"share": 0.25403, "mean": 0.28534, "std": 0.17356}, "punct": {"share": 0.11647, "mean": 0.67366, "std": 0.17491}}, "alpha_p05": 0.11437, "alpha_p50": 0.42012, "alpha_p95": 0.80928}
{"step": 8410, "total_steps": 72479, "tokens": 4409262080, "loss": 4.837826251983643, "l_mix": 2.992651581764221, "l_a": 3.099946677684784, "l_c": 3.0451449155807495, "z_loss": 16.471901178359985, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21308954060077667, "tokens_per_second": 1437923.7460898075, "eta_seconds": 23360.49318563938, "data_wait_seconds": 1.8596649169921875e-05, "mix": {"fineweb_hq": 0.50963, "dclm": 0.20761, "cosmopedia_v2": 0.20048, "finemath": 0.08228}}
{"step": 8420, "total_steps": 72479, "tokens": 4414504960, "loss": 4.861790299415588, "l_mix": 3.0086879432201385, "l_a": 3.111663728952408, "l_c": 3.0598170161247253, "z_loss": 16.580689191818237, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2758455276489258, "tokens_per_second": 1437169.4998029876, "eta_seconds": 23369.105033612253, "data_wait_seconds": 4.857778549194336e-05, "mix": {"fineweb_hq": 0.50962, "dclm": 0.2076, "cosmopedia_v2": 0.20048, "finemath": 0.0823}}
{"step": 8430, "total_steps": 72479, "tokens": 4419747840, "loss": 4.800511360168457, "l_mix": 2.969844102859497, "l_a": 3.071364313364029, "l_c": 3.0251519680023193, "z_loss": 17.121659755706787, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.24850837886333466, "tokens_per_second": 1261192.3937838038, "eta_seconds": 26625.693492531776, "data_wait_seconds": 1.811981201171875e-05, "mix": {"fineweb_hq": 0.50961, "dclm": 0.20759, "cosmopedia_v2": 0.20047, "finemath": 0.08233}}
{"step": 8440, "total_steps": 72479, "tokens": 4424990720, "loss": 4.8738513588905334, "l_mix": 3.0153893530368805, "l_a": 3.119696468114853, "l_c": 3.06971338391304, "z_loss": 16.388914585113525, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1488591581583023, "tokens_per_second": 1456885.6867311515, "eta_seconds": 23045.65110206604, "data_wait_seconds": 1.4722347259521484e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.2077, "cosmopedia_v2": 0.20036, "finemath": 0.08235}}
{"step": 8450, "total_steps": 72479, "tokens": 4430233600, "loss": 4.800918459892273, "l_mix": 2.9687870144844055, "l_a": 3.075886607170105, "l_c": 3.0257960855960846, "z_loss": 16.264695525169373, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.163462832570076, "tokens_per_second": 1438718.788050242, "eta_seconds": 23333.00755562782, "data_wait_seconds": 1.4394521713256836e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20769, "cosmopedia_v2": 0.20036, "finemath": 0.08237}}
{"step": 8460, "total_steps": 72479, "tokens": 4435476480, "loss": 4.79396116733551, "l_mix": 2.9661201536655426, "l_a": 3.0671271979808807, "l_c": 3.0201061069965363, "z_loss": 16.710195302963257, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.172443687915802, "tokens_per_second": 1382353.4457149445, "eta_seconds": 24280.616202783585, "data_wait_seconds": 3.120303153991699e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20768, "cosmopedia_v2": 0.20035, "finemath": 0.08239}}
{"step": 8470, "total_steps": 72479, "tokens": 4440719360, "loss": 3.6484989523887634, "l_mix": 2.24078831076622, "l_a": 2.3775444328784943, "l_c": 2.309714764356613, "z_loss": 15.328145146369934, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.23816734552383423, "tokens_per_second": 1309128.0296718262, "eta_seconds": 25634.735359239578, "data_wait_seconds": 2.619624137878418e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20047, "finemath": 0.08229}}
{"step": 8480, "total_steps": 72479, "tokens": 4445962240, "loss": 3.67403382062912, "l_mix": 2.257863223552704, "l_a": 2.387509822845459, "l_c": 2.327873855829239, "z_loss": 15.554425597190857, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2463700920343399, "tokens_per_second": 1439390.5214695034, "eta_seconds": 23311.19123790264, "data_wait_seconds": 1.9341707229614258e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20047, "finemath": 0.08231}}
{"step": 8490, "total_steps": 72479, "tokens": 4451205120, "loss": 3.6427017152309418, "l_mix": 2.2370022237300873, "l_a": 2.3752837777137756, "l_c": 2.30526202917099, "z_loss": 15.358182549476624, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.23027312755584717, "tokens_per_second": 1438224.5939900244, "eta_seconds": 23326.443569517138, "data_wait_seconds": 2.1755695343017578e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20766, "cosmopedia_v2": 0.20047, "finemath": 0.08233}}
{"step": 8500, "total_steps": 72479, "tokens": 4456448000, "loss": 3.6554655134677887, "l_mix": 2.246269315481186, "l_a": 2.3802555203437805, "l_c": 2.311757504940033, "z_loss": 15.922428250312805, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2252693772315979, "tokens_per_second": 1457241.356345354, "eta_seconds": 23018.43946847916, "data_wait_seconds": 2.60770320892334e-05, "mix": {"fineweb_hq": 0.50953, "dclm": 0.20765, "cosmopedia_v2": 0.20047, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.1720984, "gate_a2b/L1_absmax": 0.59235662, "gate_a2b/L1_rms": 0.21047325, "gate_a2b/L3_absmean": 0.281995, "gate_a2b/L3_absmax": 0.46984613, "gate_a2b/L3_rms": 0.28846654, "gate_a2b/L5_absmean": 0.46741959, "gate_a2b/L5_absmax": 0.65391445, "gate_a2b/L5_rms": 0.47591868, "gate_c2b/L1_absmean": 0.01743594, "gate_c2b/L1_absmax": 0.56301939, "gate_c2b/L1_rms": 0.06413653, "gate_c2b/L3_absmean": 0.22241929, "gate_c2b/L3_absmax": 0.46762282, "gate_c2b/L3_rms": 0.22865441, "gate_c2b/L5_absmean": 0.54360723, "gate_c2b/L5_absmax": 0.76086712, "gate_c2b/L5_rms": 0.55791909, "gate_b2a/L7_absmean": 0.06986438, "gate_b2a/L7_absmax": 0.25114504, "gate_b2a/L7_rms": 0.07483848, "gate_b2a/L11_absmean": 0.11760043, "gate_b2a/L11_absmax": 0.30942702, "gate_b2a/L11_rms": 0.12004084, "gate_b2a/L14_absmean": 0.28237629, "gate_b2a/L14_absmax": 0.44395509, "gate_b2a/L14_rms": 0.28349403, "gate_b2c/L3_absmean": 0.05270359, "gate_b2c/L3_absmax": 0.4725419, "gate_b2c/L3_rms": 0.07351361, "gate_b2c/L5_absmean": 0.09510957, "gate_b2c/L5_absmax": 0.54783344, "gate_b2c/L5_rms": 0.10014131, "gate_b2c/L6_absmean": 0.14692748, "gate_b2c/L6_absmax": 0.39203703, "gate_b2c/L6_rms": 0.14905199}, "streams": {"cos_a2b/round0": 0.348949, "cos_c2b/round0": 0.208255, "cos_b2a/round0": 0.17812, "cos_b2c/round0": 0.072268, "rms_a/round0": 1.137213, "rms_b/round0": 2.039069, "rms_c/round0": 1.3304, "cos_a2b/round1": 0.384128, "cos_c2b/round1": 0.267259, "cos_b2a/round1": 0.206569, "cos_b2c/round1": 0.15665, "rms_a/round1": 2.015137, "rms_b/round1": 3.752541, "rms_c/round1": 2.197824, "cos_a2b/round2": 0.480731, "cos_c2b/round2": 0.438472, "cos_b2a/round2": 0.037827, "cos_b2c/round2": 0.117136, "rms_a/round2": 4.021585, "rms_b/round2": 7.807961, "rms_c/round2": 3.489592}, "alpha_mean": 0.47235751279692195, "alpha_hist": [0.04619, 0.11247, 0.14009, 0.12881, 0.11516, 0.11363, 0.11549, 0.11787, 0.09335, 0.01692], "alpha_by_type": {"special": {"share": 0.00121, "mean": 0.68444, "std": 0.10144}, "whitespace": {"share": 0.03697, "mean": 0.64974, "std": 0.17386}, "digit": {"share": 0.01334, "mean": 0.61385, "std": 0.19036}, "word_start": {"share": 0.58226, "mean": 0.48966, "std": 0.22504}, "word_cont": {"share": 0.24866, "mean": 0.30064, "std": 0.18777}, "punct": {"share": 0.11757, "mean": 0.67582, "std": 0.20136}}, "alpha_p05": 0.10971, "alpha_p50": 0.46259, "alpha_p95": 0.83278}
{"step": 8510, "total_steps": 72479, "tokens": 4461690880, "loss": 3.6280040740966797, "l_mix": 2.2291766703128815, "l_a": 2.3594126403331757, "l_c": 2.298145294189453, "z_loss": 15.599789261817932, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.24436089396476746, "tokens_per_second": 1455320.6433574283, "eta_seconds": 23045.216341209412, "data_wait_seconds": 2.962350845336914e-05, "mix": {"fineweb_hq": 0.50952, "dclm": 0.20764, "cosmopedia_v2": 0.20047, "finemath": 0.08237}}
{"step": 8520, "total_steps": 72479, "tokens": 4466933760, "loss": 3.4596453607082367, "l_mix": 2.135096311569214, "l_a": 2.235142409801483, "l_c": 2.174043506383896, "z_loss": 17.932499527931213, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.7540003061294556, "tokens_per_second": 1405328.0064479525, "eta_seconds": 23861.287925767898, "data_wait_seconds": 2.7179718017578125e-05, "mix": {"fineweb_hq": 0.50951, "dclm": 0.20763, "cosmopedia_v2": 0.20047, "finemath": 0.08239}}
{"step": 8530, "total_steps": 72479, "tokens": 4472176640, "loss": 4.821491360664368, "l_mix": 2.9818477630615234, "l_a": 3.090567797422409, "l_c": 3.036099284887314, "z_loss": 16.43494725227356, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2220502495765686, "tokens_per_second": 1453793.3659541986, "eta_seconds": 23062.213721132277, "data_wait_seconds": 1.564621925354004e-05, "mix": {"fineweb_hq": 0.50961, "dclm": 0.20762, "cosmopedia_v2": 0.20047, "finemath": 0.0823}}
{"step": 8540, "total_steps": 72479, "tokens": 4477419520, "loss": 4.9002074003219604, "l_mix": 3.031876027584076, "l_a": 3.137775957584381, "l_c": 3.084533601999283, "z_loss": 16.38425302505493, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18904270231723785, "tokens_per_second": 1305413.2138547578, "eta_seconds": 25679.57032778263, "data_wait_seconds": 1.6421079635620117e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20761, "cosmopedia_v2": 0.20047, "finemath": 0.08232}}
{"step": 8550, "total_steps": 72479, "tokens": 4482662400, "loss": 4.793632686138153, "l_mix": 2.963857054710388, "l_a": 3.073171555995941, "l_c": 3.020512878894806, "z_loss": 16.70217776298523, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21560920774936676, "tokens_per_second": 1403250.270741482, "eta_seconds": 23885.40964562893, "data_wait_seconds": 2.0831823348999023e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.2076, "cosmopedia_v2": 0.20047, "finemath": 0.08234}}
{"step": 8560, "total_steps": 72479, "tokens": 4487905280, "loss": 4.918609440326691, "l_mix": 3.042856514453888, "l_a": 3.151913970708847, "l_c": 3.095065653324127, "z_loss": 16.59074878692627, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2256896048784256, "tokens_per_second": 1481230.3064140703, "eta_seconds": 22624.411968135835, "data_wait_seconds": 1.5556812286376953e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20759, "cosmopedia_v2": 0.20047, "finemath": 0.08236}}
{"step": 8570, "total_steps": 72479, "tokens": 4493148160, "loss": 3.674654722213745, "l_mix": 2.2571007013320923, "l_a": 2.3912598192691803, "l_c": 2.3289676904678345, "z_loss": 14.856918811798096, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2564429044723511, "tokens_per_second": 1456176.0252205166, "eta_seconds": 23010.07653722763, "data_wait_seconds": 2.6941299438476562e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20758, "cosmopedia_v2": 0.20047, "finemath": 0.08238}}
{"step": 8580, "total_steps": 72479, "tokens": 4498391040, "loss": 5.107561886310577, "l_mix": 3.163950502872467, "l_a": 3.254780501127243, "l_c": 3.2183903753757477, "z_loss": 16.59991145133972, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20406492054462433, "tokens_per_second": 1442862.3721692574, "eta_seconds": 23218.762619495392, "data_wait_seconds": 1.811981201171875e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20769, "cosmopedia_v2": 0.20035, "finemath": 0.0824}}
{"step": 8590, "total_steps": 72479, "tokens": 4503633920, "loss": 3.6425611078739166, "l_mix": 2.2364994883537292, "l_a": 2.372520625591278, "l_c": 2.309217095375061, "z_loss": 15.402298927307129, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.25239431858062744, "tokens_per_second": 1465530.2177952142, "eta_seconds": 22856.052795958516, "data_wait_seconds": 2.5898218154907227e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20768, "cosmopedia_v2": 0.20047, "finemath": 0.08231}}
{"step": 8600, "total_steps": 72479, "tokens": 4508876800, "loss": 3.642210751771927, "l_mix": 2.2373447716236115, "l_a": 2.370425909757614, "l_c": 2.307311713695526, "z_loss": 15.446457624435425, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1709982007741928, "tokens_per_second": 1384733.4939195793, "eta_seconds": 24185.876415252686, "data_wait_seconds": 3.9190053939819336e-05, "mix": {"fineweb_hq": 0.50953, "dclm": 0.20767, "cosmopedia_v2": 0.20047, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.17470331, "gate_a2b/L1_absmax": 0.60379624, "gate_a2b/L1_rms": 0.21192472, "gate_a2b/L3_absmean": 0.28161335, "gate_a2b/L3_absmax": 0.46985775, "gate_a2b/L3_rms": 0.28793949, "gate_a2b/L5_absmean": 0.46572176, "gate_a2b/L5_absmax": 0.654796, "gate_a2b/L5_rms": 0.47421533, "gate_c2b/L1_absmean": 0.01489164, "gate_c2b/L1_absmax": 0.56778383, "gate_c2b/L1_rms": 0.06413892, "gate_c2b/L3_absmean": 0.22268696, "gate_c2b/L3_absmax": 0.46228901, "gate_c2b/L3_rms": 0.22886071, "gate_c2b/L5_absmean": 0.54240876, "gate_c2b/L5_absmax": 0.75963098, "gate_c2b/L5_rms": 0.55679196, "gate_b2a/L7_absmean": 0.07067755, "gate_b2a/L7_absmax": 0.25630054, "gate_b2a/L7_rms": 0.0756093, "gate_b2a/L11_absmean": 0.11809665, "gate_b2a/L11_absmax": 0.30429381, "gate_b2a/L11_rms": 0.12036745, "gate_b2a/L14_absmean": 0.28427339, "gate_b2a/L14_absmax": 0.45425695, "gate_b2a/L14_rms": 0.28537562, "gate_b2c/L3_absmean": 0.05350339, "gate_b2c/L3_absmax": 0.47207659, "gate_b2c/L3_rms": 0.07454809, "gate_b2c/L5_absmean": 0.09603922, "gate_b2c/L5_absmax": 0.5506646, "gate_b2c/L5_rms": 0.10093238, "gate_b2c/L6_absmean": 0.14676958, "gate_b2c/L6_absmax": 0.39748937, "gate_b2c/L6_rms": 0.14889297}, "streams": {"cos_a2b/round0": 0.350197, "cos_c2b/round0": 0.208781, "cos_b2a/round0": 0.174117, "cos_b2c/round0": 0.070127, "rms_a/round0": 1.148537, "rms_b/round0": 2.031684, "rms_c/round0": 1.342603, "cos_a2b/round1": 0.383558, "cos_c2b/round1": 0.27089, "cos_b2a/round1": 0.206638, "cos_b2c/round1": 0.154791, "rms_a/round1": 1.997884, "rms_b/round1": 3.720787, "rms_c/round1": 2.213992, "cos_a2b/round2": 0.470656, "cos_c2b/round2": 0.44501, "cos_b2a/round2": 0.040544, "cos_b2c/round2": 0.118509, "rms_a/round2": 3.947595, "rms_b/round2": 7.757322, "rms_c/round2": 3.497726}, "alpha_mean": 0.4329143037656138, "alpha_hist": [0.04042, 0.13501, 0.16931, 0.15279, 0.12337, 0.10399, 0.10822, 0.1075, 0.05479, 0.00461], "alpha_by_type": {"special": {"share": 0.00104, "mean": 0.6885, "std": 0.06275}, "whitespace": {"share": 0.03307, "mean": 0.62996, "std": 0.17127}, "digit": {"share": 0.00981, "mean": 0.35117, "std": 0.16849}, "word_start": {"share": 0.56734, "mean": 0.45188, "std": 0.21452}, "word_cont": {"share": 0.26978, "mean": 0.28473, "std": 0.16168}, "punct": {"share": 0.11897, "mean": 0.62823, "std": 0.18587}}, "alpha_p05": 0.12897, "alpha_p50": 0.41466, "alpha_p95": 0.80379}
{"step": 8610, "total_steps": 72479, "tokens": 4514119680, "loss": 3.6573446691036224, "l_mix": 2.247544080018997, "l_a": 2.3765924870967865, "l_c": 2.3175752758979797, "z_loss": 15.501934051513672, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2617957890033722, "tokens_per_second": 1349133.4198176584, "eta_seconds": 24820.19182100296, "data_wait_seconds": 2.1904706954956055e-05, "mix": {"fineweb_hq": 0.50952, "dclm": 0.20767, "cosmopedia_v2": 0.20046, "finemath": 0.08235}}
{"step": 8620, "total_steps": 72479, "tokens": 4519362560, "loss": 3.681936740875244, "l_mix": 2.2634487450122833, "l_a": 2.3918542563915253, "l_c": 2.331176519393921, "z_loss": 15.787032961845398, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.24991515278816223, "tokens_per_second": 1466324.1160502913, "eta_seconds": 22832.951477456092, "data_wait_seconds": 2.0265579223632812e-05, "mix": {"fineweb_hq": 0.50951, "dclm": 0.20766, "cosmopedia_v2": 0.20046, "finemath": 0.08237}}
{"step": 8630, "total_steps": 72479, "tokens": 4524605440, "loss": 3.6273381114006042, "l_mix": 2.2284922003746033, "l_a": 2.3612974882125854, "l_c": 2.2963684797286987, "z_loss": 15.461492538452148, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21092075109481812, "tokens_per_second": 1151029.5222351532, "eta_seconds": 29082.88959174156, "data_wait_seconds": 3.203749656677246e-05, "mix": {"fineweb_hq": 0.5095, "dclm": 0.20765, "cosmopedia_v2": 0.20046, "finemath": 0.08239}}
{"step": 8640, "total_steps": 72479, "tokens": 4529848320, "loss": 4.797342896461487, "l_mix": 2.9672877490520477, "l_a": 3.074363350868225, "l_c": 3.0205321311950684, "z_loss": 15.86385452747345, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18575136363506317, "tokens_per_second": 1358098.3807130083, "eta_seconds": 24644.76955964565, "data_wait_seconds": 1.767277717590332e-05, "mix": {"fineweb_hq": 0.50961, "dclm": 0.20764, "cosmopedia_v2": 0.20046, "finemath": 0.08229}}
{"step": 8650, "total_steps": 72479, "tokens": 4535091200, "loss": 4.791129112243652, "l_mix": 2.963612526655197, "l_a": 3.0700098872184753, "l_c": 3.016124039888382, "z_loss": 16.76376986503601, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.23623208701610565, "tokens_per_second": 1474326.0832558353, "eta_seconds": 22698.356308054925, "data_wait_seconds": 2.2172927856445312e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20763, "cosmopedia_v2": 0.20046, "finemath": 0.08231}}
{"step": 8660, "total_steps": 72479, "tokens": 4540334080, "loss": 4.853695094585419, "l_mix": 3.002902090549469, "l_a": 3.1107854545116425, "l_c": 3.0530490279197693, "z_loss": 16.425933241844177, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19196254014968872, "tokens_per_second": 1329374.0223186177, "eta_seconds": 25169.391992211345, "data_wait_seconds": 2.3692846298217773e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20762, "cosmopedia_v2": 0.20046, "finemath": 0.08233}}
{"step": 8670, "total_steps": 72479, "tokens": 4545576960, "loss": 4.791531085968018, "l_mix": 2.9621121883392334, "l_a": 3.072786957025528, "l_c": 3.020020067691803, "z_loss": 15.76727819442749, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.23495635390281677, "tokens_per_second": 1297404.3959372025, "eta_seconds": 25785.555449604988, "data_wait_seconds": 0.07629993557929993, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20761, "cosmopedia_v2": 0.20046, "finemath": 0.08235}}
{"step": 8680, "total_steps": 72479, "tokens": 4550819840, "loss": 3.6891239285469055, "l_mix": 2.266755163669586, "l_a": 2.3984241485595703, "l_c": 2.3376415073871613, "z_loss": 15.48931336402893, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2404373586177826, "tokens_per_second": 1223669.000285798, "eta_seconds": 27335.04738960266, "data_wait_seconds": 1.958012580871582e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.2076, "cosmopedia_v2": 0.20046, "finemath": 0.08237}}
{"step": 8690, "total_steps": 72479, "tokens": 4556062720, "loss": 3.6604843735694885, "l_mix": 2.2479588389396667, "l_a": 2.384528696537018, "l_c": 2.31876865029335, "z_loss": 15.362270712852478, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2411092221736908, "tokens_per_second": 1392696.6188179, "eta_seconds": 24013.70605781078, "data_wait_seconds": 3.1888484954833984e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20759, "cosmopedia_v2": 0.20046, "finemath": 0.08239}}
{"step": 8700, "total_steps": 72479, "tokens": 4561305600, "loss": 5.030366063117981, "l_mix": 3.114901155233383, "l_a": 3.2045595347881317, "l_c": 3.174654930830002, "z_loss": 17.005231857299805, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20164640247821808, "tokens_per_second": 1225761.8500636537, "eta_seconds": 27279.821402716636, "data_wait_seconds": 2.574920654296875e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.2077, "cosmopedia_v2": 0.20046, "finemath": 0.0823}, "gates": {"gate_a2b/L1_absmean": 0.17466228, "gate_a2b/L1_absmax": 0.60559726, "gate_a2b/L1_rms": 0.21260855, "gate_a2b/L3_absmean": 0.28126535, "gate_a2b/L3_absmax": 0.46785924, "gate_a2b/L3_rms": 0.28764194, "gate_a2b/L5_absmean": 0.46545491, "gate_a2b/L5_absmax": 0.64842379, "gate_a2b/L5_rms": 0.47390249, "gate_c2b/L1_absmean": 0.01628489, "gate_c2b/L1_absmax": 0.56535, "gate_c2b/L1_rms": 0.06410903, "gate_c2b/L3_absmean": 0.22128685, "gate_c2b/L3_absmax": 0.45256102, "gate_c2b/L3_rms": 0.22727953, "gate_c2b/L5_absmean": 0.54345703, "gate_c2b/L5_absmax": 0.76041329, "gate_c2b/L5_rms": 0.55792719, "gate_b2a/L7_absmean": 0.07111566, "gate_b2a/L7_absmax": 0.25391567, "gate_b2a/L7_rms": 0.07595823, "gate_b2a/L11_absmean": 0.11845929, "gate_b2a/L11_absmax": 0.30387586, "gate_b2a/L11_rms": 0.12064753, "gate_b2a/L14_absmean": 0.28648001, "gate_b2a/L14_absmax": 0.45333701, "gate_b2a/L14_rms": 0.28754804, "gate_b2c/L3_absmean": 0.05413563, "gate_b2c/L3_absmax": 0.4722155, "gate_b2c/L3_rms": 0.07530826, "gate_b2c/L5_absmean": 0.09555541, "gate_b2c/L5_absmax": 0.54245263, "gate_b2c/L5_rms": 0.10043358, "gate_b2c/L6_absmean": 0.14621603, "gate_b2c/L6_absmax": 0.39416575, "gate_b2c/L6_rms": 0.14834164}, "streams": {"cos_a2b/round0": 0.376154, "cos_c2b/round0": 0.253616, "cos_b2a/round0": 0.190788, "cos_b2c/round0": 0.122624, "rms_a/round0": 1.083855, "rms_b/round0": 1.997671, "rms_c/round0": 1.293509, "cos_a2b/round1": 0.400629, "cos_c2b/round1": 0.279753, "cos_b2a/round1": 0.220195, "cos_b2c/round1": 0.177415, "rms_a/round1": 1.949472, "rms_b/round1": 3.568381, "rms_c/round1": 2.141871, "cos_a2b/round2": 0.517296, "cos_c2b/round2": 0.456772, "cos_b2a/round2": 0.092499, "cos_b2c/round2": 0.15724, "rms_a/round2": 3.912049, "rms_b/round2": 7.252748, "rms_c/round2": 3.323843}, "alpha_mean": 0.4730443530555988, "alpha_hist": [0.0076, 0.04918, 0.11494, 0.16171, 0.20026, 0.22878, 0.15285, 0.07471, 0.00983, 0.00015], "alpha_by_type": {"special": {"share": 0.00047, "mean": 0.57615, "std": 0.07755}, "whitespace": {"share": 0.04874, "mean": 0.53686, "std": 0.11549}, "digit": {"share": 0.02335, "mean": 0.5479, "std": 0.10415}, "word_start": {"share": 0.60925, "mean": 0.48809, "std": 0.15456}, "word_cont": {"share": 0.21832, "mean": 0.35449, "std": 0.14196}, "punct": {"share": 0.09987, "mean": 0.59132, "std": 0.15599}}, "alpha_p05": 0.19683, "alpha_p50": 0.48947, "alpha_p95": 0.72056}
{"step": 8710, "total_steps": 72479, "tokens": 4566548480, "loss": 5.120025098323822, "l_mix": 3.17168265581131, "l_a": 3.267180174589157, "l_c": 3.221925526857376, "z_loss": 16.1051527261734, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18188822269439697, "tokens_per_second": 1323692.0466043022, "eta_seconds": 25257.628130173685, "data_wait_seconds": 2.276897430419922e-05, "mix": {"fineweb_hq": 0.50953, "dclm": 0.20769, "cosmopedia_v2": 0.20046, "finemath": 0.08232}}
{"step": 8720, "total_steps": 72479, "tokens": 4571791360, "loss": 3.6442523896694183, "l_mix": 2.2385967075824738, "l_a": 2.3727283775806427, "l_c": 2.3076174557209015, "z_loss": 15.51889717578888, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2021031528711319, "tokens_per_second": 1395523.6058679402, "eta_seconds": 23953.789424586295, "data_wait_seconds": 2.574920654296875e-05, "mix": {"fineweb_hq": 0.50952, "dclm": 0.20768, "cosmopedia_v2": 0.20046, "finemath": 0.08234}}
{"step": 8730, "total_steps": 72479, "tokens": 4577034240, "loss": 3.6444527208805084, "l_mix": 2.2379888892173767, "l_a": 2.3768808841705322, "l_c": 2.3060234785079956, "z_loss": 15.924436569213867, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20655031502246857, "tokens_per_second": 1406727.9237542336, "eta_seconds": 23759.275086259844, "data_wait_seconds": 2.917647361755371e-05, "mix": {"fineweb_hq": 0.50951, "dclm": 0.20767, "cosmopedia_v2": 0.20046, "finemath": 0.08236}}
{"step": 8740, "total_steps": 72479, "tokens": 4582277120, "loss": 3.6615858674049377, "l_mix": 2.249922603368759, "l_a": 2.382882058620453, "l_c": 2.3175046741962433, "z_loss": 15.47242558002472, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21680141985416412, "tokens_per_second": 1237073.4535687757, "eta_seconds": 27013.426515293122, "data_wait_seconds": 2.625584602355957e-05, "mix": {"fineweb_hq": 0.5095, "dclm": 0.20767, "cosmopedia_v2": 0.20046, "finemath": 0.08238}}
{"step": 8750, "total_steps": 72479, "tokens": 4587520000, "loss": 4.798491358757019, "l_mix": 2.9676140546798706, "l_a": 3.076882779598236, "l_c": 3.0206832885742188, "z_loss": 16.075490951538086, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2025180459022522, "tokens_per_second": 1386892.8067770978, "eta_seconds": 24091.515788912773, "data_wait_seconds": 2.41696834564209e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20766, "cosmopedia_v2": 0.20046, "finemath": 0.08229}}
{"step": 8760, "total_steps": 72479, "tokens": 4592762880, "loss": 4.589041292667389, "l_mix": 2.8372881412506104, "l_a": 2.9459137320518494, "l_c": 2.887798398733139, "z_loss": 16.393723487854004, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19436948001384735, "tokens_per_second": 1400611.6102098986, "eta_seconds": 23851.79933428764, "data_wait_seconds": 2.1904706954956055e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20765, "cosmopedia_v2": 0.20046, "finemath": 0.08231}}
{"step": 8770, "total_steps": 72479, "tokens": 4598005760, "loss": 4.625963866710663, "l_mix": 2.860061913728714, "l_a": 2.9689902663230896, "l_c": 2.911758065223694, "z_loss": 16.774308443069458, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.204721599817276, "tokens_per_second": 1457539.522699629, "eta_seconds": 22916.60958197117, "data_wait_seconds": 1.901388168334961e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20046, "finemath": 0.08233}}
{"step": 8780, "total_steps": 72479, "tokens": 4603248640, "loss": 4.592108964920044, "l_mix": 2.839749187231064, "l_a": 2.944471597671509, "l_c": 2.8913321793079376, "z_loss": 16.18632221221924, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2573185861110687, "tokens_per_second": 1455108.688026607, "eta_seconds": 22951.289884257316, "data_wait_seconds": 1.5944242477416992e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20763, "cosmopedia_v2": 0.20046, "finemath": 0.08235}}
{"step": 8790, "total_steps": 72479, "tokens": 4608491520, "loss": 3.683850944042206, "l_mix": 2.264336109161377, "l_a": 2.3982721269130707, "l_c": 2.32832670211792, "z_loss": 15.3511461019516, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22678859531879425, "tokens_per_second": 1357986.7522364242, "eta_seconds": 24588.883784770966, "data_wait_seconds": 2.2113323211669922e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20762, "cosmopedia_v2": 0.20046, "finemath": 0.08237}}
{"step": 8800, "total_steps": 72479, "tokens": 4613734400, "loss": 3.6791865527629852, "l_mix": 2.2608862817287445, "l_a": 2.391902804374695, "l_c": 2.3305631577968597, "z_loss": 15.603677153587341, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.23641511797904968, "tokens_per_second": 1403046.6753740988, "eta_seconds": 23795.456087088583, "data_wait_seconds": 2.3633241653442383e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20761, "cosmopedia_v2": 0.20045, "finemath": 0.08239}, "gates": {"gate_a2b/L1_absmean": 0.17525445, "gate_a2b/L1_absmax": 0.59610021, "gate_a2b/L1_rms": 0.21215113, "gate_a2b/L3_absmean": 0.28095245, "gate_a2b/L3_absmax": 0.46315676, "gate_a2b/L3_rms": 0.28730181, "gate_a2b/L5_absmean": 0.46550104, "gate_a2b/L5_absmax": 0.65225077, "gate_a2b/L5_rms": 0.4739081, "gate_c2b/L1_absmean": 0.01560013, "gate_c2b/L1_absmax": 0.56335288, "gate_c2b/L1_rms": 0.06363581, "gate_c2b/L3_absmean": 0.22130214, "gate_c2b/L3_absmax": 0.44978908, "gate_c2b/L3_rms": 0.22737534, "gate_c2b/L5_absmean": 0.54353637, "gate_c2b/L5_absmax": 0.76765746, "gate_c2b/L5_rms": 0.55794805, "gate_b2a/L7_absmean": 0.07173392, "gate_b2a/L7_absmax": 0.25203148, "gate_b2a/L7_rms": 0.07645081, "gate_b2a/L11_absmean": 0.1189125, "gate_b2a/L11_absmax": 0.3028065, "gate_b2a/L11_rms": 0.12107742, "gate_b2a/L14_absmean": 0.28862852, "gate_b2a/L14_absmax": 0.4584485, "gate_b2a/L14_rms": 0.28970832, "gate_b2c/L3_absmean": 0.05386885, "gate_b2c/L3_absmax": 0.47047907, "gate_b2c/L3_rms": 0.07507902, "gate_b2c/L5_absmean": 0.09537011, "gate_b2c/L5_absmax": 0.54456347, "gate_b2c/L5_rms": 0.10021877, "gate_b2c/L6_absmean": 0.14554681, "gate_b2c/L6_absmax": 0.39567995, "gate_b2c/L6_rms": 0.1476538}, "streams": {"cos_a2b/round0": 0.349543, "cos_c2b/round0": 0.206818, "cos_b2a/round0": 0.17939, "cos_b2c/round0": 0.072981, "rms_a/round0": 1.125687, "rms_b/round0": 2.031324, "rms_c/round0": 1.34599, "cos_a2b/round1": 0.38663, "cos_c2b/round1": 0.270323, "cos_b2a/round1": 0.212346, "cos_b2c/round1": 0.161112, "rms_a/round1": 1.998637, "rms_b/round1": 3.72527, "rms_c/round1": 2.218505, "cos_a2b/round2": 0.479042, "cos_c2b/round2": 0.440136, "cos_b2a/round2": 0.036875, "cos_b2c/round2": 0.120677, "rms_a/round2": 3.992045, "rms_b/round2": 7.678674, "rms_c/round2": 3.52767}, "alpha_mean": 0.45382130332617265, "alpha_hist": [0.04929, 0.12044, 0.14546, 0.14175, 0.12038, 0.10793, 0.11377, 0.12061, 0.07275, 0.00763], "alpha_by_type": {"special": {"share": 0.00111, "mean": 0.73495, "std": 0.06867}, "whitespace": {"share": 0.03033, "mean": 0.7053, "std": 0.16706}, "digit": {"share": 0.01012, "mean": 0.41608, "std": 0.17244}, "word_start": {"share": 0.57596, "mean": 0.47694, "std": 0.21833}, "word_cont": {"share": 0.26523, "mean": 0.28777, "std": 0.17504}, "punct": {"share": 0.11725, "mean": 0.65139, "std": 0.18767}}, "alpha_p05": 0.09984, "alpha_p50": 0.44288, "alpha_p95": 0.83735}
{"step": 8810, "total_steps": 72479, "tokens": 4618977280, "loss": 5.108502447605133, "l_mix": 3.164736270904541, "l_a": 3.256607323884964, "l_c": 3.217204660177231, "z_loss": 16.225393891334534, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16907724738121033, "tokens_per_second": 1416346.9542853252, "eta_seconds": 23568.301941132544, "data_wait_seconds": 2.613663673400879e-05, "mix": {"fineweb_hq": 0.50953, "dclm": 0.20772, "cosmopedia_v2": 0.20045, "finemath": 0.08229}}
{"step": 8820, "total_steps": 72479, "tokens": 4624220160, "loss": 5.118715286254883, "l_mix": 3.1701387465000153, "l_a": 3.262820065021515, "l_c": 3.227040559053421, "z_loss": 16.18190634250641, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17743085324764252, "tokens_per_second": 1433043.9333466317, "eta_seconds": 23290.03948543072, "data_wait_seconds": 1.913309097290039e-05, "mix": {"fineweb_hq": 0.50952, "dclm": 0.20771, "cosmopedia_v2": 0.20045, "finemath": 0.08231}}
{"step": 8830, "total_steps": 72479, "tokens": 4629463040, "loss": 5.174717724323273, "l_mix": 3.2047520875930786, "l_a": 3.298317402601242, "l_c": 3.2628808319568634, "z_loss": 16.060729265213013, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.23322409391403198, "tokens_per_second": 1386049.5902700473, "eta_seconds": 24075.91124174595, "data_wait_seconds": 2.434849739074707e-05, "mix": {"fineweb_hq": 0.50951, "dclm": 0.2077, "cosmopedia_v2": 0.20045, "finemath": 0.08233}}
{"step": 8840, "total_steps": 72479, "tokens": 4634705920, "loss": 5.112471759319305, "l_mix": 3.167810618877411, "l_a": 3.256528377532959, "l_c": 3.2201659083366394, "z_loss": 16.52887785434723, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.24067871272563934, "tokens_per_second": 1394601.5826195034, "eta_seconds": 23924.513242936133, "data_wait_seconds": 2.6017427444458008e-05, "mix": {"fineweb_hq": 0.5095, "dclm": 0.20769, "cosmopedia_v2": 0.20045, "finemath": 0.08235}}
{"step": 8850, "total_steps": 72479, "tokens": 4639948800, "loss": 5.15115886926651, "l_mix": 3.1898736655712128, "l_a": 3.28079891204834, "l_c": 3.2514639496803284, "z_loss": 16.062586307525635, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.27832621335983276, "tokens_per_second": 1430432.0753938362, "eta_seconds": 23321.569563388824, "data_wait_seconds": 1.6808509826660156e-05, "mix": {"fineweb_hq": 0.50949, "dclm": 0.20768, "cosmopedia_v2": 0.20045, "finemath": 0.08237}}
{"step": 8860, "total_steps": 72479, "tokens": 4645191680, "loss": 3.652991443872452, "l_mix": 2.2450763285160065, "l_a": 2.375537097454071, "l_c": 2.3123658895492554, "z_loss": 15.441517353057861, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20203880965709686, "tokens_per_second": 1377100.4430108445, "eta_seconds": 24220.948037075996, "data_wait_seconds": 1.7434358596801758e-05, "mix": {"fineweb_hq": 0.50948, "dclm": 0.20767, "cosmopedia_v2": 0.20045, "finemath": 0.08239}}
{"step": 8870, "total_steps": 72479, "tokens": 4650434560, "loss": 4.654764235019684, "l_mix": 2.87774458527565, "l_a": 2.9843416810035706, "l_c": 2.933387339115143, "z_loss": 17.00796151161194, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22390887141227722, "tokens_per_second": 1344697.6853478604, "eta_seconds": 24800.693684077265, "data_wait_seconds": 1.710653305053711e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20045, "finemath": 0.0823}}
{"step": 8880, "total_steps": 72479, "tokens": 4655677440, "loss": 4.64888471364975, "l_mix": 2.8762694597244263, "l_a": 2.9795825481414795, "l_c": 2.9236345887184143, "z_loss": 16.50095558166504, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2180112600326538, "tokens_per_second": 1419608.4101335083, "eta_seconds": 23488.303023552893, "data_wait_seconds": 2.8818845748901367e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20045, "finemath": 0.08232}}
{"step": 8890, "total_steps": 72479, "tokens": 4660920320, "loss": 4.719073414802551, "l_mix": 2.917311191558838, "l_a": 3.03087654709816, "l_c": 2.9695262014865875, "z_loss": 16.412240028381348, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.23838089406490326, "tokens_per_second": 1381270.5064064772, "eta_seconds": 24136.43777766228, "data_wait_seconds": 3.141164779663086e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20045, "finemath": 0.08234}}
{"step": 8900, "total_steps": 72479, "tokens": 4666163200, "loss": 3.6377730071544647, "l_mix": 2.23446524143219, "l_a": 2.3660394847393036, "l_c": 2.3067195117473602, "z_loss": 14.799628615379333, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2193579077720642, "tokens_per_second": 1315288.6819092599, "eta_seconds": 25343.262821674347, "data_wait_seconds": 2.3603439331054688e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20764, "cosmopedia_v2": 0.20045, "finemath": 0.08236}, "gates": {"gate_a2b/L1_absmean": 0.17678145, "gate_a2b/L1_absmax": 0.59797156, "gate_a2b/L1_rms": 0.21417433, "gate_a2b/L3_absmean": 0.28082234, "gate_a2b/L3_absmax": 0.46685869, "gate_a2b/L3_rms": 0.28713778, "gate_a2b/L5_absmean": 0.46660709, "gate_a2b/L5_absmax": 0.65962237, "gate_a2b/L5_rms": 0.47499061, "gate_c2b/L1_absmean": 0.01575571, "gate_c2b/L1_absmax": 0.56007445, "gate_c2b/L1_rms": 0.06350054, "gate_c2b/L3_absmean": 0.22070017, "gate_c2b/L3_absmax": 0.45123255, "gate_c2b/L3_rms": 0.22682782, "gate_c2b/L5_absmean": 0.54401392, "gate_c2b/L5_absmax": 0.76620352, "gate_c2b/L5_rms": 0.55849916, "gate_b2a/L7_absmean": 0.07175851, "gate_b2a/L7_absmax": 0.25100863, "gate_b2a/L7_rms": 0.07643526, "gate_b2a/L11_absmean": 0.11825328, "gate_b2a/L11_absmax": 0.30438828, "gate_b2a/L11_rms": 0.12034689, "gate_b2a/L14_absmean": 0.28871191, "gate_b2a/L14_absmax": 0.45569992, "gate_b2a/L14_rms": 0.28976363, "gate_b2c/L3_absmean": 0.05427948, "gate_b2c/L3_absmax": 0.47223279, "gate_b2c/L3_rms": 0.0757225, "gate_b2c/L5_absmean": 0.09604221, "gate_b2c/L5_absmax": 0.54410684, "gate_b2c/L5_rms": 0.10087242, "gate_b2c/L6_absmean": 0.14688633, "gate_b2c/L6_absmax": 0.3958182, "gate_b2c/L6_rms": 0.14903203}, "streams": {"cos_a2b/round0": 0.349854, "cos_c2b/round0": 0.20783, "cos_b2a/round0": 0.176897, "cos_b2c/round0": 0.070268, "rms_a/round0": 1.120154, "rms_b/round0": 2.006467, "rms_c/round0": 1.337447, "cos_a2b/round1": 0.381723, "cos_c2b/round1": 0.271897, "cos_b2a/round1": 0.203929, "cos_b2c/round1": 0.156245, "rms_a/round1": 1.983309, "rms_b/round1": 3.680148, "rms_c/round1": 2.20326, "cos_a2b/round2": 0.475607, "cos_c2b/round2": 0.443111, "cos_b2a/round2": 0.034896, "cos_b2c/round2": 0.118509, "rms_a/round2": 3.962064, "rms_b/round2": 7.632284, "rms_c/round2": 3.477422}, "alpha_mean": 0.44178357997676443, "alpha_hist": [0.0486, 0.13031, 0.15652, 0.13905, 0.11937, 0.11127, 0.11398, 0.11476, 0.05911, 0.00702], "alpha_by_type": {"special": {"share": 0.00116, "mean": 0.66714, "std": 0.08869}, "whitespace": {"share": 0.03307, "mean": 0.67276, "std": 0.17385}, "digit": {"share": 0.00946, "mean": 0.50568, "std": 0.17595}, "word_start": {"share": 0.58322, "mean": 0.45463, "std": 0.21745}, "word_cont": {"share": 0.25638, "mean": 0.28715, "std": 0.1733}, "punct": {"share": 0.11671, "mean": 0.64442, "std": 0.18296}}, "alpha_p05": 0.10053, "alpha_p50": 0.43842, "alpha_p95": 0.83413}
{"step": 8910, "total_steps": 72479, "tokens": 4671406080, "loss": 3.6303821206092834, "l_mix": 2.2300987541675568, "l_a": 2.3640779852867126, "l_c": 2.2984015941619873, "z_loss": 15.394788384437561, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20485953986644745, "tokens_per_second": 1466035.6372574184, "eta_seconds": 22733.733768129347, "data_wait_seconds": 2.0235776901245117e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20763, "cosmopedia_v2": 0.20045, "finemath": 0.08238}}
{"step": 8920, "total_steps": 72479, "tokens": 4676648960, "loss": 3.301336795091629, "l_mix": 2.039237394928932, "l_a": 2.1230035424232483, "l_c": 2.078131780028343, "z_loss": 17.587960720062256, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.24635197222232819, "tokens_per_second": 1448715.4269775103, "eta_seconds": 23001.90939605236, "data_wait_seconds": 1.659989356994629e-05, "mix": {"fineweb_hq": 0.50953, "dclm": 0.20762, "cosmopedia_v2": 0.20045, "finemath": 0.0824}}
{"step": 8930, "total_steps": 72479, "tokens": 4681891840, "loss": 4.548218846321106, "l_mix": 2.81223401427269, "l_a": 2.91701340675354, "l_c": 2.864240437746048, "z_loss": 16.087010383605957, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21874675154685974, "tokens_per_second": 1373553.55568717, "eta_seconds": 24256.773952531814, "data_wait_seconds": 2.321600914001465e-05, "mix": {"fineweb_hq": 0.50963, "dclm": 0.20761, "cosmopedia_v2": 0.20045, "finemath": 0.08231}}
{"step": 8940, "total_steps": 72479, "tokens": 4687134720, "loss": 4.58024126291275, "l_mix": 2.8324742019176483, "l_a": 2.9423843026161194, "l_c": 2.877924680709839, "z_loss": 16.74225425720215, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.24222490191459656, "tokens_per_second": 1353889.6984687417, "eta_seconds": 24605.20622150898, "data_wait_seconds": 2.1576881408691406e-05, "mix": {"fineweb_hq": 0.50962, "dclm": 0.20761, "cosmopedia_v2": 0.20045, "finemath": 0.08233}}
{"step": 8950, "total_steps": 72479, "tokens": 4692377600, "loss": 4.624687135219574, "l_mix": 2.8600453436374664, "l_a": 2.9623326659202576, "l_c": 2.9143477082252502, "z_loss": 16.376068830490112, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21447525918483734, "tokens_per_second": 1435592.0540669977, "eta_seconds": 23201.223674678804, "data_wait_seconds": 3.081560134887695e-05, "mix": {"fineweb_hq": 0.50961, "dclm": 0.2076, "cosmopedia_v2": 0.20045, "finemath": 0.08235}}
{"step": 8960, "total_steps": 72479, "tokens": 4697620480, "loss": 4.656603157520294, "l_mix": 2.8798325955867767, "l_a": 2.983478218317032, "l_c": 2.9336454272270203, "z_loss": 16.334213972091675, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17296072840690613, "tokens_per_second": 1419575.6020928416, "eta_seconds": 23459.299682879446, "data_wait_seconds": 2.321600914001465e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20759, "cosmopedia_v2": 0.20045, "finemath": 0.08237}}
{"step": 8970, "total_steps": 72479, "tokens": 4702863360, "loss": 4.616028428077698, "l_mix": 2.8542718291282654, "l_a": 2.960952401161194, "l_c": 2.9062183499336243, "z_loss": 16.053930640220642, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1829633265733719, "tokens_per_second": 1440043.2673430718, "eta_seconds": 23122.226496315, "data_wait_seconds": 1.6570091247558594e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20758, "cosmopedia_v2": 0.20045, "finemath": 0.08239}}
{"step": 8980, "total_steps": 72479, "tokens": 4708106240, "loss": 4.517350912094116, "l_mix": 2.7926991879940033, "l_a": 2.899889290332794, "l_c": 2.8435046672821045, "z_loss": 16.33523917198181, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16771571338176727, "tokens_per_second": 1467364.990370811, "eta_seconds": 22688.127310156822, "data_wait_seconds": 2.1517276763916016e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20768, "cosmopedia_v2": 0.20045, "finemath": 0.08229}}
{"step": 8990, "total_steps": 72479, "tokens": 4713349120, "loss": 4.477406084537506, "l_mix": 2.7692030668258667, "l_a": 2.8714486062526703, "l_c": 2.817242443561554, "z_loss": 15.956355333328247, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20202378928661346, "tokens_per_second": 1401178.8482673801, "eta_seconds": 23756.082867765424, "data_wait_seconds": 2.574920654296875e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20768, "cosmopedia_v2": 0.20044, "finemath": 0.08231}}
{"step": 9000, "total_steps": 72479, "tokens": 4718592000, "loss": 4.516468644142151, "l_mix": 2.7918875217437744, "l_a": 2.898188978433609, "l_c": 2.844988316297531, "z_loss": 16.27792716026306, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2181016355752945, "tokens_per_second": 1423263.053674054, "eta_seconds": 23383.785496354103, "data_wait_seconds": 1.6897916793823242e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.17808136, "gate_a2b/L1_absmax": 0.59869486, "gate_a2b/L1_rms": 0.21541882, "gate_a2b/L3_absmean": 0.28277385, "gate_a2b/L3_absmax": 0.46703395, "gate_a2b/L3_rms": 0.2890895, "gate_a2b/L5_absmean": 0.46805474, "gate_a2b/L5_absmax": 0.65465051, "gate_a2b/L5_rms": 0.4763712, "gate_c2b/L1_absmean": 0.01575576, "gate_c2b/L1_absmax": 0.55412281, "gate_c2b/L1_rms": 0.0634044, "gate_c2b/L3_absmean": 0.21933983, "gate_c2b/L3_absmax": 0.44853741, "gate_c2b/L3_rms": 0.22544396, "gate_c2b/L5_absmean": 0.54423058, "gate_c2b/L5_absmax": 0.77129787, "gate_c2b/L5_rms": 0.55857778, "gate_b2a/L7_absmean": 0.07208671, "gate_b2a/L7_absmax": 0.25109324, "gate_b2a/L7_rms": 0.07672287, "gate_b2a/L11_absmean": 0.11870767, "gate_b2a/L11_absmax": 0.30854568, "gate_b2a/L11_rms": 0.12082442, "gate_b2a/L14_absmean": 0.28866312, "gate_b2a/L14_absmax": 0.45561588, "gate_b2a/L14_rms": 0.28971907, "gate_b2c/L3_absmean": 0.05530528, "gate_b2c/L3_absmax": 0.47543484, "gate_b2c/L3_rms": 0.07671614, "gate_b2c/L5_absmean": 0.09697163, "gate_b2c/L5_absmax": 0.54803008, "gate_b2c/L5_rms": 0.10179214, "gate_b2c/L6_absmean": 0.14736414, "gate_b2c/L6_absmax": 0.40031245, "gate_b2c/L6_rms": 0.14953783}, "streams": {"cos_a2b/round0": 0.371707, "cos_c2b/round0": 0.255587, "cos_b2a/round0": 0.188814, "cos_b2c/round0": 0.116092, "rms_a/round0": 1.08425, "rms_b/round0": 2.021163, "rms_c/round0": 1.322356, "cos_a2b/round1": 0.392128, "cos_c2b/round1": 0.266139, "cos_b2a/round1": 0.210003, "cos_b2c/round1": 0.161097, "rms_a/round1": 1.966619, "rms_b/round1": 3.65967, "rms_c/round1": 2.164479, "cos_a2b/round2": 0.501769, "cos_c2b/round2": 0.448481, "cos_b2a/round2": 0.067207, "cos_b2c/round2": 0.140084, "rms_a/round2": 3.983648, "rms_b/round2": 7.605314, "rms_c/round2": 3.414803}, "alpha_mean": 0.44151334621792415, "alpha_hist": [0.01814, 0.07825, 0.15671, 0.18646, 0.17885, 0.1619, 0.11937, 0.08037, 0.01926, 0.0007], "alpha_by_type": {"special": {"share": 0.00104, "mean": 0.58937, "std": 0.11933}, "whitespace": {"share": 0.02795, "mean": 0.55321, "std": 0.12975}, "digit": {"share": 0.04764, "mean": 0.55268, "std": 0.15243}, "word_start": {"share": 0.56824, "mean": 0.44989, "std": 0.16881}, "word_cont": {"share": 0.24223, "mean": 0.31371, "std": 0.14435}, "punct": {"share": 0.1129, "mean": 0.59764, "std": 0.17406}}, "alpha_p05": 0.16668, "alpha_p50": 0.43614, "alpha_p95": 0.74909}
{"step": 9010, "total_steps": 72479, "tokens": 4723834880, "loss": 3.6379315555095673, "l_mix": 2.2355911433696747, "l_a": 2.3664983212947845, "l_c": 2.3027603030204773, "z_loss": 15.627637386322021, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2161543220281601, "tokens_per_second": 1398876.1625065967, "eta_seconds": 23787.69183712006, "data_wait_seconds": 1.710653305053711e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 9020, "total_steps": 72479, "tokens": 4729077760, "loss": 3.6057222485542297, "l_mix": 2.2159086167812347, "l_a": 2.343766689300537, "l_c": 2.283769875764847, "z_loss": 15.526200652122498, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.23063291609287262, "tokens_per_second": 1419812.623874436, "eta_seconds": 23433.227478432655, "data_wait_seconds": 2.2292137145996094e-05, "mix": {"fineweb_hq": 0.50953, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08237}}
{"step": 9030, "total_steps": 72479, "tokens": 4734320640, "loss": 3.307820677757263, "l_mix": 2.042155236005783, "l_a": 2.1284637600183487, "l_c": 2.084300622344017, "z_loss": 18.36020302772522, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2472209930419922, "tokens_per_second": 1392547.74811579, "eta_seconds": 23888.2647701025, "data_wait_seconds": 1.6748905181884766e-05, "mix": {"fineweb_hq": 0.50952, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08239}}
{"step": 9040, "total_steps": 72479, "tokens": 4739563520, "loss": 4.634757459163666, "l_mix": 2.865448921918869, "l_a": 2.9724992215633392, "l_c": 2.9197826385498047, "z_loss": 16.23845624923706, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20850378274917603, "tokens_per_second": 1439258.5361082521, "eta_seconds": 23109.334145021436, "data_wait_seconds": 1.9848346710205078e-05, "mix": {"fineweb_hq": 0.50962, "dclm": 0.20763, "cosmopedia_v2": 0.20044, "finemath": 0.0823}}
{"step": 9050, "total_steps": 72479, "tokens": 4744806400, "loss": 4.615686535835266, "l_mix": 2.8554845452308655, "l_a": 2.956193596124649, "l_c": 2.905723452568054, "z_loss": 16.268567204475403, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2201513946056366, "tokens_per_second": 1352119.035606839, "eta_seconds": 24594.77507250309, "data_wait_seconds": 1.761317253112793e-05, "mix": {"fineweb_hq": 0.50961, "dclm": 0.20762, "cosmopedia_v2": 0.20044, "finemath": 0.08232}}
{"step": 9060, "total_steps": 72479, "tokens": 4750049280, "loss": 4.55970823764801, "l_mix": 2.8180760741233826, "l_a": 2.9311647713184357, "l_c": 2.868817090988159, "z_loss": 16.37728762626648, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18816140294075012, "tokens_per_second": 1392585.756507404, "eta_seconds": 23876.318220710757, "data_wait_seconds": 2.238154411315918e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20762, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 9070, "total_steps": 72479, "tokens": 4755292160, "loss": 4.692393004894257, "l_mix": 2.9023377299308777, "l_a": 3.0053076446056366, "l_c": 2.9559970796108246, "z_loss": 16.63767099380493, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20992402732372284, "tokens_per_second": 1359868.2664287076, "eta_seconds": 24446.91049325466, "data_wait_seconds": 1.9758939743041992e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20761, "cosmopedia_v2": 0.20044, "finemath": 0.08236}}
{"step": 9080, "total_steps": 72479, "tokens": 4760535040, "loss": 4.5664674043655396, "l_mix": 2.823045700788498, "l_a": 2.92867311835289, "l_c": 2.877193659543991, "z_loss": 16.61633801460266, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20619510114192963, "tokens_per_second": 1462297.757881695, "eta_seconds": 22730.89371356964, "data_wait_seconds": 2.1249055862426758e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.2076, "cosmopedia_v2": 0.20044, "finemath": 0.08238}}
{"step": 9090, "total_steps": 72479, "tokens": 4765777920, "loss": 5.080289244651794, "l_mix": 3.1464591920375824, "l_a": 3.2390957176685333, "l_c": 3.2016299664974213, "z_loss": 16.122114777565002, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18195699155330658, "tokens_per_second": 1398683.1753753969, "eta_seconds": 23760.98648865223, "data_wait_seconds": 4.7653913497924805e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.2077, "cosmopedia_v2": 0.20044, "finemath": 0.08229}}
{"step": 9100, "total_steps": 72479, "tokens": 4771020800, "loss": 5.107330024242401, "l_mix": 3.1632341742515564, "l_a": 3.2538409531116486, "l_c": 3.2211343348026276, "z_loss": 16.032620787620544, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1851615160703659, "tokens_per_second": 1346942.05463394, "eta_seconds": 24669.843099546433, "data_wait_seconds": 1.7464160919189453e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20769, "cosmopedia_v2": 0.20044, "finemath": 0.08231}, "gates": {"gate_a2b/L1_absmean": 0.17992239, "gate_a2b/L1_absmax": 0.59822249, "gate_a2b/L1_rms": 0.21711993, "gate_a2b/L3_absmean": 0.28397313, "gate_a2b/L3_absmax": 0.46647489, "gate_a2b/L3_rms": 0.29014122, "gate_a2b/L5_absmean": 0.46879464, "gate_a2b/L5_absmax": 0.65398282, "gate_a2b/L5_rms": 0.47703698, "gate_c2b/L1_absmean": 0.0162695, "gate_c2b/L1_absmax": 0.55832636, "gate_c2b/L1_rms": 0.06343196, "gate_c2b/L3_absmean": 0.21920581, "gate_c2b/L3_absmax": 0.45267582, "gate_c2b/L3_rms": 0.22527853, "gate_c2b/L5_absmean": 0.54330963, "gate_c2b/L5_absmax": 0.75977135, "gate_c2b/L5_rms": 0.5575828, "gate_b2a/L7_absmean": 0.07164607, "gate_b2a/L7_absmax": 0.24983126, "gate_b2a/L7_rms": 0.07627982, "gate_b2a/L11_absmean": 0.11845146, "gate_b2a/L11_absmax": 0.30935007, "gate_b2a/L11_rms": 0.12066863, "gate_b2a/L14_absmean": 0.28869358, "gate_b2a/L14_absmax": 0.46298823, "gate_b2a/L14_rms": 0.28977099, "gate_b2c/L3_absmean": 0.05573987, "gate_b2c/L3_absmax": 0.47703952, "gate_b2c/L3_rms": 0.07695351, "gate_b2c/L5_absmean": 0.09689802, "gate_b2c/L5_absmax": 0.54747671, "gate_b2c/L5_rms": 0.1018193, "gate_b2c/L6_absmean": 0.14788607, "gate_b2c/L6_absmax": 0.40682611, "gate_b2c/L6_rms": 0.15012442}, "streams": {"cos_a2b/round0": 0.371866, "cos_c2b/round0": 0.250235, "cos_b2a/round0": 0.190058, "cos_b2c/round0": 0.120268, "rms_a/round0": 1.065821, "rms_b/round0": 1.984959, "rms_c/round0": 1.294773, "cos_a2b/round1": 0.395797, "cos_c2b/round1": 0.279454, "cos_b2a/round1": 0.220691, "cos_b2c/round1": 0.170869, "rms_a/round1": 1.91664, "rms_b/round1": 3.558107, "rms_c/round1": 2.132012, "cos_a2b/round2": 0.522359, "cos_c2b/round2": 0.44287, "cos_b2a/round2": 0.094161, "cos_b2c/round2": 0.155152, "rms_a/round2": 3.883109, "rms_b/round2": 7.217203, "rms_c/round2": 3.325847}, "alpha_mean": 0.4722950589433168, "alpha_hist": [0.0097, 0.04396, 0.11566, 0.17551, 0.2092, 0.20404, 0.14508, 0.08159, 0.01492, 0.00034], "alpha_by_type": {"special": {"share": 0.00067, "mean": 0.56018, "std": 0.17024}, "whitespace": {"share": 0.04915, "mean": 0.5516, "std": 0.11383}, "digit": {"share": 0.02831, "mean": 0.50905, "std": 0.14012}, "word_start": {"share": 0.56151, "mean": 0.49529, "std": 0.15275}, "word_cont": {"share": 0.25203, "mean": 0.34926, "std": 0.14374}, "punct": {"share": 0.10834, "mean": 0.59323, "std": 0.15468}}, "alpha_p05": 0.19163, "alpha_p50": 0.47195, "alpha_p95": 0.75055}
{"step": 9110, "total_steps": 72479, "tokens": 4776263680, "loss": 5.025534391403198, "l_mix": 3.1119893193244934, "l_a": 3.2061555981636047, "l_c": 3.1670375168323517, "z_loss": 15.87023675441742, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19843778014183044, "tokens_per_second": 1471596.779754683, "eta_seconds": 22576.569022893906, "data_wait_seconds": 2.3931264877319336e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20768, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 9120, "total_steps": 72479, "tokens": 4781506560, "loss": 4.98731142282486, "l_mix": 3.0879409313201904, "l_a": 3.18281489610672, "l_c": 3.1429237723350525, "z_loss": 16.48776936531067, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18850193917751312, "tokens_per_second": 1435240.2217230154, "eta_seconds": 23144.810805344583, "data_wait_seconds": 2.5510787963867188e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20768, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 9130, "total_steps": 72479, "tokens": 4786749440, "loss": 5.152537167072296, "l_mix": 3.190131664276123, "l_a": 3.2860662043094635, "l_c": 3.249781161546707, "z_loss": 16.51352882385254, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1624062955379486, "tokens_per_second": 1314558.0023616361, "eta_seconds": 25265.61814110279, "data_wait_seconds": 3.471970558166504e-05, "mix": {"fineweb_hq": 0.50953, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08237}}
{"step": 9140, "total_steps": 72479, "tokens": 4791992320, "loss": 2.9705276042222977, "l_mix": 1.8337307572364807, "l_a": 1.9114431403577328, "l_c": 1.8717244863510132, "z_loss": 18.464821577072144, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.32116517424583435, "tokens_per_second": 1464957.4137932642, "eta_seconds": 22668.15220656395, "data_wait_seconds": 3.30805778503418e-05, "mix": {"fineweb_hq": 0.50952, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08239}}
{"step": 9150, "total_steps": 72479, "tokens": 4797235200, "loss": 4.765322148799896, "l_mix": 2.9475001394748688, "l_a": 3.0559580326080322, "l_c": 2.9980223774909973, "z_loss": 16.27866220474243, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2514975070953369, "tokens_per_second": 1451522.4392183882, "eta_seconds": 22874.351684069632, "data_wait_seconds": 1.4543533325195312e-05, "mix": {"fineweb_hq": 0.50962, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.0823}}
{"step": 9160, "total_steps": 72479, "tokens": 4802478080, "loss": 4.5817819237709045, "l_mix": 2.833343982696533, "l_a": 2.9376421868801117, "l_c": 2.8850361108779907, "z_loss": 16.34380292892456, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2016671597957611, "tokens_per_second": 1344153.3922289195, "eta_seconds": 24697.621613669395, "data_wait_seconds": 1.9758939743041992e-05, "mix": {"fineweb_hq": 0.50961, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08231}}
{"step": 9170, "total_steps": 72479, "tokens": 4807720960, "loss": 4.5517637729644775, "l_mix": 2.8132075369358063, "l_a": 2.9266718327999115, "l_c": 2.862964540719986, "z_loss": 16.65183115005493, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19778791069984436, "tokens_per_second": 1399981.5474018934, "eta_seconds": 23708.990346050265, "data_wait_seconds": 1.677870750427246e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20763, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 9180, "total_steps": 72479, "tokens": 4812963840, "loss": 4.543740808963776, "l_mix": 2.8083622455596924, "l_a": 2.917935013771057, "l_c": 2.8611955642700195, "z_loss": 16.393446445465088, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19298377633094788, "tokens_per_second": 1376514.615189362, "eta_seconds": 24109.374318146703, "data_wait_seconds": 2.345442771911621e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20763, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 9190, "total_steps": 72479, "tokens": 4818206720, "loss": 4.641452610492706, "l_mix": 2.871446430683136, "l_a": 2.9759977757930756, "l_c": 2.918604850769043, "z_loss": 16.25381636619568, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22223512828350067, "tokens_per_second": 1261575.2206712107, "eta_seconds": 26301.77153792381, "data_wait_seconds": 1.728534698486328e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20762, "cosmopedia_v2": 0.20044, "finemath": 0.08237}}
{"step": 9200, "total_steps": 72479, "tokens": 4823449600, "loss": 3.5021986067295074, "l_mix": 2.1647550463676453, "l_a": 2.2471842169761658, "l_c": 2.204790472984314, "z_loss": 18.51181960105896, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2880263924598694, "tokens_per_second": 1439250.0582186526, "eta_seconds": 23051.18569393158, "data_wait_seconds": 2.8759241104125977e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20761, "cosmopedia_v2": 0.20043, "finemath": 0.08239}, "gates": {"gate_a2b/L1_absmean": 0.18107896, "gate_a2b/L1_absmax": 0.60630804, "gate_a2b/L1_rms": 0.21812774, "gate_a2b/L3_absmean": 0.28414294, "gate_a2b/L3_absmax": 0.46836206, "gate_a2b/L3_rms": 0.2902233, "gate_a2b/L5_absmean": 0.46992061, "gate_a2b/L5_absmax": 0.65232009, "gate_a2b/L5_rms": 0.47819027, "gate_c2b/L1_absmean": 0.01526288, "gate_c2b/L1_absmax": 0.56616974, "gate_c2b/L1_rms": 0.06315535, "gate_c2b/L3_absmean": 0.21887384, "gate_c2b/L3_absmax": 0.45356241, "gate_c2b/L3_rms": 0.22493306, "gate_c2b/L5_absmean": 0.54342574, "gate_c2b/L5_absmax": 0.76938999, "gate_c2b/L5_rms": 0.5577414, "gate_b2a/L7_absmean": 0.07213366, "gate_b2a/L7_absmax": 0.25016242, "gate_b2a/L7_rms": 0.07664081, "gate_b2a/L11_absmean": 0.11852054, "gate_b2a/L11_absmax": 0.3073957, "gate_b2a/L11_rms": 0.12071203, "gate_b2a/L14_absmean": 0.29055911, "gate_b2a/L14_absmax": 0.46149501, "gate_b2a/L14_rms": 0.29163653, "gate_b2c/L3_absmean": 0.05575692, "gate_b2c/L3_absmax": 0.4786773, "gate_b2c/L3_rms": 0.07717229, "gate_b2c/L5_absmean": 0.09640193, "gate_b2c/L5_absmax": 0.54243177, "gate_b2c/L5_rms": 0.10115606, "gate_b2c/L6_absmean": 0.14823133, "gate_b2c/L6_absmax": 0.40474749, "gate_b2c/L6_rms": 0.15038289}, "streams": {"cos_a2b/round0": 0.422347, "cos_c2b/round0": 0.296688, "cos_b2a/round0": 0.191586, "cos_b2c/round0": 0.143893, "rms_a/round0": 1.085247, "rms_b/round0": 1.97566, "rms_c/round0": 1.245732, "cos_a2b/round1": 0.369509, "cos_c2b/round1": 0.235231, "cos_b2a/round1": 0.193549, "cos_b2c/round1": 0.145756, "rms_a/round1": 1.910896, "rms_b/round1": 3.593353, "rms_c/round1": 2.156946, "cos_a2b/round2": 0.452877, "cos_c2b/round2": 0.516896, "cos_b2a/round2": 0.099113, "cos_b2c/round2": 0.119208, "rms_a/round2": 3.753765, "rms_b/round2": 7.852345, "rms_c/round2": 3.513454}, "alpha_mean": 0.42004531970311376, "alpha_hist": [0.01199, 0.04771, 0.14999, 0.25476, 0.23946, 0.17729, 0.0903, 0.02573, 0.00273, 3e-05], "alpha_by_type": {"special": {"share": 0.00055, "mean": 0.56184, "std": 0.10895}, "whitespace": {"share": 0.10498, "mean": 0.50038, "std": 0.12503}, "digit": {"share": 0.15228, "mean": 0.3943, "std": 0.13228}, "word_start": {"share": 0.30347, "mean": 0.43332, "std": 0.14557}, "word_cont": {"share": 0.19051, "mean": 0.34027, "std": 0.1163}, "punct": {"share": 0.24821, "mean": 0.44655, "std": 0.14951}}, "alpha_p05": 0.18536, "alpha_p50": 0.39981, "alpha_p95": 0.64355}
{"step": 9210, "total_steps": 72479, "tokens": 4828692480, "loss": 5.100399494171143, "l_mix": 3.160279482603073, "l_a": 3.248928487300873, "l_c": 3.212831676006317, "z_loss": 15.919137120246887, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.185702845454216, "tokens_per_second": 1450599.2122201566, "eta_seconds": 22867.2242426157, "data_wait_seconds": 1.6063451766967773e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20771, "cosmopedia_v2": 0.20043, "finemath": 0.0823}}
{"step": 9220, "total_steps": 72479, "tokens": 4833935360, "loss": 5.045704007148743, "l_mix": 3.126330316066742, "l_a": 3.2146736681461334, "l_c": 3.1779598891735077, "z_loss": 15.836325526237488, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1795395016670227, "tokens_per_second": 1419267.4821389755, "eta_seconds": 23368.346706581116, "data_wait_seconds": 2.390146255493164e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.2077, "cosmopedia_v2": 0.20043, "finemath": 0.08232}}
{"step": 9230, "total_steps": 72479, "tokens": 4839178240, "loss": 5.201227128505707, "l_mix": 3.221652865409851, "l_a": 3.3142028152942657, "l_c": 3.2788795828819275, "z_loss": 16.493194341659546, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19759196043014526, "tokens_per_second": 1389964.4336906117, "eta_seconds": 23857.223183727263, "data_wait_seconds": 2.110004425048828e-05, "mix": {"fineweb_hq": 0.50953, "dclm": 0.20769, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 9240, "total_steps": 72479, "tokens": 4844421120, "loss": 5.061853647232056, "l_mix": 3.13421294093132, "l_a": 3.2302700579166412, "l_c": 3.189799427986145, "z_loss": 16.19740641117096, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1800462305545807, "tokens_per_second": 1443577.1184037135, "eta_seconds": 22967.563290739057, "data_wait_seconds": 2.3990869522094727e-05, "mix": {"fineweb_hq": 0.50952, "dclm": 0.20768, "cosmopedia_v2": 0.20043, "finemath": 0.08236}}
{"step": 9250, "total_steps": 72479, "tokens": 4849664000, "loss": 5.1474289894104, "l_mix": 3.1887876093387604, "l_a": 3.278098851442337, "l_c": 3.2453369200229645, "z_loss": 16.1055828332901, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16196565330028534, "tokens_per_second": 1445320.1570814385, "eta_seconds": 22936.23719947338, "data_wait_seconds": 2.8461217880249023e-05, "mix": {"fineweb_hq": 0.50951, "dclm": 0.20768, "cosmopedia_v2": 0.20043, "finemath": 0.08238}}
{"step": 9260, "total_steps": 72479, "tokens": 4854906880, "loss": 4.741901993751526, "l_mix": 2.9333861768245697, "l_a": 3.042978525161743, "l_c": 2.9799795150756836, "z_loss": 16.283142685890198, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18685577809810638, "tokens_per_second": 1377198.1582139074, "eta_seconds": 24066.952801465988, "data_wait_seconds": 1.7255544662475586e-05, "mix": {"fineweb_hq": 0.50961, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08229}}
{"step": 9270, "total_steps": 72479, "tokens": 4860149760, "loss": 4.701427698135376, "l_mix": 2.9068396389484406, "l_a": 3.0133600533008575, "l_c": 2.9631468653678894, "z_loss": 16.35958158969879, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19225773215293884, "tokens_per_second": 1399104.1854926583, "eta_seconds": 23686.384856557845, "data_wait_seconds": 1.6421079635620117e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08231}}
{"step": 9280, "total_steps": 72479, "tokens": 4865392640, "loss": 4.5403642654418945, "l_mix": 2.8075211942195892, "l_a": 2.912016600370407, "l_c": 2.858655333518982, "z_loss": 16.413278818130493, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22211380302906036, "tokens_per_second": 1428244.4369701091, "eta_seconds": 23199.444334816933, "data_wait_seconds": 1.5348196029663086e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 9290, "total_steps": 72479, "tokens": 4870635520, "loss": 4.706401705741882, "l_mix": 2.9110099375247955, "l_a": 3.015590190887451, "l_c": 2.9636090993881226, "z_loss": 16.320083618164062, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21013928949832916, "tokens_per_second": 1434523.4112540726, "eta_seconds": 23094.24452197552, "data_wait_seconds": 2.1249055862426758e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 9300, "total_steps": 72479, "tokens": 4875878400, "loss": 4.581408798694611, "l_mix": 2.8331211507320404, "l_a": 2.9359180331230164, "l_c": 2.8863807022571564, "z_loss": 15.980233073234558, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.218380406498909, "tokens_per_second": 1439228.487149933, "eta_seconds": 23015.10277745724, "data_wait_seconds": 2.3633241653442383e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20763, "cosmopedia_v2": 0.20043, "finemath": 0.08237}, "gates": {"gate_a2b/L1_absmean": 0.18344471, "gate_a2b/L1_absmax": 0.60455662, "gate_a2b/L1_rms": 0.21987304, "gate_a2b/L3_absmean": 0.28461328, "gate_a2b/L3_absmax": 0.46763247, "gate_a2b/L3_rms": 0.29073656, "gate_a2b/L5_absmean": 0.47019598, "gate_a2b/L5_absmax": 0.65892231, "gate_a2b/L5_rms": 0.47846261, "gate_c2b/L1_absmean": 0.01577189, "gate_c2b/L1_absmax": 0.55864006, "gate_c2b/L1_rms": 0.06312411, "gate_c2b/L3_absmean": 0.21829286, "gate_c2b/L3_absmax": 0.45349681, "gate_c2b/L3_rms": 0.22432338, "gate_c2b/L5_absmean": 0.54371935, "gate_c2b/L5_absmax": 0.77246624, "gate_c2b/L5_rms": 0.55799741, "gate_b2a/L7_absmean": 0.07221151, "gate_b2a/L7_absmax": 0.25020275, "gate_b2a/L7_rms": 0.07667976, "gate_b2a/L11_absmean": 0.11861029, "gate_b2a/L11_absmax": 0.3108567, "gate_b2a/L11_rms": 0.12076801, "gate_b2a/L14_absmean": 0.29033071, "gate_b2a/L14_absmax": 0.458525, "gate_b2a/L14_rms": 0.29142529, "gate_b2c/L3_absmean": 0.05659094, "gate_b2c/L3_absmax": 0.48235592, "gate_b2c/L3_rms": 0.07797465, "gate_b2c/L5_absmean": 0.09777417, "gate_b2c/L5_absmax": 0.54783016, "gate_b2c/L5_rms": 0.10255596, "gate_b2c/L6_absmean": 0.14919667, "gate_b2c/L6_absmax": 0.40751913, "gate_b2c/L6_rms": 0.15140118}, "streams": {"cos_a2b/round0": 0.367065, "cos_c2b/round0": 0.248634, "cos_b2a/round0": 0.189968, "cos_b2c/round0": 0.113342, "rms_a/round0": 1.070587, "rms_b/round0": 1.991865, "rms_c/round0": 1.330898, "cos_a2b/round1": 0.391857, "cos_c2b/round1": 0.275257, "cos_b2a/round1": 0.216431, "cos_b2c/round1": 0.150919, "rms_a/round1": 1.908566, "rms_b/round1": 3.594239, "rms_c/round1": 2.175685, "cos_a2b/round2": 0.494701, "cos_c2b/round2": 0.470949, "cos_b2a/round2": 0.085027, "cos_b2c/round2": 0.144246, "rms_a/round2": 3.918484, "rms_b/round2": 7.623796, "rms_c/round2": 3.440015}, "alpha_mean": 0.44148167394449445, "alpha_hist": [0.01643, 0.07253, 0.15195, 0.19383, 0.18991, 0.15814, 0.131, 0.07072, 0.01501, 0.00047], "alpha_by_type": {"special": {"share": 0.00098, "mean": 0.64056, "std": 0.11908}, "whitespace": {"share": 0.02707, "mean": 0.57419, "std": 0.12881}, "digit": {"share": 0.03035, "mean": 0.55764, "std": 0.12194}, "word_start": {"share": 0.55055, "mean": 0.45421, "std": 0.1707}, "word_cont": {"share": 0.26018, "mean": 0.33093, "std": 0.14488}, "punct": {"share": 0.13087, "mean": 0.55183, "std": 0.16849}}, "alpha_p05": 0.16941, "alpha_p50": 0.45241, "alpha_p95": 0.72875}
{"step": 9310, "total_steps": 72479, "tokens": 4881121280, "loss": 3.452005386352539, "l_mix": 2.1342224031686783, "l_a": 2.216136947274208, "l_c": 2.1705761700868607, "z_loss": 17.68977451324463, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.23176610469818115, "tokens_per_second": 1292648.0467859579, "eta_seconds": 25620.85538623333, "data_wait_seconds": 3.018975257873535e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20763, "cosmopedia_v2": 0.20043, "finemath": 0.08238}}
{"step": 9320, "total_steps": 72479, "tokens": 4886364160, "loss": 3.4017788767814636, "l_mix": 2.1018827706575394, "l_a": 2.1852338314056396, "l_c": 2.1417746394872665, "z_loss": 17.93557572364807, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.29435381293296814, "tokens_per_second": 1400931.4060220683, "eta_seconds": 23636.778824186324, "data_wait_seconds": 1.65402889251709e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20762, "cosmopedia_v2": 0.20043, "finemath": 0.0824}}
{"step": 9330, "total_steps": 72479, "tokens": 4891607040, "loss": 5.078660726547241, "l_mix": 3.1458753645420074, "l_a": 3.238631933927536, "l_c": 3.1986920535564423, "z_loss": 15.880241513252258, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20550023019313812, "tokens_per_second": 1404074.4773194885, "eta_seconds": 23580.13299636841, "data_wait_seconds": 1.7195940017700195e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20772, "cosmopedia_v2": 0.20043, "finemath": 0.08232}}
{"step": 9340, "total_steps": 72479, "tokens": 4896849920, "loss": 5.1295225620269775, "l_mix": 3.177369564771652, "l_a": 3.2695400714874268, "l_c": 3.232294499874115, "z_loss": 16.02634298801422, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18734309077262878, "tokens_per_second": 1423553.9269747215, "eta_seconds": 23253.78716235161, "data_wait_seconds": 2.9653310775756836e-05, "mix": {"fineweb_hq": 0.50953, "dclm": 0.20771, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 9350, "total_steps": 72479, "tokens": 4902092800, "loss": 5.03545206785202, "l_mix": 3.1184495985507965, "l_a": 3.2133560478687286, "l_c": 3.171246975660324, "z_loss": 16.214701890945435, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17936046421527863, "tokens_per_second": 1433328.5418594645, "eta_seconds": 23091.549624109266, "data_wait_seconds": 1.582503318786621e-05, "mix": {"fineweb_hq": 0.50952, "dclm": 0.2077, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 9360, "total_steps": 72479, "tokens": 4907335680, "loss": 5.116007387638092, "l_mix": 3.1690179109573364, "l_a": 3.258038818836212, "l_c": 3.226474314928055, "z_loss": 16.355334043502808, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17736142873764038, "tokens_per_second": 1178691.1486387253, "eta_seconds": 28075.661983394624, "data_wait_seconds": 1.5676021575927734e-05, "mix": {"fineweb_hq": 0.50951, "dclm": 0.20769, "cosmopedia_v2": 0.20043, "finemath": 0.08237}}
{"step": 9370, "total_steps": 72479, "tokens": 4912578560, "loss": 4.516798138618469, "l_mix": 2.792286455631256, "l_a": 2.897445410490036, "l_c": 2.845453441143036, "z_loss": 16.418520212173462, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15372727811336517, "tokens_per_second": 1420151.5209778186, "eta_seconds": 23298.423374724385, "data_wait_seconds": 2.276897430419922e-05, "mix": {"fineweb_hq": 0.50961, "dclm": 0.20768, "cosmopedia_v2": 0.20043, "finemath": 0.08228}}
{"step": 9380, "total_steps": 72479, "tokens": 4917821440, "loss": 4.598438620567322, "l_mix": 2.8435928523540497, "l_a": 2.951511174440384, "l_c": 2.892547518014908, "z_loss": 16.281155705451965, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18433047831058502, "tokens_per_second": 1364036.7515327572, "eta_seconds": 24253.04778249264, "data_wait_seconds": 2.294778823852539e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20768, "cosmopedia_v2": 0.20043, "finemath": 0.0823}}
{"step": 9390, "total_steps": 72479, "tokens": 4923064320, "loss": 4.485832095146179, "l_mix": 2.773234784603119, "l_a": 2.8765496909618378, "l_c": 2.8266220688819885, "z_loss": 16.458056092262268, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.167775958776474, "tokens_per_second": 1428125.89556607, "eta_seconds": 23160.987231373787, "data_wait_seconds": 1.7076730728149414e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08232}}
{"step": 9400, "total_steps": 72479, "tokens": 4928307200, "loss": 4.575599670410156, "l_mix": 2.8294301331043243, "l_a": 2.935718685388565, "l_c": 2.8794456720352173, "z_loss": 16.20219874382019, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19646693766117096, "tokens_per_second": 1356729.077118713, "eta_seconds": 24375.951919770243, "data_wait_seconds": 1.7464160919189453e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.18539129, "gate_a2b/L1_absmax": 0.60434622, "gate_a2b/L1_rms": 0.22140186, "gate_a2b/L3_absmean": 0.28509909, "gate_a2b/L3_absmax": 0.46646443, "gate_a2b/L3_rms": 0.29117867, "gate_a2b/L5_absmean": 0.47226855, "gate_a2b/L5_absmax": 0.66845149, "gate_a2b/L5_rms": 0.4804171, "gate_c2b/L1_absmean": 0.01572289, "gate_c2b/L1_absmax": 0.55469447, "gate_c2b/L1_rms": 0.0630546, "gate_c2b/L3_absmean": 0.21787147, "gate_c2b/L3_absmax": 0.4493466, "gate_c2b/L3_rms": 0.22371928, "gate_c2b/L5_absmean": 0.54387474, "gate_c2b/L5_absmax": 0.76691759, "gate_c2b/L5_rms": 0.5580073, "gate_b2a/L7_absmean": 0.07216679, "gate_b2a/L7_absmax": 0.24887106, "gate_b2a/L7_rms": 0.07658307, "gate_b2a/L11_absmean": 0.11896447, "gate_b2a/L11_absmax": 0.31452486, "gate_b2a/L11_rms": 0.12108574, "gate_b2a/L14_absmean": 0.29046318, "gate_b2a/L14_absmax": 0.45616275, "gate_b2a/L14_rms": 0.2915186, "gate_b2c/L3_absmean": 0.05713017, "gate_b2c/L3_absmax": 0.48145619, "gate_b2c/L3_rms": 0.07853592, "gate_b2c/L5_absmean": 0.09840876, "gate_b2c/L5_absmax": 0.54935592, "gate_b2c/L5_rms": 0.10321181, "gate_b2c/L6_absmean": 0.14967772, "gate_b2c/L6_absmax": 0.40534282, "gate_b2c/L6_rms": 0.15189114}, "streams": {"cos_a2b/round0": 0.367963, "cos_c2b/round0": 0.247458, "cos_b2a/round0": 0.193942, "cos_b2c/round0": 0.115992, "rms_a/round0": 1.048534, "rms_b/round0": 1.96968, "rms_c/round0": 1.317048, "cos_a2b/round1": 0.397597, "cos_c2b/round1": 0.27242, "cos_b2a/round1": 0.222301, "cos_b2c/round1": 0.158568, "rms_a/round1": 1.894229, "rms_b/round1": 3.545834, "rms_c/round1": 2.15473, "cos_a2b/round2": 0.507265, "cos_c2b/round2": 0.455651, "cos_b2a/round2": 0.076364, "cos_b2c/round2": 0.145816, "rms_a/round2": 3.896423, "rms_b/round2": 7.459334, "rms_c/round2": 3.411376}, "alpha_mean": 0.4682479501279886, "alpha_hist": [0.00957, 0.05405, 0.13507, 0.1842, 0.18764, 0.16589, 0.14313, 0.09875, 0.02121, 0.00049], "alpha_by_type": {"special": {"share": 0.00102, "mean": 0.58344, "std": 0.09681}, "whitespace": {"share": 0.02303, "mean": 0.55836, "std": 0.12019}, "digit": {"share": 0.02249, "mean": 0.53672, "std": 0.13925}, "word_start": {"share": 0.60664, "mean": 0.48951, "std": 0.16761}, "word_cont": {"share": 0.24675, "mean": 0.34131, "std": 0.14407}, "punct": {"share": 0.10007, "mean": 0.61506, "std": 0.15994}}, "alpha_p05": 0.19683, "alpha_p50": 0.46794, "alpha_p95": 0.77219}
{"step": 9410, "total_steps": 72479, "tokens": 4933550080, "loss": 4.577545583248138, "l_mix": 2.8299732506275177, "l_a": 2.938215136528015, "l_c": 2.8814241886138916, "z_loss": 16.804686307907104, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18156488239765167, "tokens_per_second": 1103898.6313267557, "eta_seconds": 29954.127067136764, "data_wait_seconds": 3.096461296081543e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08236}}
{"step": 9420, "total_steps": 72479, "tokens": 4938792960, "loss": 4.23547825217247, "l_mix": 2.592321828007698, "l_a": 2.586416006088257, "l_c": 2.884773477911949, "z_loss": 17.995851755142212, "lr": 0.0023, "wd": 0.1, "grad_norm": 2.823411464691162, "tokens_per_second": 1219890.1431211683, "eta_seconds": 27101.6838511467, "data_wait_seconds": 2.777576446533203e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20764, "cosmopedia_v2": 0.20042, "finemath": 0.08238}}
{"step": 9430, "total_steps": 72479, "tokens": 4944035840, "loss": 3.4250752925872803, "l_mix": 2.11504590511322, "l_a": 2.20749631524086, "l_c": 2.153263211250305, "z_loss": 18.01422691345215, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17609094083309174, "tokens_per_second": 1416608.633279592, "eta_seconds": 23334.48585264683, "data_wait_seconds": 2.35140323638916e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20764, "cosmopedia_v2": 0.20042, "finemath": 0.0824}}
{"step": 9440, "total_steps": 72479, "tokens": 4949278720, "loss": 4.537043869495392, "l_mix": 2.806439310312271, "l_a": 2.9057872593402863, "l_c": 2.8575587570667267, "z_loss": 16.006797790527344, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17722082138061523, "tokens_per_second": 1427434.3327444715, "eta_seconds": 23153.843559622765, "data_wait_seconds": 1.7464160919189453e-05, "mix": {"fineweb_hq": 0.50964, "dclm": 0.20763, "cosmopedia_v2": 0.20042, "finemath": 0.08231}}
{"step": 9450, "total_steps": 72479, "tokens": 4954521600, "loss": 4.6760963797569275, "l_mix": 2.891221672296524, "l_a": 3.0027161240577698, "l_c": 2.941324770450592, "z_loss": 16.62325620651245, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1914782077074051, "tokens_per_second": 1371764.7486088339, "eta_seconds": 24089.661427378654, "data_wait_seconds": 2.1457672119140625e-05, "mix": {"fineweb_hq": 0.50963, "dclm": 0.20762, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 9460, "total_steps": 72479, "tokens": 4959764480, "loss": 4.726808965206146, "l_mix": 2.923340827226639, "l_a": 3.027327060699463, "l_c": 2.9788959622383118, "z_loss": 16.012423634529114, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19977575540542603, "tokens_per_second": 1347118.0558235163, "eta_seconds": 24526.51074578762, "data_wait_seconds": 1.9103288650512695e-05, "mix": {"fineweb_hq": 0.50962, "dclm": 0.20761, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 9470, "total_steps": 72479, "tokens": 4965007360, "loss": 4.695060670375824, "l_mix": 2.903840512037277, "l_a": 3.009309768676758, "l_c": 2.9559280276298523, "z_loss": 16.48737359046936, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15195591747760773, "tokens_per_second": 1410484.720264519, "eta_seconds": 23420.929073095325, "data_wait_seconds": 2.5510787963867188e-05, "mix": {"fineweb_hq": 0.50961, "dclm": 0.2076, "cosmopedia_v2": 0.20042, "finemath": 0.08237}}
{"step": 9480, "total_steps": 72479, "tokens": 4970250240, "loss": 4.832390904426575, "l_mix": 2.9904347956180573, "l_a": 3.093531996011734, "l_c": 3.0409377813339233, "z_loss": 16.150354146957397, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19183216989040375, "tokens_per_second": 1371286.0572562933, "eta_seconds": 24086.600703930857, "data_wait_seconds": 1.9162893295288086e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20759, "cosmopedia_v2": 0.20042, "finemath": 0.08238}}
{"step": 9490, "total_steps": 72479, "tokens": 4975493120, "loss": 4.785066664218903, "l_mix": 2.961197018623352, "l_a": 3.059960275888443, "l_c": 3.0141531825065613, "z_loss": 16.355823040008545, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15876059234142303, "tokens_per_second": 1413450.595283303, "eta_seconds": 23364.365859127043, "data_wait_seconds": 1.4990568161010742e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20769, "cosmopedia_v2": 0.20042, "finemath": 0.0823}}
{"step": 9500, "total_steps": 72479, "tokens": 4980736000, "loss": 4.878036260604858, "l_mix": 3.018324553966522, "l_a": 3.121257573366165, "l_c": 3.07230606675148, "z_loss": 16.424980640411377, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17313288152217865, "tokens_per_second": 1285279.1998739399, "eta_seconds": 25690.24220981598, "data_wait_seconds": 2.0205974578857422e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20768, "cosmopedia_v2": 0.20042, "finemath": 0.08232}, "gates": {"gate_a2b/L1_absmean": 0.18744595, "gate_a2b/L1_absmax": 0.60282868, "gate_a2b/L1_rms": 0.22304809, "gate_a2b/L3_absmean": 0.2859641, "gate_a2b/L3_absmax": 0.46955732, "gate_a2b/L3_rms": 0.29208604, "gate_a2b/L5_absmean": 0.47351161, "gate_a2b/L5_absmax": 0.65641302, "gate_a2b/L5_rms": 0.48153737, "gate_c2b/L1_absmean": 0.01586449, "gate_c2b/L1_absmax": 0.55406439, "gate_c2b/L1_rms": 0.06302189, "gate_c2b/L3_absmean": 0.2178032, "gate_c2b/L3_absmax": 0.45210731, "gate_c2b/L3_rms": 0.22364853, "gate_c2b/L5_absmean": 0.54162568, "gate_c2b/L5_absmax": 0.76544976, "gate_c2b/L5_rms": 0.55557138, "gate_b2a/L7_absmean": 0.07224676, "gate_b2a/L7_absmax": 0.25059822, "gate_b2a/L7_rms": 0.07663999, "gate_b2a/L11_absmean": 0.11908033, "gate_b2a/L11_absmax": 0.30817729, "gate_b2a/L11_rms": 0.12119982, "gate_b2a/L14_absmean": 0.28984764, "gate_b2a/L14_absmax": 0.4668479, "gate_b2a/L14_rms": 0.29095542, "gate_b2c/L3_absmean": 0.05719021, "gate_b2c/L3_absmax": 0.48683432, "gate_b2c/L3_rms": 0.07841203, "gate_b2c/L5_absmean": 0.09860232, "gate_b2c/L5_absmax": 0.55284989, "gate_b2c/L5_rms": 0.10342503, "gate_b2c/L6_absmean": 0.15049188, "gate_b2c/L6_absmax": 0.41166657, "gate_b2c/L6_rms": 0.15272444}, "streams": {"cos_a2b/round0": 0.375004, "cos_c2b/round0": 0.253644, "cos_b2a/round0": 0.18373, "cos_b2c/round0": 0.116679, "rms_a/round0": 1.0606, "rms_b/round0": 1.980198, "rms_c/round0": 1.309995, "cos_a2b/round1": 0.396189, "cos_c2b/round1": 0.270548, "cos_b2a/round1": 0.216838, "cos_b2c/round1": 0.153181, "rms_a/round1": 1.904716, "rms_b/round1": 3.599646, "rms_c/round1": 2.148643, "cos_a2b/round2": 0.508849, "cos_c2b/round2": 0.449859, "cos_b2a/round2": 0.083045, "cos_b2c/round2": 0.141018, "rms_a/round2": 3.885784, "rms_b/round2": 7.459414, "rms_c/round2": 3.385133}, "alpha_mean": 0.4500052532930212, "alpha_hist": [0.01517, 0.06181, 0.14172, 0.18607, 0.19426, 0.18005, 0.13809, 0.07266, 0.01004, 0.00012], "alpha_by_type": {"special": {"share": 0.00116, "mean": 0.5444, "std": 0.12811}, "whitespace": {"share": 0.02722, "mean": 0.51262, "std": 0.1189}, "digit": {"share": 0.03548, "mean": 0.5314, "std": 0.13966}, "word_start": {"share": 0.58284, "mean": 0.47071, "std": 0.16264}, "word_cont": {"share": 0.24663, "mean": 0.32886, "std": 0.14198}, "punct": {"share": 0.10667, "mean": 0.57288, "std": 0.16309}}, "alpha_p05": 0.17554, "alpha_p50": 0.45108, "alpha_p95": 0.74762}
{"step": 9510, "total_steps": 72479, "tokens": 4985978880, "loss": 4.762181341648102, "l_mix": 2.944904536008835, "l_a": 3.0517871379852295, "l_c": 3.0004013180732727, "z_loss": 16.202125906944275, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16798633337020874, "tokens_per_second": 1248070.2396468164, "eta_seconds": 26451.949596476552, "data_wait_seconds": 4.0143728256225586e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20768, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 9520, "total_steps": 72479, "tokens": 4991221760, "loss": 4.779010534286499, "l_mix": 2.9550870954990387, "l_a": 3.064067155122757, "l_c": 3.0103276669979095, "z_loss": 16.048966884613037, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19194026291370392, "tokens_per_second": 1365386.8564817873, "eta_seconds": 24175.308291053774, "data_wait_seconds": 3.713369369506836e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 9530, "total_steps": 72479, "tokens": 4996464640, "loss": 4.812471926212311, "l_mix": 2.977357119321823, "l_a": 3.081064850091934, "l_c": 3.030574291944504, "z_loss": 16.230385184288025, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1926085352897644, "tokens_per_second": 1338439.4688490485, "eta_seconds": 24658.123195052147, "data_wait_seconds": 1.9371509552001953e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08237}}
{"step": 9540, "total_steps": 72479, "tokens": 5001707520, "loss": 3.274374395608902, "l_mix": 2.0217345654964447, "l_a": 2.1087049543857574, "l_c": 2.0605606138706207, "z_loss": 18.601678371429443, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18722280859947205, "tokens_per_second": 1474975.685998638, "eta_seconds": 22372.004328775405, "data_wait_seconds": 2.4825334548950195e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08239}}
{"step": 9550, "total_steps": 72479, "tokens": 5006950400, "loss": 4.910777807235718, "l_mix": 3.037939965724945, "l_a": 3.1433771550655365, "l_c": 3.094055026769638, "z_loss": 16.081088423728943, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1764647215604782, "tokens_per_second": 1433932.2260972252, "eta_seconds": 23008.702190756798, "data_wait_seconds": 3.2514333724975586e-05, "mix": {"fineweb_hq": 0.50963, "dclm": 0.20764, "cosmopedia_v2": 0.20042, "finemath": 0.0823}}
{"step": 9560, "total_steps": 72479, "tokens": 5012193280, "loss": 4.924818634986877, "l_mix": 3.0481393337249756, "l_a": 3.149524450302124, "l_c": 3.100824147462845, "z_loss": 15.746730089187622, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1842561662197113, "tokens_per_second": 1371604.2351447444, "eta_seconds": 24050.433665013315, "data_wait_seconds": 2.0772218704223633e-05, "mix": {"fineweb_hq": 0.50962, "dclm": 0.20764, "cosmopedia_v2": 0.20042, "finemath": 0.08232}}
{"step": 9570, "total_steps": 72479, "tokens": 5017436160, "loss": 4.846460700035095, "l_mix": 2.997818797826767, "l_a": 3.104077845811844, "l_c": 3.0527533292770386, "z_loss": 15.925003290176392, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1658097803592682, "tokens_per_second": 1377703.6007315598, "eta_seconds": 23940.15213031769, "data_wait_seconds": 1.6987323760986328e-05, "mix": {"fineweb_hq": 0.50961, "dclm": 0.20763, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 9580, "total_steps": 72479, "tokens": 5022679040, "loss": 5.015244126319885, "l_mix": 3.1040362417697906, "l_a": 3.2084943652153015, "l_c": 3.1568700671195984, "z_loss": 15.985493540763855, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18690694868564606, "tokens_per_second": 1421963.2711560226, "eta_seconds": 23191.309917020797, "data_wait_seconds": 1.531839370727539e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20762, "cosmopedia_v2": 0.20042, "finemath": 0.08236}}
{"step": 9590, "total_steps": 72479, "tokens": 5027921920, "loss": 4.972888827323914, "l_mix": 3.0788447856903076, "l_a": 3.1794417798519135, "l_c": 3.12854540348053, "z_loss": 16.47764301300049, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2088308334350586, "tokens_per_second": 1396916.6491404925, "eta_seconds": 23603.375371241567, "data_wait_seconds": 1.7255544662475586e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20761, "cosmopedia_v2": 0.20042, "finemath": 0.08238}}
{"step": 9600, "total_steps": 72479, "tokens": 5033164800, "loss": 4.843376457691193, "l_mix": 2.99555441737175, "l_a": 3.0983089804649353, "l_c": 3.0556898713111877, "z_loss": 16.224092960357666, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21567638218402863, "tokens_per_second": 1323087.320043075, "eta_seconds": 24916.49995627403, "data_wait_seconds": 1.6868114471435547e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.2076, "cosmopedia_v2": 0.20042, "finemath": 0.0824}, "gates": {"gate_a2b/L1_absmean": 0.18790746, "gate_a2b/L1_absmax": 0.60572034, "gate_a2b/L1_rms": 0.22343843, "gate_a2b/L3_absmean": 0.28637013, "gate_a2b/L3_absmax": 0.47021016, "gate_a2b/L3_rms": 0.29237407, "gate_a2b/L5_absmean": 0.47352552, "gate_a2b/L5_absmax": 0.6615507, "gate_a2b/L5_rms": 0.48153958, "gate_c2b/L1_absmean": 0.01623786, "gate_c2b/L1_absmax": 0.55209529, "gate_c2b/L1_rms": 0.06327625, "gate_c2b/L3_absmean": 0.217274, "gate_c2b/L3_absmax": 0.44709557, "gate_c2b/L3_rms": 0.22315547, "gate_c2b/L5_absmean": 0.54213303, "gate_c2b/L5_absmax": 0.7697553, "gate_c2b/L5_rms": 0.55615658, "gate_b2a/L7_absmean": 0.07287358, "gate_b2a/L7_absmax": 0.24954428, "gate_b2a/L7_rms": 0.07722076, "gate_b2a/L11_absmean": 0.11892197, "gate_b2a/L11_absmax": 0.30703416, "gate_b2a/L11_rms": 0.12102264, "gate_b2a/L14_absmean": 0.29075795, "gate_b2a/L14_absmax": 0.4640767, "gate_b2a/L14_rms": 0.29185903, "gate_b2c/L3_absmean": 0.05778189, "gate_b2c/L3_absmax": 0.48898649, "gate_b2c/L3_rms": 0.07898971, "gate_b2c/L5_absmean": 0.09856187, "gate_b2c/L5_absmax": 0.55376703, "gate_b2c/L5_rms": 0.10352303, "gate_b2c/L6_absmean": 0.15005682, "gate_b2c/L6_absmax": 0.41136926, "gate_b2c/L6_rms": 0.1523208}, "streams": {"cos_a2b/round0": 0.369339, "cos_c2b/round0": 0.247562, "cos_b2a/round0": 0.185156, "cos_b2c/round0": 0.115291, "rms_a/round0": 1.052759, "rms_b/round0": 1.971146, "rms_c/round0": 1.316489, "cos_a2b/round1": 0.394195, "cos_c2b/round1": 0.267643, "cos_b2a/round1": 0.211614, "cos_b2c/round1": 0.157, "rms_a/round1": 1.889804, "rms_b/round1": 3.576477, "rms_c/round1": 2.134414, "cos_a2b/round2": 0.512392, "cos_c2b/round2": 0.443943, "cos_b2a/round2": 0.074483, "cos_b2c/round2": 0.144272, "rms_a/round2": 3.846564, "rms_b/round2": 7.354207, "rms_c/round2": 3.337883}, "alpha_mean": 0.43620915129006477, "alpha_hist": [0.02049, 0.07474, 0.14973, 0.19655, 0.18723, 0.16971, 0.12277, 0.06715, 0.01147, 0.00015], "alpha_by_type": {"special": {"share": 0.00107, "mean": 0.55427, "std": 0.09702}, "whitespace": {"share": 0.02376, "mean": 0.51084, "std": 0.12453}, "digit": {"share": 0.02919, "mean": 0.47991, "std": 0.13381}, "word_start": {"share": 0.58942, "mean": 0.46215, "std": 0.16249}, "word_cont": {"share": 0.25462, "mean": 0.31032, "std": 0.1472}, "punct": {"share": 0.10194, "mean": 0.56949, "std": 0.17022}}, "alpha_p05": 0.16968, "alpha_p50": 0.4225, "alpha_p95": 0.71183}
{"step": 9610, "total_steps": 72479, "tokens": 5038407680, "loss": 4.970601677894592, "l_mix": 3.0780939161777496, "l_a": 3.1706471145153046, "l_c": 3.1322599053382874, "z_loss": 16.355053186416626, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17803847789764404, "tokens_per_second": 1397163.030143267, "eta_seconds": 23591.708026099208, "data_wait_seconds": 1.7851591110229492e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.2077, "cosmopedia_v2": 0.20042, "finemath": 0.08231}}
{"step": 9620, "total_steps": 72479, "tokens": 5043650560, "loss": 4.997143030166626, "l_mix": 3.094250649213791, "l_a": 3.190237134695053, "l_c": 3.14738929271698, "z_loss": 16.04495358467102, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18133114278316498, "tokens_per_second": 1428012.3814000855, "eta_seconds": 23078.38490846157, "data_wait_seconds": 4.3004751205444336e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20769, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 9630, "total_steps": 72479, "tokens": 5048893440, "loss": 5.0080941915512085, "l_mix": 3.1010273694992065, "l_a": 3.191970616579056, "l_c": 3.1594922840595245, "z_loss": 16.28000009059906, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17044973373413086, "tokens_per_second": 1283969.8125785906, "eta_seconds": 25663.357649993894, "data_wait_seconds": 1.7970800399780273e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20768, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 9640, "total_steps": 72479, "tokens": 5054136320, "loss": 4.894431471824646, "l_mix": 3.0313603281974792, "l_a": 3.1243983805179596, "l_c": 3.0803744792938232, "z_loss": 16.392398834228516, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1940694898366928, "tokens_per_second": 1369732.0178181627, "eta_seconds": 24052.68563735485, "data_wait_seconds": 1.6242265701293945e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20768, "cosmopedia_v2": 0.20041, "finemath": 0.08237}}
{"step": 9650, "total_steps": 72479, "tokens": 5059379200, "loss": 3.3682673275470734, "l_mix": 2.07994444668293, "l_a": 2.175013318657875, "l_c": 2.1135027557611465, "z_loss": 17.67952871322632, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.28758448362350464, "tokens_per_second": 1395938.196427566, "eta_seconds": 23597.3847812891, "data_wait_seconds": 2.047419548034668e-05, "mix": {"fineweb_hq": 0.50953, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08238}}
{"step": 9660, "total_steps": 72479, "tokens": 5064622080, "loss": 4.754320383071899, "l_mix": 2.9418074786663055, "l_a": 3.042888879776001, "l_c": 2.993430584669113, "z_loss": 16.169530391693115, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.14854411780834198, "tokens_per_second": 1439161.2347807365, "eta_seconds": 22885.029888272282, "data_wait_seconds": 1.9162893295288086e-05, "mix": {"fineweb_hq": 0.50963, "dclm": 0.20766, "cosmopedia_v2": 0.20041, "finemath": 0.0823}}
{"step": 9670, "total_steps": 72479, "tokens": 5069864960, "loss": 4.835409879684448, "l_mix": 2.991169184446335, "l_a": 3.097249209880829, "l_c": 3.044764667749405, "z_loss": 16.365480422973633, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15907415747642517, "tokens_per_second": 1415602.9568292014, "eta_seconds": 23262.175904011725, "data_wait_seconds": 2.3365020751953125e-05, "mix": {"fineweb_hq": 0.50962, "dclm": 0.20765, "cosmopedia_v2": 0.20041, "finemath": 0.08232}}
{"step": 9680, "total_steps": 72479, "tokens": 5075107840, "loss": 4.848041296005249, "l_mix": 3.000039130449295, "l_a": 3.1009731590747833, "l_c": 3.0535764396190643, "z_loss": 16.371357917785645, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15002460777759552, "tokens_per_second": 1429703.6055859404, "eta_seconds": 23029.08237998486, "data_wait_seconds": 1.8656253814697266e-05, "mix": {"fineweb_hq": 0.50961, "dclm": 0.20764, "cosmopedia_v2": 0.20041, "finemath": 0.08233}}
{"step": 9690, "total_steps": 72479, "tokens": 5080350720, "loss": 4.827189803123474, "l_mix": 2.986266791820526, "l_a": 3.0936162173748016, "l_c": 3.0372851490974426, "z_loss": 16.52474546432495, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18436655402183533, "tokens_per_second": 1458284.7473795419, "eta_seconds": 22574.136698031423, "data_wait_seconds": 2.3573637008666992e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20764, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 9700, "total_steps": 72479, "tokens": 5085593600, "loss": 4.96379679441452, "l_mix": 3.072484076023102, "l_a": 3.1733652353286743, "l_c": 3.125560462474823, "z_loss": 16.347771167755127, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20040160417556763, "tokens_per_second": 1419973.3414299155, "eta_seconds": 23179.50301718712, "data_wait_seconds": 1.8298625946044922e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20763, "cosmopedia_v2": 0.20041, "finemath": 0.08237}, "gates": {"gate_a2b/L1_absmean": 0.18985699, "gate_a2b/L1_absmax": 0.61657178, "gate_a2b/L1_rms": 0.22520548, "gate_a2b/L3_absmean": 0.28692245, "gate_a2b/L3_absmax": 0.47200066, "gate_a2b/L3_rms": 0.29295167, "gate_a2b/L5_absmean": 0.47356978, "gate_a2b/L5_absmax": 0.66872561, "gate_a2b/L5_rms": 0.48159653, "gate_c2b/L1_absmean": 0.01587682, "gate_c2b/L1_absmax": 0.54833418, "gate_c2b/L1_rms": 0.06344639, "gate_c2b/L3_absmean": 0.21682477, "gate_c2b/L3_absmax": 0.44132191, "gate_c2b/L3_rms": 0.22257343, "gate_c2b/L5_absmean": 0.54264432, "gate_c2b/L5_absmax": 0.77707314, "gate_c2b/L5_rms": 0.55663407, "gate_b2a/L7_absmean": 0.0727222, "gate_b2a/L7_absmax": 0.24945833, "gate_b2a/L7_rms": 0.07697448, "gate_b2a/L11_absmean": 0.11945337, "gate_b2a/L11_absmax": 0.30960795, "gate_b2a/L11_rms": 0.12153301, "gate_b2a/L14_absmean": 0.29085812, "gate_b2a/L14_absmax": 0.46631476, "gate_b2a/L14_rms": 0.29196751, "gate_b2c/L3_absmean": 0.05790659, "gate_b2c/L3_absmax": 0.49136242, "gate_b2c/L3_rms": 0.07948493, "gate_b2c/L5_absmean": 0.09863862, "gate_b2c/L5_absmax": 0.56346011, "gate_b2c/L5_rms": 0.10364308, "gate_b2c/L6_absmean": 0.15054566, "gate_b2c/L6_absmax": 0.40963948, "gate_b2c/L6_rms": 0.15287177}, "streams": {"cos_a2b/round0": 0.357505, "cos_c2b/round0": 0.230527, "cos_b2a/round0": 0.184183, "cos_b2c/round0": 0.106751, "rms_a/round0": 1.050034, "rms_b/round0": 1.99999, "rms_c/round0": 1.32579, "cos_a2b/round1": 0.377421, "cos_c2b/round1": 0.264061, "cos_b2a/round1": 0.215866, "cos_b2c/round1": 0.150613, "rms_a/round1": 1.860939, "rms_b/round1": 3.635663, "rms_c/round1": 2.134077, "cos_a2b/round2": 0.499468, "cos_c2b/round2": 0.452903, "cos_b2a/round2": 0.078543, "cos_b2c/round2": 0.147185, "rms_a/round2": 3.865721, "rms_b/round2": 7.582499, "rms_c/round2": 3.354901}, "alpha_mean": 0.4446332410634426, "alpha_hist": [0.0146, 0.05948, 0.14108, 0.20927, 0.21516, 0.15994, 0.10791, 0.07167, 0.02039, 0.00049], "alpha_by_type": {"special": {"share": 0.00107, "mean": 0.58637, "std": 0.10591}, "whitespace": {"share": 0.02081, "mean": 0.55051, "std": 0.12289}, "digit": {"share": 0.02972, "mean": 0.47771, "std": 0.13914}, "word_start": {"share": 0.5218, "mean": 0.47754, "std": 0.16212}, "word_cont": {"share": 0.33206, "mean": 0.33971, "std": 0.13794}, "punct": {"share": 0.09453, "mean": 0.59624, "std": 0.16503}}, "alpha_p05": 0.18803, "alpha_p50": 0.43686, "alpha_p95": 0.74688}
{"step": 9710, "total_steps": 72479, "tokens": 5090836480, "loss": 4.7874420285224915, "l_mix": 2.9605022370815277, "l_a": 3.067053258419037, "l_c": 3.017277956008911, "z_loss": 16.401734352111816, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22263914346694946, "tokens_per_second": 1404471.918747656, "eta_seconds": 23431.606593704222, "data_wait_seconds": 1.5079975128173828e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20762, "cosmopedia_v2": 0.20041, "finemath": 0.08239}}
{"step": 9720, "total_steps": 72479, "tokens": 5096079360, "loss": 4.951560318470001, "l_mix": 3.068100541830063, "l_a": 3.158165603876114, "l_c": 3.114659309387207, "z_loss": 16.121264338493347, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18193471431732178, "tokens_per_second": 1460453.9005209627, "eta_seconds": 22529.838552427293, "data_wait_seconds": 1.5556812286376953e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20772, "cosmopedia_v2": 0.20041, "finemath": 0.0823}}
{"step": 9730, "total_steps": 72479, "tokens": 5101322240, "loss": 4.981164991855621, "l_mix": 3.084479957818985, "l_a": 3.177934944629669, "l_c": 3.1388998329639435, "z_loss": 16.345973253250122, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19938212633132935, "tokens_per_second": 1454879.8533041435, "eta_seconds": 22612.552945375443, "data_wait_seconds": 1.5228986740112305e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20771, "cosmopedia_v2": 0.20041, "finemath": 0.08232}}
{"step": 9740, "total_steps": 72479, "tokens": 5106565120, "loss": 5.034943342208862, "l_mix": 3.1193586587905884, "l_a": 3.2072333693504333, "l_c": 3.1725293695926666, "z_loss": 16.559219121932983, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1870507001876831, "tokens_per_second": 1374518.309904159, "eta_seconds": 23930.786949133875, "data_wait_seconds": 1.811981201171875e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.2077, "cosmopedia_v2": 0.20041, "finemath": 0.08234}}
{"step": 9750, "total_steps": 72479, "tokens": 5111808000, "loss": 5.067855060100555, "l_mix": 3.1384296119213104, "l_a": 3.2305326759815216, "l_c": 3.1954540014266968, "z_loss": 16.29230499267578, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18040567636489868, "tokens_per_second": 1473173.9505619036, "eta_seconds": 22324.629036140443, "data_wait_seconds": 2.0951032638549805e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20769, "cosmopedia_v2": 0.20041, "finemath": 0.08236}}
{"step": 9760, "total_steps": 72479, "tokens": 5117050880, "loss": 4.976727604866028, "l_mix": 3.0823446214199066, "l_a": 3.1742317378520966, "l_c": 3.134894907474518, "z_loss": 16.449129581451416, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.14260202646255493, "tokens_per_second": 1377074.5719337887, "eta_seconds": 23878.749736714362, "data_wait_seconds": 1.7642974853515625e-05, "mix": {"fineweb_hq": 0.50953, "dclm": 0.20768, "cosmopedia_v2": 0.20041, "finemath": 0.08238}}
{"step": 9770, "total_steps": 72479, "tokens": 5122293760, "loss": 4.905583202838898, "l_mix": 3.035706579685211, "l_a": 3.1383300125598907, "l_c": 3.089098483324051, "z_loss": 16.480223655700684, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19607867300510406, "tokens_per_second": 1354747.0571385482, "eta_seconds": 24268.42414512634, "data_wait_seconds": 2.1457672119140625e-05, "mix": {"fineweb_hq": 0.50962, "dclm": 0.20768, "cosmopedia_v2": 0.20041, "finemath": 0.08229}}
{"step": 9780, "total_steps": 72479, "tokens": 5127536640, "loss": 4.748657286167145, "l_mix": 2.9367475509643555, "l_a": 3.0436713099479675, "l_c": 2.9904611706733704, "z_loss": 16.699716091156006, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2209872156381607, "tokens_per_second": 1472181.983302058, "eta_seconds": 22328.98764204979, "data_wait_seconds": 1.6868114471435547e-05, "mix": {"fineweb_hq": 0.50961, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08231}}
{"step": 9790, "total_steps": 72479, "tokens": 5132779520, "loss": 4.842465937137604, "l_mix": 2.99600350856781, "l_a": 3.1006791591644287, "l_c": 3.0487849712371826, "z_loss": 16.230956077575684, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19167956709861755, "tokens_per_second": 1443692.9311081483, "eta_seconds": 22765.984181118012, "data_wait_seconds": 2.0176172256469727e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20766, "cosmopedia_v2": 0.20041, "finemath": 0.08233}}
{"step": 9800, "total_steps": 72479, "tokens": 5138022400, "loss": 4.898695588111877, "l_mix": 3.0318816006183624, "l_a": 3.1333709955215454, "l_c": 3.0838306844234467, "z_loss": 16.534024000167847, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16557198762893677, "tokens_per_second": 1337412.8284279264, "eta_seconds": 24571.207074952126, "data_wait_seconds": 1.9073486328125e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20765, "cosmopedia_v2": 0.20041, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.19114266, "gate_a2b/L1_absmax": 0.61080551, "gate_a2b/L1_rms": 0.22624536, "gate_a2b/L3_absmean": 0.28740522, "gate_a2b/L3_absmax": 0.47123671, "gate_a2b/L3_rms": 0.2933816, "gate_a2b/L5_absmean": 0.47457486, "gate_a2b/L5_absmax": 0.65953571, "gate_a2b/L5_rms": 0.48261991, "gate_c2b/L1_absmean": 0.01584819, "gate_c2b/L1_absmax": 0.54635495, "gate_c2b/L1_rms": 0.0630739, "gate_c2b/L3_absmean": 0.21659516, "gate_c2b/L3_absmax": 0.438674, "gate_c2b/L3_rms": 0.22239661, "gate_c2b/L5_absmean": 0.54248381, "gate_c2b/L5_absmax": 0.76164299, "gate_c2b/L5_rms": 0.55652714, "gate_b2a/L7_absmean": 0.07243847, "gate_b2a/L7_absmax": 0.2469264, "gate_b2a/L7_rms": 0.07682472, "gate_b2a/L11_absmean": 0.1192468, "gate_b2a/L11_absmax": 0.31263393, "gate_b2a/L11_rms": 0.12136934, "gate_b2a/L14_absmean": 0.29104185, "gate_b2a/L14_absmax": 0.45962995, "gate_b2a/L14_rms": 0.29211709, "gate_b2c/L3_absmean": 0.0587625, "gate_b2c/L3_absmax": 0.4899236, "gate_b2c/L3_rms": 0.08002777, "gate_b2c/L5_absmean": 0.09938735, "gate_b2c/L5_absmax": 0.56059808, "gate_b2c/L5_rms": 0.10434383, "gate_b2c/L6_absmean": 0.15119046, "gate_b2c/L6_absmax": 0.41196239, "gate_b2c/L6_rms": 0.15351917}, "streams": {"cos_a2b/round0": 0.379262, "cos_c2b/round0": 0.255181, "cos_b2a/round0": 0.184544, "cos_b2c/round0": 0.12067, "rms_a/round0": 1.039872, "rms_b/round0": 1.939859, "rms_c/round0": 1.312809, "cos_a2b/round1": 0.397021, "cos_c2b/round1": 0.26634, "cos_b2a/round1": 0.214328, "cos_b2c/round1": 0.152261, "rms_a/round1": 1.873796, "rms_b/round1": 3.561593, "rms_c/round1": 2.14969, "cos_a2b/round2": 0.508367, "cos_c2b/round2": 0.456965, "cos_b2a/round2": 0.081057, "cos_b2c/round2": 0.137993, "rms_a/round2": 3.855132, "rms_b/round2": 7.491054, "rms_c/round2": 3.373293}, "alpha_mean": 0.45015091957723996, "alpha_hist": [0.01271, 0.05518, 0.13927, 0.20164, 0.20177, 0.1768, 0.13, 0.0703, 0.01215, 0.00018], "alpha_by_type": {"special": {"share": 0.00089, "mean": 0.60948, "std": 0.11628}, "whitespace": {"share": 0.0264, "mean": 0.54084, "std": 0.12526}, "digit": {"share": 0.04678, "mean": 0.47173, "std": 0.12494}, "word_start": {"share": 0.54781, "mean": 0.47312, "std": 0.15993}, "word_cont": {"share": 0.26245, "mean": 0.33424, "std": 0.13892}, "punct": {"share": 0.11568, "mean": 0.57373, "std": 0.15865}}, "alpha_p05": 0.17696, "alpha_p50": 0.43374, "alpha_p95": 0.75563}
{"step": 9810, "total_steps": 72479, "tokens": 5143265280, "loss": 4.787796080112457, "l_mix": 2.961370199918747, "l_a": 3.0688926577568054, "l_c": 3.0137607753276825, "z_loss": 16.298818588256836, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15522059798240662, "tokens_per_second": 1385881.8735387025, "eta_seconds": 23708.084577298167, "data_wait_seconds": 1.7821788787841797e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20041, "finemath": 0.08236}}
{"step": 9820, "total_steps": 72479, "tokens": 5148508160, "loss": 4.881583869457245, "l_mix": 3.0210146605968475, "l_a": 3.1217251420021057, "l_c": 3.074727565050125, "z_loss": 16.33338689804077, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18065448105335236, "tokens_per_second": 1400528.4729407895, "eta_seconds": 23456.404083681107, "data_wait_seconds": 1.627206802368164e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20764, "cosmopedia_v2": 0.20041, "finemath": 0.08238}}
{"step": 9830, "total_steps": 72479, "tokens": 5153751040, "loss": 3.476786285638809, "l_mix": 2.1478533893823624, "l_a": 2.2297704219818115, "l_c": 2.193833887577057, "z_loss": 18.51550602912903, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.3845876157283783, "tokens_per_second": 1356430.1452660256, "eta_seconds": 24215.120127367973, "data_wait_seconds": 4.532933235168457e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20763, "cosmopedia_v2": 0.20041, "finemath": 0.0824}}
{"step": 9840, "total_steps": 72479, "tokens": 5158993920, "loss": 5.0190815925598145, "l_mix": 3.1091725528240204, "l_a": 3.201428860425949, "l_c": 3.1593925058841705, "z_loss": 16.62487769126892, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2025337517261505, "tokens_per_second": 1467030.9825474527, "eta_seconds": 22385.945779395104, "data_wait_seconds": 2.5004148483276367e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20772, "cosmopedia_v2": 0.20041, "finemath": 0.08232}}
{"step": 9850, "total_steps": 72479, "tokens": 5164236800, "loss": 5.053171277046204, "l_mix": 3.129250317811966, "l_a": 3.2188510298728943, "l_c": 3.1888328790664673, "z_loss": 16.156545758247375, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20203179121017456, "tokens_per_second": 1383033.4070746284, "eta_seconds": 23741.749826169016, "data_wait_seconds": 1.9103288650512695e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20772, "cosmopedia_v2": 0.20041, "finemath": 0.08234}}
{"step": 9860, "total_steps": 72479, "tokens": 5169479680, "loss": 4.948859691619873, "l_mix": 3.0649354457855225, "l_a": 3.1546750962734222, "l_c": 3.1194352209568024, "z_loss": 16.910749197006226, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17255358397960663, "tokens_per_second": 1356141.3810553555, "eta_seconds": 24208.678188443184, "data_wait_seconds": 1.6480684280395508e-05, "mix": {"fineweb_hq": 0.50953, "dclm": 0.20771, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 9870, "total_steps": 72479, "tokens": 5174722560, "loss": 4.941249370574951, "l_mix": 3.059684604406357, "l_a": 3.151052176952362, "l_c": 3.1154086589813232, "z_loss": 16.26413142681122, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16123411059379578, "tokens_per_second": 1471950.9989093354, "eta_seconds": 22300.43487610817, "data_wait_seconds": 1.9729137420654297e-05, "mix": {"fineweb_hq": 0.50952, "dclm": 0.2077, "cosmopedia_v2": 0.20041, "finemath": 0.08237}}
{"step": 9880, "total_steps": 72479, "tokens": 5179965440, "loss": 4.749274969100952, "l_mix": 2.9378945529460907, "l_a": 3.04326269030571, "l_c": 2.9891773760318756, "z_loss": 16.48367929458618, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16431017220020294, "tokens_per_second": 1465324.359045317, "eta_seconds": 22397.70622074604, "data_wait_seconds": 1.5467405319213867e-05, "mix": {"fineweb_hq": 0.50962, "dclm": 0.20769, "cosmopedia_v2": 0.2004, "finemath": 0.08229}}
{"step": 9890, "total_steps": 72479, "tokens": 5185208320, "loss": 4.8432793617248535, "l_mix": 2.9958814084529877, "l_a": 3.101706236600876, "l_c": 3.0508465468883514, "z_loss": 16.3198219537735, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16019761562347412, "tokens_per_second": 1424724.1484314918, "eta_seconds": 23032.29131627083, "data_wait_seconds": 1.4960765838623047e-05, "mix": {"fineweb_hq": 0.50961, "dclm": 0.20768, "cosmopedia_v2": 0.2004, "finemath": 0.08231}}
{"step": 9900, "total_steps": 72479, "tokens": 5190451200, "loss": 4.710314571857452, "l_mix": 2.913423627614975, "l_a": 3.0164695084095, "l_c": 2.967716485261917, "z_loss": 16.351803064346313, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16154798865318298, "tokens_per_second": 1360187.224823705, "eta_seconds": 24121.2519520998, "data_wait_seconds": 1.5944242477416992e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20768, "cosmopedia_v2": 0.2004, "finemath": 0.08232}, "gates": {"gate_a2b/L1_absmean": 0.19365889, "gate_a2b/L1_absmax": 0.60377032, "gate_a2b/L1_rms": 0.22850317, "gate_a2b/L3_absmean": 0.28889289, "gate_a2b/L3_absmax": 0.47243041, "gate_a2b/L3_rms": 0.29475603, "gate_a2b/L5_absmean": 0.47625166, "gate_a2b/L5_absmax": 0.6543225, "gate_a2b/L5_rms": 0.48421425, "gate_c2b/L1_absmean": 0.01525975, "gate_c2b/L1_absmax": 0.54916042, "gate_c2b/L1_rms": 0.06250191, "gate_c2b/L3_absmean": 0.21582377, "gate_c2b/L3_absmax": 0.43670234, "gate_c2b/L3_rms": 0.22156858, "gate_c2b/L5_absmean": 0.54166597, "gate_c2b/L5_absmax": 0.7676124, "gate_c2b/L5_rms": 0.55557466, "gate_b2a/L7_absmean": 0.0728045, "gate_b2a/L7_absmax": 0.25081038, "gate_b2a/L7_rms": 0.07716918, "gate_b2a/L11_absmean": 0.11956984, "gate_b2a/L11_absmax": 0.31661996, "gate_b2a/L11_rms": 0.12169015, "gate_b2a/L14_absmean": 0.29114124, "gate_b2a/L14_absmax": 0.45971516, "gate_b2a/L14_rms": 0.29220963, "gate_b2c/L3_absmean": 0.0595245, "gate_b2c/L3_absmax": 0.49437413, "gate_b2c/L3_rms": 0.08080527, "gate_b2c/L5_absmean": 0.10010438, "gate_b2c/L5_absmax": 0.55913168, "gate_b2c/L5_rms": 0.10497213, "gate_b2c/L6_absmean": 0.15191421, "gate_b2c/L6_absmax": 0.40721712, "gate_b2c/L6_rms": 0.1542123}, "streams": {"cos_a2b/round0": 0.372664, "cos_c2b/round0": 0.254234, "cos_b2a/round0": 0.18367, "cos_b2c/round0": 0.115296, "rms_a/round0": 1.03478, "rms_b/round0": 1.931073, "rms_c/round0": 1.313321, "cos_a2b/round1": 0.396055, "cos_c2b/round1": 0.269459, "cos_b2a/round1": 0.211864, "cos_b2c/round1": 0.156108, "rms_a/round1": 1.859888, "rms_b/round1": 3.509115, "rms_c/round1": 2.12777, "cos_a2b/round2": 0.507432, "cos_c2b/round2": 0.458925, "cos_b2a/round2": 0.072926, "cos_b2c/round2": 0.139233, "rms_a/round2": 3.805787, "rms_b/round2": 7.432339, "rms_c/round2": 3.331659}, "alpha_mean": 0.4389544220633397, "alpha_hist": [0.01613, 0.07753, 0.1552, 0.18599, 0.18532, 0.17107, 0.12659, 0.06938, 0.01259, 0.00021], "alpha_by_type": {"special": {"share": 0.0011, "mean": 0.56236, "std": 0.09251}, "whitespace": {"share": 0.02785, "mean": 0.53873, "std": 0.11872}, "digit": {"share": 0.03239, "mean": 0.43724, "std": 0.12216}, "word_start": {"share": 0.57782, "mean": 0.4612, "std": 0.16976}, "word_cont": {"share": 0.24191, "mean": 0.32183, "std": 0.14208}, "punct": {"share": 0.11893, "mean": 0.54509, "std": 0.18286}}, "alpha_p05": 0.13893, "alpha_p50": 0.42871, "alpha_p95": 0.74798}
{"step": 9910, "total_steps": 72479, "tokens": 5195694080, "loss": 4.773261845111847, "l_mix": 2.9535011649131775, "l_a": 3.055430829524994, "l_c": 3.00495982170105, "z_loss": 16.434162139892578, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1590932011604309, "tokens_per_second": 1355435.5475960257, "eta_seconds": 24201.944482111932, "data_wait_seconds": 1.6301870346069336e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20767, "cosmopedia_v2": 0.2004, "finemath": 0.08234}}
{"step": 9920, "total_steps": 72479, "tokens": 5200936960, "loss": 4.86279433965683, "l_mix": 3.0096960067749023, "l_a": 3.107371836900711, "l_c": 3.064248412847519, "z_loss": 16.121541619300842, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16554595530033112, "tokens_per_second": 1402846.7188004672, "eta_seconds": 23380.268530011177, "data_wait_seconds": 2.059340476989746e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.2004, "finemath": 0.08236}}
{"step": 9930, "total_steps": 72479, "tokens": 5206179840, "loss": 4.872149586677551, "l_mix": 3.0159886479377747, "l_a": 3.1145657002925873, "l_c": 3.067084938287735, "z_loss": 16.65595817565918, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19808392226696014, "tokens_per_second": 1473693.9403619324, "eta_seconds": 22252.71422636509, "data_wait_seconds": 3.68654727935791e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.2004, "finemath": 0.08238}}
{"step": 9940, "total_steps": 72479, "tokens": 5211422720, "loss": 3.2443727552890778, "l_mix": 2.0038503110408783, "l_a": 2.086040809750557, "l_c": 2.042965844273567, "z_loss": 18.20364499092102, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20649808645248413, "tokens_per_second": 1429958.1555048418, "eta_seconds": 22929.655043244362, "data_wait_seconds": 2.714991569519043e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.2004, "finemath": 0.08239}}
{"step": 9950, "total_steps": 72479, "tokens": 5216665600, "loss": 3.2376545667648315, "l_mix": 2.000032812356949, "l_a": 2.082819625735283, "l_c": 2.036344900727272, "z_loss": 18.723400592803955, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2825930416584015, "tokens_per_second": 1412293.6424478958, "eta_seconds": 23212.7394520998, "data_wait_seconds": 2.6047229766845703e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20764, "cosmopedia_v2": 0.2004, "finemath": 0.08241}}
{"step": 9960, "total_steps": 72479, "tokens": 5221908480, "loss": 4.725589096546173, "l_mix": 2.9239050149917603, "l_a": 3.026132643222809, "l_c": 2.9739701747894287, "z_loss": 16.5313663482666, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15824337303638458, "tokens_per_second": 1457133.7875549234, "eta_seconds": 22494.819454431534, "data_wait_seconds": 2.3931264877319336e-05, "mix": {"fineweb_hq": 0.50964, "dclm": 0.20763, "cosmopedia_v2": 0.2004, "finemath": 0.08233}}
{"step": 9970, "total_steps": 72479, "tokens": 5227151360, "loss": 4.829403817653656, "l_mix": 2.9876735508441925, "l_a": 3.09127277135849, "l_c": 3.0423605144023895, "z_loss": 16.402761220932007, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17790287733078003, "tokens_per_second": 1305388.028900356, "eta_seconds": 25105.72938194275, "data_wait_seconds": 1.710653305053711e-05, "mix": {"fineweb_hq": 0.50963, "dclm": 0.20762, "cosmopedia_v2": 0.2004, "finemath": 0.08235}}
{"step": 9980, "total_steps": 72479, "tokens": 5232394240, "loss": 4.831809163093567, "l_mix": 2.989160180091858, "l_a": 3.0937304496765137, "l_c": 3.043107271194458, "z_loss": 15.976054549217224, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17444980144500732, "tokens_per_second": 1415168.2262968426, "eta_seconds": 23154.473866152763, "data_wait_seconds": 2.3245811462402344e-05, "mix": {"fineweb_hq": 0.50962, "dclm": 0.20762, "cosmopedia_v2": 0.2004, "finemath": 0.08236}}
{"step": 9990, "total_steps": 72479, "tokens": 5237637120, "loss": 4.681468069553375, "l_mix": 2.8968929648399353, "l_a": 2.996360182762146, "l_c": 2.946813225746155, "z_loss": 16.22943341732025, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.23301321268081665, "tokens_per_second": 1440687.5443507924, "eta_seconds": 22740.692775797845, "data_wait_seconds": 2.047419548034668e-05, "mix": {"fineweb_hq": 0.50961, "dclm": 0.20761, "cosmopedia_v2": 0.2004, "finemath": 0.08238}}
{"step": 10000, "total_steps": 72479, "tokens": 5242880000, "loss": 4.835721015930176, "l_mix": 2.992414206266403, "l_a": 3.093875288963318, "l_c": 3.0450479090213776, "z_loss": 16.29648995399475, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15860134363174438, "tokens_per_second": 1144323.0911612567, "eta_seconds": 28625.647952938078, "data_wait_seconds": 1.52587890625e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.2077, "cosmopedia_v2": 0.2004, "finemath": 0.0823}, "gates": {"gate_a2b/L1_absmean": 0.19466357, "gate_a2b/L1_absmax": 0.60770559, "gate_a2b/L1_rms": 0.22927071, "gate_a2b/L3_absmean": 0.28872648, "gate_a2b/L3_absmax": 0.47512645, "gate_a2b/L3_rms": 0.29463255, "gate_a2b/L5_absmean": 0.47638664, "gate_a2b/L5_absmax": 0.65946579, "gate_a2b/L5_rms": 0.48432991, "gate_c2b/L1_absmean": 0.01551597, "gate_c2b/L1_absmax": 0.54528773, "gate_c2b/L1_rms": 0.06222838, "gate_c2b/L3_absmean": 0.21558487, "gate_c2b/L3_absmax": 0.437727, "gate_c2b/L3_rms": 0.22129495, "gate_c2b/L5_absmean": 0.54149646, "gate_c2b/L5_absmax": 0.77492905, "gate_c2b/L5_rms": 0.55542421, "gate_b2a/L7_absmean": 0.07275026, "gate_b2a/L7_absmax": 0.25165093, "gate_b2a/L7_rms": 0.07713518, "gate_b2a/L11_absmean": 0.1191587, "gate_b2a/L11_absmax": 0.31605795, "gate_b2a/L11_rms": 0.12129695, "gate_b2a/L14_absmean": 0.2910642, "gate_b2a/L14_absmax": 0.45877716, "gate_b2a/L14_rms": 0.29216701, "gate_b2c/L3_absmean": 0.05951207, "gate_b2c/L3_absmax": 0.49452069, "gate_b2c/L3_rms": 0.08093708, "gate_b2c/L5_absmean": 0.09990012, "gate_b2c/L5_absmax": 0.56195319, "gate_b2c/L5_rms": 0.10492527, "gate_b2c/L6_absmean": 0.15183906, "gate_b2c/L6_absmax": 0.41670802, "gate_b2c/L6_rms": 0.15420821}, "streams": {"cos_a2b/round0": 0.367922, "cos_c2b/round0": 0.252583, "cos_b2a/round0": 0.182821, "cos_b2c/round0": 0.119622, "rms_a/round0": 1.027556, "rms_b/round0": 1.922701, "rms_c/round0": 1.288887, "cos_a2b/round1": 0.39983, "cos_c2b/round1": 0.280272, "cos_b2a/round1": 0.216586, "cos_b2c/round1": 0.160944, "rms_a/round1": 1.85597, "rms_b/round1": 3.47326, "rms_c/round1": 2.106524, "cos_a2b/round2": 0.527907, "cos_c2b/round2": 0.443254, "cos_b2a/round2": 0.090287, "cos_b2c/round2": 0.152395, "rms_a/round2": 3.87093, "rms_b/round2": 7.281352, "rms_c/round2": 3.295436}, "alpha_mean": 0.4820589326839695, "alpha_hist": [0.00735, 0.04219, 0.10645, 0.17183, 0.1994, 0.20825, 0.15878, 0.08855, 0.01711, 9e-05], "alpha_by_type": {"special": {"share": 0.00078, "mean": 0.52534, "std": 0.11865}, "whitespace": {"share": 0.03781, "mean": 0.54929, "std": 0.11395}, "digit": {"share": 0.01756, "mean": 0.51259, "std": 0.12658}, "word_start": {"share": 0.56047, "mean": 0.50079, "std": 0.15686}, "word_cont": {"share": 0.26857, "mean": 0.37539, "std": 0.14241}, "punct": {"share": 0.11481, "mean": 0.61304, "std": 0.15435}}, "alpha_p05": 0.19315, "alpha_p50": 0.47774, "alpha_p95": 0.74428}
{"step": 10010, "total_steps": 72479, "tokens": 5248122880, "loss": 4.848163723945618, "l_mix": 2.9987698197364807, "l_a": 3.1041689813137054, "l_c": 3.0550116300582886, "z_loss": 16.39620614051819, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1682843118906021, "tokens_per_second": 1438335.3158813235, "eta_seconds": 22770.59230234623, "data_wait_seconds": 1.84476375579834e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20769, "cosmopedia_v2": 0.2004, "finemath": 0.08232}}
{"step": 10020, "total_steps": 72479, "tokens": 5253365760, "loss": 4.8789361119270325, "l_mix": 3.0196183621883392, "l_a": 3.121471732854843, "l_c": 3.0707820653915405, "z_loss": 16.41448187828064, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.14814352989196777, "tokens_per_second": 1424775.9341963925, "eta_seconds": 22983.616866374014, "data_wait_seconds": 2.8192996978759766e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20768, "cosmopedia_v2": 0.2004, "finemath": 0.08234}}
{"step": 10030, "total_steps": 72479, "tokens": 5258608640, "loss": 4.86396324634552, "l_mix": 3.009928345680237, "l_a": 3.1116922199726105, "l_c": 3.0630334615707397, "z_loss": 16.17111349105835, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1751784086227417, "tokens_per_second": 1407331.560258529, "eta_seconds": 23264.781545853617, "data_wait_seconds": 2.2023916244506836e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20768, "cosmopedia_v2": 0.2004, "finemath": 0.08235}}
{"step": 10040, "total_steps": 72479, "tokens": 5263851520, "loss": 4.882196128368378, "l_mix": 3.021204948425293, "l_a": 3.123280555009842, "l_c": 3.07450670003891, "z_loss": 16.550331354141235, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19804412126541138, "tokens_per_second": 1406068.9716865846, "eta_seconds": 23281.943554115296, "data_wait_seconds": 3.2782554626464844e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.2004, "finemath": 0.08237}}
{"step": 10050, "total_steps": 72479, "tokens": 5269094400, "loss": 3.2613510191440582, "l_mix": 2.0142419785261154, "l_a": 2.0985251367092133, "l_c": 2.0523149371147156, "z_loss": 18.56944179534912, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2718321681022644, "tokens_per_second": 1234983.0294414258, "eta_seconds": 26503.016455864905, "data_wait_seconds": 2.8878450393676758e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20766, "cosmopedia_v2": 0.2004, "finemath": 0.08239}}
{"step": 10060, "total_steps": 72479, "tokens": 5274337280, "loss": 3.1631155610084534, "l_mix": 1.953828439116478, "l_a": 2.031821444630623, "l_c": 1.9933393746614456, "z_loss": 17.389036417007446, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2425534874200821, "tokens_per_second": 1334532.8762866515, "eta_seconds": 24522.088030576706, "data_wait_seconds": 2.396106719970703e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20765, "cosmopedia_v2": 0.2004, "finemath": 0.08241}}
{"step": 10070, "total_steps": 72479, "tokens": 5279580160, "loss": 4.690224647521973, "l_mix": 2.901442766189575, "l_a": 3.0026692748069763, "l_c": 2.9544320702552795, "z_loss": 16.514610290527344, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.14991578459739685, "tokens_per_second": 1288514.9272799515, "eta_seconds": 25393.799558901785, "data_wait_seconds": 2.440810203552246e-05, "mix": {"fineweb_hq": 0.50963, "dclm": 0.20765, "cosmopedia_v2": 0.2004, "finemath": 0.08232}}
{"step": 10080, "total_steps": 72479, "tokens": 5284823040, "loss": 4.748198568820953, "l_mix": 2.937373459339142, "l_a": 3.0408410131931305, "l_c": 2.98976930975914, "z_loss": 16.41874611377716, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.13219180703163147, "tokens_per_second": 1406794.0689268066, "eta_seconds": 23255.036138272288, "data_wait_seconds": 2.1189451217651367e-05, "mix": {"fineweb_hq": 0.50962, "dclm": 0.20764, "cosmopedia_v2": 0.2004, "finemath": 0.08234}}
{"step": 10090, "total_steps": 72479, "tokens": 5290065920, "loss": 4.672246336936951, "l_mix": 2.890499860048294, "l_a": 2.99237596988678, "l_c": 2.9413716793060303, "z_loss": 16.221869111061096, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1926996111869812, "tokens_per_second": 1336786.8892149127, "eta_seconds": 24468.97429642677, "data_wait_seconds": 2.1696090698242188e-05, "mix": {"fineweb_hq": 0.50961, "dclm": 0.20763, "cosmopedia_v2": 0.2004, "finemath": 0.08236}}
{"step": 10100, "total_steps": 72479, "tokens": 5295308800, "loss": 4.608174383640289, "l_mix": 2.8487898409366608, "l_a": 2.9604138135910034, "l_c": 2.8986105024814606, "z_loss": 16.770894050598145, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19663597643375397, "tokens_per_second": 1170013.4811113724, "eta_seconds": 27952.29429402351, "data_wait_seconds": 0.06428545713424683, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20762, "cosmopedia_v2": 0.2004, "finemath": 0.08238}, "gates": {"gate_a2b/L1_absmean": 0.19625752, "gate_a2b/L1_absmax": 0.60784805, "gate_a2b/L1_rms": 0.23071717, "gate_a2b/L3_absmean": 0.29081932, "gate_a2b/L3_absmax": 0.47456399, "gate_a2b/L3_rms": 0.29666975, "gate_a2b/L5_absmean": 0.47869387, "gate_a2b/L5_absmax": 0.66481668, "gate_a2b/L5_rms": 0.48660135, "gate_c2b/L1_absmean": 0.01545241, "gate_c2b/L1_absmax": 0.54202825, "gate_c2b/L1_rms": 0.06197648, "gate_c2b/L3_absmean": 0.21513188, "gate_c2b/L3_absmax": 0.43722549, "gate_c2b/L3_rms": 0.22085467, "gate_c2b/L5_absmean": 0.54183197, "gate_c2b/L5_absmax": 0.77390391, "gate_c2b/L5_rms": 0.55567908, "gate_b2a/L7_absmean": 0.07316633, "gate_b2a/L7_absmax": 0.25116962, "gate_b2a/L7_rms": 0.0773899, "gate_b2a/L11_absmean": 0.11994403, "gate_b2a/L11_absmax": 0.31757388, "gate_b2a/L11_rms": 0.12199754, "gate_b2a/L14_absmean": 0.29241014, "gate_b2a/L14_absmax": 0.45424402, "gate_b2a/L14_rms": 0.29345241, "gate_b2c/L3_absmean": 0.06066428, "gate_b2c/L3_absmax": 0.49539915, "gate_b2c/L3_rms": 0.08212604, "gate_b2c/L5_absmean": 0.10082226, "gate_b2c/L5_absmax": 0.5606842, "gate_b2c/L5_rms": 0.10574126, "gate_b2c/L6_absmean": 0.15307404, "gate_b2c/L6_absmax": 0.41433579, "gate_b2c/L6_rms": 0.15541172}, "streams": {"cos_a2b/round0": 0.367244, "cos_c2b/round0": 0.246775, "cos_b2a/round0": 0.186503, "cos_b2c/round0": 0.110403, "rms_a/round0": 1.014472, "rms_b/round0": 1.913171, "rms_c/round0": 1.308582, "cos_a2b/round1": 0.393079, "cos_c2b/round1": 0.267994, "cos_b2a/round1": 0.216308, "cos_b2c/round1": 0.152598, "rms_a/round1": 1.837566, "rms_b/round1": 3.546388, "rms_c/round1": 2.124822, "cos_a2b/round2": 0.487296, "cos_c2b/round2": 0.471692, "cos_b2a/round2": 0.077601, "cos_b2c/round2": 0.133654, "rms_a/round2": 3.772175, "rms_b/round2": 7.587241, "rms_c/round2": 3.381964}, "alpha_mean": 0.4294258118857286, "alpha_hist": [0.01942, 0.08043, 0.16605, 0.19839, 0.19362, 0.146, 0.10779, 0.071, 0.017, 0.00031], "alpha_by_type": {"special": {"share": 0.00078, "mean": 0.59151, "std": 0.10641}, "whitespace": {"share": 0.02608, "mean": 0.54319, "std": 0.11652}, "digit": {"share": 0.0309, "mean": 0.48263, "std": 0.10781}, "word_start": {"share": 0.55922, "mean": 0.45242, "std": 0.17508}, "word_cont": {"share": 0.28131, "mean": 0.31643, "std": 0.1373}, "punct": {"share": 0.10172, "mean": 0.56895, "std": 0.17657}}, "alpha_p05": 0.15432, "alpha_p50": 0.40946, "alpha_p95": 0.7345}
{"step": 10110, "total_steps": 72479, "tokens": 5300551680, "loss": 4.495208144187927, "l_mix": 2.7766498625278473, "l_a": 2.893054723739624, "l_c": 2.8299481570720673, "z_loss": 16.57322907447815, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20106728374958038, "tokens_per_second": 1477415.3389510354, "eta_seconds": 22132.786502146722, "data_wait_seconds": 1.683831214904785e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20762, "cosmopedia_v2": 0.2004, "finemath": 0.08239}}
{"step": 10120, "total_steps": 72479, "tokens": 5305794560, "loss": 5.081219494342804, "l_mix": 3.1486796438694, "l_a": 3.2335396707057953, "l_c": 3.2027731239795685, "z_loss": 16.458314418792725, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.14830999076366425, "tokens_per_second": 1364639.0183539835, "eta_seconds": 23958.039417219163, "data_wait_seconds": 1.5854835510253906e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20771, "cosmopedia_v2": 0.2004, "finemath": 0.08231}}
{"step": 10130, "total_steps": 72479, "tokens": 5311037440, "loss": 5.089520335197449, "l_mix": 3.1528711020946503, "l_a": 3.2448132038116455, "l_c": 3.2052420377731323, "z_loss": 16.32702088356018, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1570185422897339, "tokens_per_second": 1422595.057360574, "eta_seconds": 22978.31160235405, "data_wait_seconds": 1.7702579498291016e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.2077, "cosmopedia_v2": 0.20039, "finemath": 0.08233}}
{"step": 10140, "total_steps": 72479, "tokens": 5316280320, "loss": 5.050024211406708, "l_mix": 3.1285365223884583, "l_a": 3.217612713575363, "l_c": 3.1818977892398834, "z_loss": 16.345008611679077, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1755952537059784, "tokens_per_second": 1459940.4966360976, "eta_seconds": 22386.932691645623, "data_wait_seconds": 2.4974346160888672e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20769, "cosmopedia_v2": 0.20039, "finemath": 0.08235}}
{"step": 10150, "total_steps": 72479, "tokens": 5321523200, "loss": 4.889310002326965, "l_mix": 3.02830770611763, "l_a": 3.1166954338550568, "l_c": 3.0812118351459503, "z_loss": 16.299824714660645, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1600756049156189, "tokens_per_second": 1463460.1072261021, "eta_seconds": 22329.509762954713, "data_wait_seconds": 1.8417835235595703e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.20039, "finemath": 0.08236}}
{"step": 10160, "total_steps": 72479, "tokens": 5326766080, "loss": 3.1309148371219635, "l_mix": 1.9332174509763718, "l_a": 2.0153622776269913, "l_c": 1.9710878431797028, "z_loss": 17.623192071914673, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.31580692529678345, "tokens_per_second": 1374806.702628829, "eta_seconds": 23765.59832704067, "data_wait_seconds": 5.8531761169433594e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20768, "cosmopedia_v2": 0.20039, "finemath": 0.08238}}
{"step": 10170, "total_steps": 72479, "tokens": 5332008960, "loss": 3.216943860054016, "l_mix": 1.9867502748966217, "l_a": 2.0795717984437943, "l_c": 2.015060156583786, "z_loss": 18.039427042007446, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.3407122492790222, "tokens_per_second": 1385516.7059805237, "eta_seconds": 23578.10689036846, "data_wait_seconds": 1.7076730728149414e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20767, "cosmopedia_v2": 0.20039, "finemath": 0.0824}}
{"step": 10180, "total_steps": 72479, "tokens": 5337251840, "loss": 4.3206411600112915, "l_mix": 2.668784648180008, "l_a": 2.7807253301143646, "l_c": 2.7199675738811493, "z_loss": 16.486018180847168, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1441335827112198, "tokens_per_second": 1355309.0700970888, "eta_seconds": 24099.756160902976, "data_wait_seconds": 2.917647361755371e-05, "mix": {"fineweb_hq": 0.50963, "dclm": 0.20766, "cosmopedia_v2": 0.20039, "finemath": 0.08232}}
{"step": 10190, "total_steps": 72479, "tokens": 5342494720, "loss": 4.319305002689362, "l_mix": 2.66743665933609, "l_a": 2.7801955342292786, "l_c": 2.7205228209495544, "z_loss": 16.528729915618896, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16247734427452087, "tokens_per_second": 1439653.9993197888, "eta_seconds": 22684.183315873146, "data_wait_seconds": 2.0056962966918945e-05, "mix": {"fineweb_hq": 0.50962, "dclm": 0.20765, "cosmopedia_v2": 0.20039, "finemath": 0.08234}}
{"step": 10200, "total_steps": 72479, "tokens": 5347737600, "loss": 4.36581951379776, "l_mix": 2.695454776287079, "l_a": 2.8133582174777985, "l_c": 2.748937666416168, "z_loss": 16.75837230682373, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17913225293159485, "tokens_per_second": 1357199.1652677706, "eta_seconds": 24058.46775300503, "data_wait_seconds": 1.8149614334106445e-05, "mix": {"fineweb_hq": 0.50961, "dclm": 0.20765, "cosmopedia_v2": 0.20039, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.19724546, "gate_a2b/L1_absmax": 0.60978043, "gate_a2b/L1_rms": 0.2315634, "gate_a2b/L3_absmean": 0.2918601, "gate_a2b/L3_absmax": 0.4744834, "gate_a2b/L3_rms": 0.29766122, "gate_a2b/L5_absmean": 0.47973663, "gate_a2b/L5_absmax": 0.66575062, "gate_a2b/L5_rms": 0.4875932, "gate_c2b/L1_absmean": 0.01569791, "gate_c2b/L1_absmax": 0.53844965, "gate_c2b/L1_rms": 0.06184279, "gate_c2b/L3_absmean": 0.21548013, "gate_c2b/L3_absmax": 0.4407503, "gate_c2b/L3_rms": 0.22113855, "gate_c2b/L5_absmean": 0.54180688, "gate_c2b/L5_absmax": 0.76348639, "gate_c2b/L5_rms": 0.55556417, "gate_b2a/L7_absmean": 0.07315861, "gate_b2a/L7_absmax": 0.24995372, "gate_b2a/L7_rms": 0.0774101, "gate_b2a/L11_absmean": 0.11995211, "gate_b2a/L11_absmax": 0.3196103, "gate_b2a/L11_rms": 0.1220484, "gate_b2a/L14_absmean": 0.29319769, "gate_b2a/L14_absmax": 0.45621401, "gate_b2a/L14_rms": 0.29424432, "gate_b2c/L3_absmean": 0.06098841, "gate_b2c/L3_absmax": 0.49712348, "gate_b2c/L3_rms": 0.08243474, "gate_b2c/L5_absmean": 0.10150491, "gate_b2c/L5_absmax": 0.5637064, "gate_b2c/L5_rms": 0.10649785, "gate_b2c/L6_absmean": 0.15298313, "gate_b2c/L6_absmax": 0.41597375, "gate_b2c/L6_rms": 0.15535031}, "streams": {"cos_a2b/round0": 0.369499, "cos_c2b/round0": 0.259776, "cos_b2a/round0": 0.180948, "cos_b2c/round0": 0.124793, "rms_a/round0": 1.005842, "rms_b/round0": 1.89362, "rms_c/round0": 1.296961, "cos_a2b/round1": 0.405156, "cos_c2b/round1": 0.268572, "cos_b2a/round1": 0.210901, "cos_b2c/round1": 0.155747, "rms_a/round1": 1.848053, "rms_b/round1": 3.496326, "rms_c/round1": 2.115992, "cos_a2b/round2": 0.504513, "cos_c2b/round2": 0.460814, "cos_b2a/round2": 0.06985, "cos_b2c/round2": 0.130243, "rms_a/round2": 3.814439, "rms_b/round2": 7.504517, "rms_c/round2": 3.392746}, "alpha_mean": 0.4632353534206146, "alpha_hist": [0.02354, 0.07806, 0.14021, 0.15965, 0.15428, 0.15614, 0.15202, 0.10301, 0.03217, 0.0009], "alpha_by_type": {"special": {"share": 0.00061, "mean": 0.64896, "std": 0.09074}, "whitespace": {"share": 0.02324, "mean": 0.58672, "std": 0.13618}, "digit": {"share": 0.03226, "mean": 0.55611, "std": 0.13008}, "word_start": {"share": 0.60143, "mean": 0.48446, "std": 0.18625}, "word_cont": {"share": 0.24364, "mean": 0.3207, "std": 0.15718}, "punct": {"share": 0.09883, "mean": 0.62494, "std": 0.17074}}, "alpha_p05": 0.15406, "alpha_p50": 0.45, "alpha_p95": 0.77865}
{"step": 10210, "total_steps": 72479, "tokens": 5352980480, "loss": 4.328363001346588, "l_mix": 2.6730003356933594, "l_a": 2.785477876663208, "l_c": 2.7268970012664795, "z_loss": 16.500093460083008, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17882336676120758, "tokens_per_second": 1474075.7499017965, "eta_seconds": 22147.362151622772, "data_wait_seconds": 2.1070241928100586e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20764, "cosmopedia_v2": 0.20039, "finemath": 0.08237}}
{"step": 10220, "total_steps": 72479, "tokens": 5358223360, "loss": 4.458747148513794, "l_mix": 2.7557351291179657, "l_a": 2.864748030900955, "l_c": 2.806664824485779, "z_loss": 15.880809426307678, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2124592363834381, "tokens_per_second": 1434198.8538739656, "eta_seconds": 22759.49845018387, "data_wait_seconds": 1.5556812286376953e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20763, "cosmopedia_v2": 0.20039, "finemath": 0.08239}}
{"step": 10230, "total_steps": 72479, "tokens": 5363466240, "loss": 3.586156874895096, "l_mix": 2.2037141919136047, "l_a": 2.332958161830902, "l_c": 2.2700317203998566, "z_loss": 15.457058668136597, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17636969685554504, "tokens_per_second": 1340717.41679714, "eta_seconds": 24342.4925365448, "data_wait_seconds": 2.872943878173828e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20762, "cosmopedia_v2": 0.20049, "finemath": 0.08231}}
{"step": 10240, "total_steps": 72479, "tokens": 5368709120, "loss": 5.071061432361603, "l_mix": 3.1412278711795807, "l_a": 3.2348829209804535, "l_c": 3.192438781261444, "z_loss": 16.370568990707397, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22472485899925232, "tokens_per_second": 1415123.2380096924, "eta_seconds": 23058.882757019997, "data_wait_seconds": 1.7434358596801758e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20771, "cosmopedia_v2": 0.20039, "finemath": 0.08232}}
{"step": 10250, "total_steps": 72479, "tokens": 5373952000, "loss": 5.100545287132263, "l_mix": 3.1605591475963593, "l_a": 3.2443806529045105, "l_c": 3.216831237077713, "z_loss": 16.225770711898804, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1981319934129715, "tokens_per_second": 1434135.2507316687, "eta_seconds": 22749.540488147733, "data_wait_seconds": 1.8030405044555664e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20771, "cosmopedia_v2": 0.20039, "finemath": 0.08234}}
{"step": 10260, "total_steps": 72479, "tokens": 5379194880, "loss": 4.999352812767029, "l_mix": 3.096286118030548, "l_a": 3.188013345003128, "l_c": 3.150004357099533, "z_loss": 16.612915754318237, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16434210538864136, "tokens_per_second": 1335401.730047782, "eta_seconds": 24427.611809992788, "data_wait_seconds": 2.0265579223632812e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.2077, "cosmopedia_v2": 0.20039, "finemath": 0.08236}}
{"step": 10270, "total_steps": 72479, "tokens": 5384437760, "loss": 5.130630731582642, "l_mix": 3.178844153881073, "l_a": 3.2690166532993317, "l_c": 3.231593579053879, "z_loss": 16.03442656993866, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1507532000541687, "tokens_per_second": 1228923.5445245078, "eta_seconds": 26539.838330316543, "data_wait_seconds": 1.940131187438965e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20769, "cosmopedia_v2": 0.20039, "finemath": 0.08238}}
{"step": 10280, "total_steps": 72479, "tokens": 5389680640, "loss": 3.563093423843384, "l_mix": 2.2025221586227417, "l_a": 2.285492330789566, "l_c": 2.243827849626541, "z_loss": 17.751662611961365, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.23338589072227478, "tokens_per_second": 1404143.9595355403, "eta_seconds": 23224.249259161952, "data_wait_seconds": 2.4586915969848633e-05, "mix": {"fineweb_hq": 0.50953, "dclm": 0.20768, "cosmopedia_v2": 0.20039, "finemath": 0.08239}}
{"step": 10290, "total_steps": 72479, "tokens": 5394923520, "loss": 4.290008544921875, "l_mix": 2.648580938577652, "l_a": 2.762238532304764, "l_c": 2.7037843763828278, "z_loss": 16.20737111568451, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1895838975906372, "tokens_per_second": 1427988.8276166872, "eta_seconds": 22832.77417962551, "data_wait_seconds": 2.7835369110107422e-05, "mix": {"fineweb_hq": 0.50962, "dclm": 0.20768, "cosmopedia_v2": 0.20039, "finemath": 0.08231}}
{"step": 10300, "total_steps": 72479, "tokens": 5400166400, "loss": 4.376902163028717, "l_mix": 2.7024158239364624, "l_a": 2.8154203593730927, "l_c": 2.760664016008377, "z_loss": 16.609468460083008, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.28955012559890747, "tokens_per_second": 1408331.0163802728, "eta_seconds": 23147.75658054352, "data_wait_seconds": 1.8358230590820312e-05, "mix": {"fineweb_hq": 0.50961, "dclm": 0.20767, "cosmopedia_v2": 0.20039, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.19972205, "gate_a2b/L1_absmax": 0.61439681, "gate_a2b/L1_rms": 0.23368166, "gate_a2b/L3_absmean": 0.29277188, "gate_a2b/L3_absmax": 0.47387835, "gate_a2b/L3_rms": 0.29852489, "gate_a2b/L5_absmean": 0.48084792, "gate_a2b/L5_absmax": 0.66676259, "gate_a2b/L5_rms": 0.48864913, "gate_c2b/L1_absmean": 0.01544112, "gate_c2b/L1_absmax": 0.53595585, "gate_c2b/L1_rms": 0.06169663, "gate_c2b/L3_absmean": 0.21546124, "gate_c2b/L3_absmax": 0.43924633, "gate_c2b/L3_rms": 0.22107245, "gate_c2b/L5_absmean": 0.54161972, "gate_c2b/L5_absmax": 0.7743963, "gate_c2b/L5_rms": 0.55537832, "gate_b2a/L7_absmean": 0.07333413, "gate_b2a/L7_absmax": 0.24929379, "gate_b2a/L7_rms": 0.07748562, "gate_b2a/L11_absmean": 0.11991564, "gate_b2a/L11_absmax": 0.3184053, "gate_b2a/L11_rms": 0.12198979, "gate_b2a/L14_absmean": 0.29287872, "gate_b2a/L14_absmax": 0.45603049, "gate_b2a/L14_rms": 0.29394731, "gate_b2c/L3_absmean": 0.06183633, "gate_b2c/L3_absmax": 0.4977577, "gate_b2c/L3_rms": 0.08335242, "gate_b2c/L5_absmean": 0.10253354, "gate_b2c/L5_absmax": 0.56947291, "gate_b2c/L5_rms": 0.10754224, "gate_b2c/L6_absmean": 0.15394822, "gate_b2c/L6_absmax": 0.41719514, "gate_b2c/L6_rms": 0.15628089}, "streams": {"cos_a2b/round0": 0.366301, "cos_c2b/round0": 0.250979, "cos_b2a/round0": 0.176688, "cos_b2c/round0": 0.109254, "rms_a/round0": 1.002998, "rms_b/round0": 1.885361, "rms_c/round0": 1.300466, "cos_a2b/round1": 0.39212, "cos_c2b/round1": 0.264146, "cos_b2a/round1": 0.21013, "cos_b2c/round1": 0.152227, "rms_a/round1": 1.832078, "rms_b/round1": 3.528078, "rms_c/round1": 2.13877, "cos_a2b/round2": 0.489415, "cos_c2b/round2": 0.465922, "cos_b2a/round2": 0.053479, "cos_b2c/round2": 0.126289, "rms_a/round2": 3.743204, "rms_b/round2": 7.52036, "rms_c/round2": 3.416798}, "alpha_mean": 0.40476156971135424, "alpha_hist": [0.04393, 0.13045, 0.17993, 0.16534, 0.14647, 0.13379, 0.11191, 0.07048, 0.01715, 0.00055], "alpha_by_type": {"special": {"share": 0.00117, "mean": 0.60688, "std": 0.09332}, "whitespace": {"share": 0.02306, "mean": 0.55386, "std": 0.13243}, "digit": {"share": 0.01959, "mean": 0.49007, "std": 0.15315}, "word_start": {"share": 0.59747, "mean": 0.42379, "std": 0.1889}, "word_cont": {"share": 0.26019, "mean": 0.27672, "std": 0.15504}, "punct": {"share": 0.09851, "mean": 0.57331, "std": 0.18402}}, "alpha_p05": 0.11241, "alpha_p50": 0.37342, "alpha_p95": 0.73867}
{"step": 10310, "total_steps": 72479, "tokens": 5405409280, "loss": 4.177485942840576, "l_mix": 2.578813076019287, "l_a": 2.6948592960834503, "l_c": 2.628533899784088, "z_loss": 16.54810404777527, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20891469717025757, "tokens_per_second": 1465214.5197638625, "eta_seconds": 22245.521206855774, "data_wait_seconds": 2.1755695343017578e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20766, "cosmopedia_v2": 0.20039, "finemath": 0.08235}}
{"step": 10320, "total_steps": 72479, "tokens": 5410652160, "loss": 4.219610750675201, "l_mix": 2.605171024799347, "l_a": 2.7197964191436768, "l_c": 2.656096935272217, "z_loss": 16.716432809829712, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18254303932189941, "tokens_per_second": 1402717.3233961526, "eta_seconds": 23232.9188842535, "data_wait_seconds": 2.2858381271362305e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20766, "cosmopedia_v2": 0.20039, "finemath": 0.08236}}
{"step": 10330, "total_steps": 72479, "tokens": 5415895040, "loss": 4.251522034406662, "l_mix": 2.625338315963745, "l_a": 2.736260175704956, "l_c": 2.6788623929023743, "z_loss": 16.469602823257446, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18705302476882935, "tokens_per_second": 1418266.077802455, "eta_seconds": 22974.51474161148, "data_wait_seconds": 2.0563602447509766e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20039, "finemath": 0.08238}}
{"step": 10340, "total_steps": 72479, "tokens": 5421137920, "loss": 3.540964186191559, "l_mix": 2.17630398273468, "l_a": 2.3049697279930115, "l_c": 2.238739162683487, "z_loss": 15.475231528282166, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17160490155220032, "tokens_per_second": 1423867.5067106897, "eta_seconds": 22880.45192298889, "data_wait_seconds": 1.9788742065429688e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20764, "cosmopedia_v2": 0.20048, "finemath": 0.0823}}
{"step": 10350, "total_steps": 72479, "tokens": 5426380800, "loss": 3.562313884496689, "l_mix": 2.1898527443408966, "l_a": 2.314123511314392, "l_c": 2.2555564641952515, "z_loss": 15.57132887840271, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18917858600616455, "tokens_per_second": 1336342.9326905343, "eta_seconds": 24375.09740588665, "data_wait_seconds": 2.3603439331054688e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20763, "cosmopedia_v2": 0.20048, "finemath": 0.08232}}
{"step": 10360, "total_steps": 72479, "tokens": 5431623680, "loss": 3.55255663394928, "l_mix": 2.1832932829856873, "l_a": 2.3087081611156464, "l_c": 2.2503256797790527, "z_loss": 15.531495571136475, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18615563213825226, "tokens_per_second": 1425603.4475718646, "eta_seconds": 22845.235347509384, "data_wait_seconds": 2.9772520065307617e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20763, "cosmopedia_v2": 0.20048, "finemath": 0.08234}}
{"step": 10370, "total_steps": 72479, "tokens": 5436866560, "loss": 3.577110677957535, "l_mix": 2.198871612548828, "l_a": 2.322298586368561, "l_c": 2.266582638025284, "z_loss": 15.746219873428345, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20139823853969574, "tokens_per_second": 1407677.9509854154, "eta_seconds": 23132.424123859404, "data_wait_seconds": 1.5974044799804688e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20762, "cosmopedia_v2": 0.20048, "finemath": 0.08235}}
{"step": 10380, "total_steps": 72479, "tokens": 5442109440, "loss": 5.052264630794525, "l_mix": 3.12909072637558, "l_a": 3.221003383398056, "l_c": 3.1840912997722626, "z_loss": 16.454646348953247, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1605655997991562, "tokens_per_second": 1436834.2619418516, "eta_seconds": 22659.37093398571, "data_wait_seconds": 2.5659799575805664e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20771, "cosmopedia_v2": 0.20039, "finemath": 0.08237}}
{"step": 10390, "total_steps": 72479, "tokens": 5447352320, "loss": 5.1209716796875, "l_mix": 3.172500640153885, "l_a": 3.2653802037239075, "l_c": 3.2242104709148407, "z_loss": 15.938224077224731, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19346469640731812, "tokens_per_second": 1422821.8571802182, "eta_seconds": 22878.842820501326, "data_wait_seconds": 1.800060272216797e-05, "mix": {"fineweb_hq": 0.50953, "dclm": 0.2077, "cosmopedia_v2": 0.20038, "finemath": 0.08239}}
{"step": 10400, "total_steps": 72479, "tokens": 5452595200, "loss": 4.191271245479584, "l_mix": 2.587654232978821, "l_a": 2.7012591660022736, "l_c": 2.6385541558265686, "z_loss": 16.730125427246094, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20448195934295654, "tokens_per_second": 1332629.1838535531, "eta_seconds": 24423.35433318615, "data_wait_seconds": 2.7358531951904297e-05, "mix": {"fineweb_hq": 0.50962, "dclm": 0.20769, "cosmopedia_v2": 0.20038, "finemath": 0.08231}, "gates": {"gate_a2b/L1_absmean": 0.20154913, "gate_a2b/L1_absmax": 0.62127888, "gate_a2b/L1_rms": 0.2354168, "gate_a2b/L3_absmean": 0.29348245, "gate_a2b/L3_absmax": 0.47454032, "gate_a2b/L3_rms": 0.29918629, "gate_a2b/L5_absmean": 0.48197341, "gate_a2b/L5_absmax": 0.66881102, "gate_a2b/L5_rms": 0.48975059, "gate_c2b/L1_absmean": 0.01572531, "gate_c2b/L1_absmax": 0.53488034, "gate_c2b/L1_rms": 0.06150161, "gate_c2b/L3_absmean": 0.21504585, "gate_c2b/L3_absmax": 0.43217412, "gate_c2b/L3_rms": 0.22066301, "gate_c2b/L5_absmean": 0.54123443, "gate_c2b/L5_absmax": 0.7707234, "gate_c2b/L5_rms": 0.55484521, "gate_b2a/L7_absmean": 0.07330172, "gate_b2a/L7_absmax": 0.24804962, "gate_b2a/L7_rms": 0.07749089, "gate_b2a/L11_absmean": 0.12019309, "gate_b2a/L11_absmax": 0.31950015, "gate_b2a/L11_rms": 0.12231096, "gate_b2a/L14_absmean": 0.29388493, "gate_b2a/L14_absmax": 0.45762876, "gate_b2a/L14_rms": 0.29493812, "gate_b2c/L3_absmean": 0.06240933, "gate_b2c/L3_absmax": 0.50101191, "gate_b2c/L3_rms": 0.08380869, "gate_b2c/L5_absmean": 0.10241725, "gate_b2c/L5_absmax": 0.57295626, "gate_b2c/L5_rms": 0.10746258, "gate_b2c/L6_absmean": 0.15423945, "gate_b2c/L6_absmax": 0.4209595, "gate_b2c/L6_rms": 0.15665115}, "streams": {"cos_a2b/round0": 0.370759, "cos_c2b/round0": 0.248728, "cos_b2a/round0": 0.184712, "cos_b2c/round0": 0.107208, "rms_a/round0": 1.001012, "rms_b/round0": 1.894787, "rms_c/round0": 1.311015, "cos_a2b/round1": 0.39936, "cos_c2b/round1": 0.273085, "cos_b2a/round1": 0.220055, "cos_b2c/round1": 0.151009, "rms_a/round1": 1.814143, "rms_b/round1": 3.51743, "rms_c/round1": 2.140877, "cos_a2b/round2": 0.488655, "cos_c2b/round2": 0.479098, "cos_b2a/round2": 0.059216, "cos_b2c/round2": 0.131555, "rms_a/round2": 3.742936, "rms_b/round2": 7.586242, "rms_c/round2": 3.450476}, "alpha_mean": 0.42560295206075693, "alpha_hist": [0.021, 0.10457, 0.17638, 0.18367, 0.16144, 0.13919, 0.11418, 0.07805, 0.0211, 0.00043], "alpha_by_type": {"special": {"share": 0.00092, "mean": 0.56233, "std": 0.09841}, "whitespace": {"share": 0.02171, "mean": 0.54273, "std": 0.11591}, "digit": {"share": 0.01689, "mean": 0.44334, "std": 0.13568}, "word_start": {"share": 0.60052, "mean": 0.44234, "std": 0.18468}, "word_cont": {"share": 0.26122, "mean": 0.31123, "std": 0.14823}, "punct": {"share": 0.09874, "mean": 0.5963, "std": 0.17416}}, "alpha_p05": 0.13823, "alpha_p50": 0.41442, "alpha_p95": 0.74242}
{"step": 10410, "total_steps": 72479, "tokens": 5457838080, "loss": 4.25520384311676, "l_mix": 2.627720206975937, "l_a": 2.738825708627701, "l_c": 2.6806159019470215, "z_loss": 16.51077699661255, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1655730903148651, "tokens_per_second": 1380796.0833477126, "eta_seconds": 23567.58703508377, "data_wait_seconds": 1.996755599975586e-05, "mix": {"fineweb_hq": 0.50961, "dclm": 0.20768, "cosmopedia_v2": 0.20038, "finemath": 0.08232}}
{"step": 10420, "total_steps": 72479, "tokens": 5463080960, "loss": 4.18749612569809, "l_mix": 2.585328549146652, "l_a": 2.7014006972312927, "l_c": 2.6337053775787354, "z_loss": 16.357425212860107, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1567249298095703, "tokens_per_second": 1400146.3647179073, "eta_seconds": 23238.134106469155, "data_wait_seconds": 1.9073486328125e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20768, "cosmopedia_v2": 0.20038, "finemath": 0.08234}}
{"step": 10430, "total_steps": 72479, "tokens": 5468323840, "loss": 4.3649343848228455, "l_mix": 2.696485698223114, "l_a": 2.806912064552307, "l_c": 2.7489953339099884, "z_loss": 16.764434576034546, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16746079921722412, "tokens_per_second": 1406264.092421845, "eta_seconds": 23133.31207652092, "data_wait_seconds": 1.6748905181884766e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20767, "cosmopedia_v2": 0.20038, "finemath": 0.08236}}
{"step": 10440, "total_steps": 72479, "tokens": 5473566720, "loss": 4.186347544193268, "l_mix": 2.583893656730652, "l_a": 2.7005912959575653, "l_c": 2.635344445705414, "z_loss": 16.730605363845825, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16637884080410004, "tokens_per_second": 1440934.8793778582, "eta_seconds": 22573.055658173562, "data_wait_seconds": 1.7464160919189453e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20038, "finemath": 0.08238}}
{"step": 10450, "total_steps": 72479, "tokens": 5478809600, "loss": 3.4571729004383087, "l_mix": 2.1241662800312042, "l_a": 2.2500394880771637, "l_c": 2.1880396008491516, "z_loss": 15.828888058662415, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18768201768398285, "tokens_per_second": 1414936.394199813, "eta_seconds": 22984.114681982996, "data_wait_seconds": 2.3365020751953125e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20048, "finemath": 0.0823}}
{"step": 10460, "total_steps": 72479, "tokens": 5484052480, "loss": 3.5331974029541016, "l_mix": 2.172035336494446, "l_a": 2.29293754696846, "l_c": 2.239110589027405, "z_loss": 15.476157546043396, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18266522884368896, "tokens_per_second": 1379227.001022712, "eta_seconds": 23575.392192792893, "data_wait_seconds": 2.7924776077270508e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20048, "finemath": 0.08231}}
{"step": 10470, "total_steps": 72479, "tokens": 5489295360, "loss": 3.5730443000793457, "l_mix": 2.196725010871887, "l_a": 2.323356479406357, "l_c": 2.25924614071846, "z_loss": 15.38414990901947, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22948037087917328, "tokens_per_second": 1311851.522857384, "eta_seconds": 24782.20593225956, "data_wait_seconds": 2.1129846572875977e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20764, "cosmopedia_v2": 0.20048, "finemath": 0.08233}}
{"step": 10480, "total_steps": 72479, "tokens": 5494538240, "loss": 3.5802049338817596, "l_mix": 2.2002192437648773, "l_a": 2.329398900270462, "l_c": 2.2654367089271545, "z_loss": 15.349230170249939, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.166845440864563, "tokens_per_second": 1416903.0008077356, "eta_seconds": 22941.11290149689, "data_wait_seconds": 3.88026237487793e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20763, "cosmopedia_v2": 0.20048, "finemath": 0.08235}}
{"step": 10490, "total_steps": 72479, "tokens": 5499781120, "loss": 3.4363140165805817, "l_mix": 2.111116945743561, "l_a": 2.2375950813293457, "l_c": 2.1745969653129578, "z_loss": 15.393774390220642, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1966218501329422, "tokens_per_second": 1338361.3490880555, "eta_seconds": 24283.493283891676, "data_wait_seconds": 3.8057565689086914e-05, "mix": {"fineweb_hq": 0.50953, "dclm": 0.20763, "cosmopedia_v2": 0.20048, "finemath": 0.08236}}
{"step": 10500, "total_steps": 72479, "tokens": 5505024000, "loss": 3.557606667280197, "l_mix": 2.1859039664268494, "l_a": 2.3153832256793976, "l_c": 2.2517582178115845, "z_loss": 15.602052330970764, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.222726508975029, "tokens_per_second": 1400731.6057520476, "eta_seconds": 23198.4812926054, "data_wait_seconds": 2.1845102310180664e-05, "mix": {"fineweb_hq": 0.50952, "dclm": 0.20762, "cosmopedia_v2": 0.20048, "finemath": 0.08238}, "gates": {"gate_a2b/L1_absmean": 0.2033658, "gate_a2b/L1_absmax": 0.61384535, "gate_a2b/L1_rms": 0.23695737, "gate_a2b/L3_absmean": 0.29548177, "gate_a2b/L3_absmax": 0.47678223, "gate_a2b/L3_rms": 0.30114302, "gate_a2b/L5_absmean": 0.48298565, "gate_a2b/L5_absmax": 0.67272675, "gate_a2b/L5_rms": 0.490747, "gate_c2b/L1_absmean": 0.01571191, "gate_c2b/L1_absmax": 0.53244406, "gate_c2b/L1_rms": 0.0612934, "gate_c2b/L3_absmean": 0.21455351, "gate_c2b/L3_absmax": 0.42998078, "gate_c2b/L3_rms": 0.22013152, "gate_c2b/L5_absmean": 0.54113436, "gate_c2b/L5_absmax": 0.7674132, "gate_c2b/L5_rms": 0.55459845, "gate_b2a/L7_absmean": 0.07363411, "gate_b2a/L7_absmax": 0.24759625, "gate_b2a/L7_rms": 0.07770353, "gate_b2a/L11_absmean": 0.12034935, "gate_b2a/L11_absmax": 0.32302299, "gate_b2a/L11_rms": 0.12245995, "gate_b2a/L14_absmean": 0.29406762, "gate_b2a/L14_absmax": 0.45943141, "gate_b2a/L14_rms": 0.2951327, "gate_b2c/L3_absmean": 0.06314828, "gate_b2c/L3_absmax": 0.50507981, "gate_b2c/L3_rms": 0.08470564, "gate_b2c/L5_absmean": 0.1038716, "gate_b2c/L5_absmax": 0.57614243, "gate_b2c/L5_rms": 0.10891367, "gate_b2c/L6_absmean": 0.15500848, "gate_b2c/L6_absmax": 0.41795507, "gate_b2c/L6_rms": 0.15741931}, "streams": {"cos_a2b/round0": 0.356304, "cos_c2b/round0": 0.218745, "cos_b2a/round0": 0.172199, "cos_b2c/round0": 0.078171, "rms_a/round0": 1.018958, "rms_b/round0": 1.858644, "rms_c/round0": 1.312282, "cos_a2b/round1": 0.383468, "cos_c2b/round1": 0.265182, "cos_b2a/round1": 0.211589, "cos_b2c/round1": 0.149015, "rms_a/round1": 1.810197, "rms_b/round1": 3.532912, "rms_c/round1": 2.194675, "cos_a2b/round2": 0.481606, "cos_c2b/round2": 0.45231, "cos_b2a/round2": 0.034155, "cos_b2c/round2": 0.11407, "rms_a/round2": 3.72638, "rms_b/round2": 7.496449, "rms_c/round2": 3.494222}, "alpha_mean": 0.4389104497379108, "alpha_hist": [0.05855, 0.13005, 0.15323, 0.13776, 0.12086, 0.10742, 0.10851, 0.10751, 0.07014, 0.00597], "alpha_by_type": {"special": {"share": 0.00108, "mean": 0.58907, "std": 0.09837}, "whitespace": {"share": 0.03468, "mean": 0.61458, "std": 0.18945}, "digit": {"share": 0.01158, "mean": 0.42234, "std": 0.16791}, "word_start": {"share": 0.55884, "mean": 0.47092, "std": 0.22084}, "word_cont": {"share": 0.26532, "mean": 0.26344, "std": 0.16614}, "punct": {"share": 0.12849, "mean": 0.6148, "std": 0.204}}, "alpha_p05": 0.09671, "alpha_p50": 0.42298, "alpha_p95": 0.83894}
{"step": 10510, "total_steps": 72479, "tokens": 5510266880, "loss": 4.234561502933502, "l_mix": 2.6146185398101807, "l_a": 2.7282914221286774, "l_c": 2.6661433577537537, "z_loss": 16.124815940856934, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.23224423825740814, "tokens_per_second": 1434765.6385367059, "eta_seconds": 22644.536640238763, "data_wait_seconds": 2.1338462829589844e-05, "mix": {"fineweb_hq": 0.50961, "dclm": 0.20761, "cosmopedia_v2": 0.20048, "finemath": 0.0823}}
{"step": 10520, "total_steps": 72479, "tokens": 5515509760, "loss": 4.318361222743988, "l_mix": 2.6670331358909607, "l_a": 2.7803668677806854, "l_c": 2.718497008085251, "z_loss": 16.689331531524658, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1963672935962677, "tokens_per_second": 1375347.2921905993, "eta_seconds": 23619.023628759383, "data_wait_seconds": 1.8149614334106445e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.2077, "cosmopedia_v2": 0.20038, "finemath": 0.08232}}
{"step": 10530, "total_steps": 72479, "tokens": 5520752640, "loss": 4.336924254894257, "l_mix": 2.678831100463867, "l_a": 2.792354315519333, "l_c": 2.729112982749939, "z_loss": 16.528189420700073, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17306484282016754, "tokens_per_second": 1378694.335445637, "eta_seconds": 23557.881161165238, "data_wait_seconds": 2.9295682907104492e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20769, "cosmopedia_v2": 0.20038, "finemath": 0.08234}}
{"step": 10540, "total_steps": 72479, "tokens": 5525995520, "loss": 4.2639360427856445, "l_mix": 2.6326880156993866, "l_a": 2.750423401594162, "l_c": 2.681515097618103, "z_loss": 16.66481375694275, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15578241646289825, "tokens_per_second": 1375837.3455566962, "eta_seconds": 23602.98950808048, "data_wait_seconds": 1.6391277313232422e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20769, "cosmopedia_v2": 0.20038, "finemath": 0.08235}}
{"step": 10550, "total_steps": 72479, "tokens": 5531238400, "loss": 4.165680110454559, "l_mix": 2.5704557299613953, "l_a": 2.687776744365692, "l_c": 2.624138832092285, "z_loss": 16.496530055999756, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17722424864768982, "tokens_per_second": 1426398.42846934, "eta_seconds": 22762.666379857066, "data_wait_seconds": 2.5451183319091797e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20768, "cosmopedia_v2": 0.20038, "finemath": 0.08237}}
{"step": 10560, "total_steps": 72479, "tokens": 5536481280, "loss": 3.5788485407829285, "l_mix": 2.1993124783039093, "l_a": 2.3297257125377655, "l_c": 2.26355242729187, "z_loss": 15.526016473770142, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19837641716003418, "tokens_per_second": 1418749.5777680743, "eta_seconds": 22881.69045524597, "data_wait_seconds": 2.530217170715332e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20047, "finemath": 0.08229}}
{"step": 10570, "total_steps": 72479, "tokens": 5541724160, "loss": 3.596953183412552, "l_mix": 2.2103119492530823, "l_a": 2.3376431465148926, "l_c": 2.2792663276195526, "z_loss": 15.68326711654663, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19814667105674744, "tokens_per_second": 1485396.86294255, "eta_seconds": 21851.49746963978, "data_wait_seconds": 2.09808349609375e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20047, "finemath": 0.08231}}
{"step": 10580, "total_steps": 72479, "tokens": 5546967040, "loss": 3.5362824201583862, "l_mix": 2.172168105840683, "l_a": 2.3012754321098328, "l_c": 2.2406934797763824, "z_loss": 15.235753059387207, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2097076177597046, "tokens_per_second": 1395970.1868307635, "eta_seconds": 23247.561601352692, "data_wait_seconds": 2.658367156982422e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20766, "cosmopedia_v2": 0.20047, "finemath": 0.08233}}
{"step": 10590, "total_steps": 72479, "tokens": 5552209920, "loss": 3.5772338211536407, "l_mix": 2.1986174285411835, "l_a": 2.325479209423065, "l_c": 2.2647595405578613, "z_loss": 15.447496056556702, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20172667503356934, "tokens_per_second": 1419906.8679775, "eta_seconds": 22851.96357858181, "data_wait_seconds": 3.5822391510009766e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20765, "cosmopedia_v2": 0.20047, "finemath": 0.08234}}
{"step": 10600, "total_steps": 72479, "tokens": 5557452800, "loss": 3.5541929602622986, "l_mix": 2.183957576751709, "l_a": 2.3151362240314484, "l_c": 2.246971011161804, "z_loss": 16.031329035758972, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2080780267715454, "tokens_per_second": 1353779.344151018, "eta_seconds": 23964.331626248357, "data_wait_seconds": 2.8312206268310547e-05, "mix": {"fineweb_hq": 0.50953, "dclm": 0.20764, "cosmopedia_v2": 0.20047, "finemath": 0.08236}, "gates": {"gate_a2b/L1_absmean": 0.20492892, "gate_a2b/L1_absmax": 0.61319739, "gate_a2b/L1_rms": 0.23790205, "gate_a2b/L3_absmean": 0.29581633, "gate_a2b/L3_absmax": 0.47649699, "gate_a2b/L3_rms": 0.30148995, "gate_a2b/L5_absmean": 0.48356381, "gate_a2b/L5_absmax": 0.67138612, "gate_a2b/L5_rms": 0.49124041, "gate_c2b/L1_absmean": 0.01606961, "gate_c2b/L1_absmax": 0.52928931, "gate_c2b/L1_rms": 0.06107041, "gate_c2b/L3_absmean": 0.21358876, "gate_c2b/L3_absmax": 0.43907022, "gate_c2b/L3_rms": 0.21907969, "gate_c2b/L5_absmean": 0.54078352, "gate_c2b/L5_absmax": 0.75629395, "gate_c2b/L5_rms": 0.55427551, "gate_b2a/L7_absmean": 0.07360119, "gate_b2a/L7_absmax": 0.24753177, "gate_b2a/L7_rms": 0.07767453, "gate_b2a/L11_absmean": 0.12034293, "gate_b2a/L11_absmax": 0.32327244, "gate_b2a/L11_rms": 0.12247407, "gate_b2a/L14_absmean": 0.29408532, "gate_b2a/L14_absmax": 0.45844257, "gate_b2a/L14_rms": 0.29513383, "gate_b2c/L3_absmean": 0.06369425, "gate_b2c/L3_absmax": 0.50340283, "gate_b2c/L3_rms": 0.0850088, "gate_b2c/L5_absmean": 0.10395871, "gate_b2c/L5_absmax": 0.57753778, "gate_b2c/L5_rms": 0.10907599, "gate_b2c/L6_absmean": 0.15526247, "gate_b2c/L6_absmax": 0.42010772, "gate_b2c/L6_rms": 0.15768056}, "streams": {"cos_a2b/round0": 0.357514, "cos_c2b/round0": 0.219869, "cos_b2a/round0": 0.171468, "cos_b2c/round0": 0.082431, "rms_a/round0": 1.015741, "rms_b/round0": 1.860971, "rms_c/round0": 1.306215, "cos_a2b/round1": 0.388678, "cos_c2b/round1": 0.266924, "cos_b2a/round1": 0.205313, "cos_b2c/round1": 0.156464, "rms_a/round1": 1.817621, "rms_b/round1": 3.486222, "rms_c/round1": 2.176139, "cos_a2b/round2": 0.493023, "cos_c2b/round2": 0.442946, "cos_b2a/round2": 0.036021, "cos_b2c/round2": 0.11978, "rms_a/round2": 3.725638, "rms_b/round2": 7.369211, "rms_c/round2": 3.449309}, "alpha_mean": 0.4500805006888555, "alpha_hist": [0.04349, 0.11296, 0.15121, 0.14499, 0.13412, 0.11798, 0.11624, 0.11469, 0.05879, 0.00552], "alpha_by_type": {"special": {"share": 0.00114, "mean": 0.69074, "std": 0.09672}, "whitespace": {"share": 0.03368, "mean": 0.65777, "std": 0.17072}, "digit": {"share": 0.01509, "mean": 0.46603, "std": 0.14699}, "word_start": {"share": 0.56874, "mean": 0.46992, "std": 0.20986}, "word_cont": {"share": 0.25455, "mean": 0.28682, "std": 0.16816}, "punct": {"share": 0.1268, "mean": 0.62961, "std": 0.18719}}, "alpha_p05": 0.10195, "alpha_p50": 0.43254, "alpha_p95": 0.81168}
{"step": 10610, "total_steps": 72479, "tokens": 5562695680, "loss": 3.5328406989574432, "l_mix": 2.171137422323227, "l_a": 2.296885371208191, "l_c": 2.237001419067383, "z_loss": 15.371122598648071, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19818320870399475, "tokens_per_second": 1336897.5790141278, "eta_seconds": 24263.021177673338, "data_wait_seconds": 2.828240394592285e-05, "mix": {"fineweb_hq": 0.50952, "dclm": 0.20763, "cosmopedia_v2": 0.20047, "finemath": 0.08238}}
{"step": 10620, "total_steps": 72479, "tokens": 5567938560, "loss": 4.225028038024902, "l_mix": 2.6086868345737457, "l_a": 2.721144735813141, "l_c": 2.660926789045334, "z_loss": 17.19583296775818, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2620871067047119, "tokens_per_second": 1452759.7489467438, "eta_seconds": 22324.35983686447, "data_wait_seconds": 1.677870750427246e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20763, "cosmopedia_v2": 0.20038, "finemath": 0.08239}}
{"step": 10630, "total_steps": 72479, "tokens": 5573181440, "loss": 4.228166967630386, "l_mix": 2.6113967895507812, "l_a": 2.7212648391723633, "l_c": 2.6622494757175446, "z_loss": 17.158766269683838, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16507671773433685, "tokens_per_second": 1287742.6441345904, "eta_seconds": 25181.03183093071, "data_wait_seconds": 2.8580427169799805e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20762, "cosmopedia_v2": 0.20047, "finemath": 0.08231}}
{"step": 10640, "total_steps": 72479, "tokens": 5578424320, "loss": 4.198788613080978, "l_mix": 2.5925761461257935, "l_a": 2.7072715759277344, "l_c": 2.6411856412887573, "z_loss": 16.752634048461914, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19152237474918365, "tokens_per_second": 1436597.5298962975, "eta_seconds": 22568.21758167744, "data_wait_seconds": 1.7434358596801758e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20761, "cosmopedia_v2": 0.20047, "finemath": 0.08233}}
{"step": 10650, "total_steps": 72479, "tokens": 5583667200, "loss": 3.5802301168441772, "l_mix": 2.198932021856308, "l_a": 2.3302015364170074, "l_c": 2.269039750099182, "z_loss": 15.256621956825256, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2235497087240219, "tokens_per_second": 1430080.4423460616, "eta_seconds": 22667.39813518524, "data_wait_seconds": 3.904104232788086e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20761, "cosmopedia_v2": 0.20047, "finemath": 0.08235}}
{"step": 10660, "total_steps": 72479, "tokens": 5588910080, "loss": 4.991847217082977, "l_mix": 3.092228442430496, "l_a": 3.1804737746715546, "l_c": 3.1461624801158905, "z_loss": 16.278563499450684, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2954602837562561, "tokens_per_second": 1370326.0896373224, "eta_seconds": 23652.005254149437, "data_wait_seconds": 4.9740076065063477e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20769, "cosmopedia_v2": 0.20038, "finemath": 0.08236}}
{"step": 10670, "total_steps": 72479, "tokens": 5594152960, "loss": 4.994397222995758, "l_mix": 3.0941928923130035, "l_a": 3.181806594133377, "l_c": 3.146750658750534, "z_loss": 16.371159315109253, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1890316903591156, "tokens_per_second": 1392443.081436986, "eta_seconds": 23272.561316156385, "data_wait_seconds": 3.096461296081543e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20769, "cosmopedia_v2": 0.20037, "finemath": 0.08238}}
{"step": 10680, "total_steps": 72479, "tokens": 5599395840, "loss": 3.572856068611145, "l_mix": 2.194932222366333, "l_a": 2.327638864517212, "l_c": 2.260335326194763, "z_loss": 15.315729975700378, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1989029198884964, "tokens_per_second": 1436541.5028403895, "eta_seconds": 22554.499155044556, "data_wait_seconds": 1.8835067749023438e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20768, "cosmopedia_v2": 0.20047, "finemath": 0.0823}}
{"step": 10690, "total_steps": 72479, "tokens": 5604638720, "loss": 3.5814707279205322, "l_mix": 2.200530171394348, "l_a": 2.3275305926799774, "l_c": 2.2704308927059174, "z_loss": 15.520745277404785, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18494679033756256, "tokens_per_second": 1379640.273144554, "eta_seconds": 23480.926051950453, "data_wait_seconds": 2.1398067474365234e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20767, "cosmopedia_v2": 0.20047, "finemath": 0.08232}}
{"step": 10700, "total_steps": 72479, "tokens": 5609881600, "loss": 3.5158026814460754, "l_mix": 2.160785585641861, "l_a": 2.289251059293747, "l_c": 2.2223159968852997, "z_loss": 15.4689040184021, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20174744725227356, "tokens_per_second": 1434868.151139413, "eta_seconds": 22573.49452371597, "data_wait_seconds": 1.800060272216797e-05, "mix": {"fineweb_hq": 0.50953, "dclm": 0.20766, "cosmopedia_v2": 0.20047, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.20646249, "gate_a2b/L1_absmax": 0.6225791, "gate_a2b/L1_rms": 0.23964369, "gate_a2b/L3_absmean": 0.29647177, "gate_a2b/L3_absmax": 0.47526926, "gate_a2b/L3_rms": 0.30217022, "gate_a2b/L5_absmean": 0.48460609, "gate_a2b/L5_absmax": 0.67378879, "gate_a2b/L5_rms": 0.49229392, "gate_c2b/L1_absmean": 0.01508181, "gate_c2b/L1_absmax": 0.52849221, "gate_c2b/L1_rms": 0.06091424, "gate_c2b/L3_absmean": 0.21356793, "gate_c2b/L3_absmax": 0.43889916, "gate_c2b/L3_rms": 0.21906707, "gate_c2b/L5_absmean": 0.54027301, "gate_c2b/L5_absmax": 0.75734997, "gate_c2b/L5_rms": 0.55360025, "gate_b2a/L7_absmean": 0.07395607, "gate_b2a/L7_absmax": 0.24572122, "gate_b2a/L7_rms": 0.0779418, "gate_b2a/L11_absmean": 0.12098487, "gate_b2a/L11_absmax": 0.327281, "gate_b2a/L11_rms": 0.12305252, "gate_b2a/L14_absmean": 0.29456314, "gate_b2a/L14_absmax": 0.45970589, "gate_b2a/L14_rms": 0.29560229, "gate_b2c/L3_absmean": 0.06431367, "gate_b2c/L3_absmax": 0.50262696, "gate_b2c/L3_rms": 0.08588494, "gate_b2c/L5_absmean": 0.10434495, "gate_b2c/L5_absmax": 0.57689297, "gate_b2c/L5_rms": 0.10943586, "gate_b2c/L6_absmean": 0.15574512, "gate_b2c/L6_absmax": 0.42245075, "gate_b2c/L6_rms": 0.1581803}, "streams": {"cos_a2b/round0": 0.358772, "cos_c2b/round0": 0.21829, "cos_b2a/round0": 0.168275, "cos_b2c/round0": 0.077515, "rms_a/round0": 1.015927, "rms_b/round0": 1.873154, "rms_c/round0": 1.318907, "cos_a2b/round1": 0.386447, "cos_c2b/round1": 0.265606, "cos_b2a/round1": 0.216892, "cos_b2c/round1": 0.150257, "rms_a/round1": 1.796454, "rms_b/round1": 3.529829, "rms_c/round1": 2.196, "cos_a2b/round2": 0.483837, "cos_c2b/round2": 0.445094, "cos_b2a/round2": 0.04043, "cos_b2c/round2": 0.123154, "rms_a/round2": 3.700367, "rms_b/round2": 7.457841, "rms_c/round2": 3.480012}, "alpha_mean": 0.4443918821957844, "alpha_hist": [0.04605, 0.12352, 0.15782, 0.14783, 0.12488, 0.10628, 0.10786, 0.11417, 0.0638, 0.0078], "alpha_by_type": {"special": {"share": 0.00105, "mean": 0.71413, "std": 0.10994}, "whitespace": {"share": 0.03313, "mean": 0.68971, "std": 0.1569}, "digit": {"share": 0.01324, "mean": 0.48844, "std": 0.19196}, "word_start": {"share": 0.55757, "mean": 0.46073, "std": 0.21529}, "word_cont": {"share": 0.27528, "mean": 0.28916, "std": 0.17013}, "punct": {"share": 0.11972, "mean": 0.65009, "std": 0.18426}}, "alpha_p05": 0.09641, "alpha_p50": 0.42561, "alpha_p95": 0.83114}
{"step": 10710, "total_steps": 72479, "tokens": 5615124480, "loss": 3.5593859255313873, "l_mix": 2.1874193251132965, "l_a": 2.3153958916664124, "l_c": 2.252695620059967, "z_loss": 15.390888333320618, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22008949518203735, "tokens_per_second": 1449188.6840530564, "eta_seconds": 22346.81089382172, "data_wait_seconds": 1.5497207641601562e-05, "mix": {"fineweb_hq": 0.50952, "dclm": 0.20766, "cosmopedia_v2": 0.20047, "finemath": 0.08235}}
{"step": 10720, "total_steps": 72479, "tokens": 5620367360, "loss": 3.595450669527054, "l_mix": 2.210140645503998, "l_a": 2.3370184302330017, "l_c": 2.2754237949848175, "z_loss": 15.772767305374146, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2615157663822174, "tokens_per_second": 1431649.0544783904, "eta_seconds": 22616.927305412293, "data_wait_seconds": 1.5854835510253906e-05, "mix": {"fineweb_hq": 0.50951, "dclm": 0.20765, "cosmopedia_v2": 0.20047, "finemath": 0.08237}}
{"step": 10730, "total_steps": 72479, "tokens": 5625610240, "loss": 4.250512361526489, "l_mix": 2.6250791251659393, "l_a": 2.7340337932109833, "l_c": 2.6786504685878754, "z_loss": 16.278037309646606, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.26335737109184265, "tokens_per_second": 1375837.5177175528, "eta_seconds": 23530.58358643055, "data_wait_seconds": 2.8759241104125977e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20764, "cosmopedia_v2": 0.20037, "finemath": 0.08239}}
{"step": 10740, "total_steps": 72479, "tokens": 5630853120, "loss": 4.25409123301506, "l_mix": 2.627509504556656, "l_a": 2.7396392226219177, "l_c": 2.676851511001587, "z_loss": 16.344594717025757, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21493761241436005, "tokens_per_second": 1323501.4817775514, "eta_seconds": 24457.106605219844, "data_wait_seconds": 1.71661376953125e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20764, "cosmopedia_v2": 0.20047, "finemath": 0.08231}}
{"step": 10750, "total_steps": 72479, "tokens": 5636096000, "loss": 4.163873702287674, "l_mix": 2.5704879462718964, "l_a": 2.6840085089206696, "l_c": 2.621659606695175, "z_loss": 16.852471113204956, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15465545654296875, "tokens_per_second": 1418458.5598727672, "eta_seconds": 22816.157530117034, "data_wait_seconds": 2.8014183044433594e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20763, "cosmopedia_v2": 0.20047, "finemath": 0.08233}}
{"step": 10760, "total_steps": 72479, "tokens": 5641338880, "loss": 3.5575478971004486, "l_mix": 2.186228722333908, "l_a": 2.311822682619095, "l_c": 2.2540979385375977, "z_loss": 15.429311752319336, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1798733025789261, "tokens_per_second": 1432012.0902167193, "eta_seconds": 22596.54879527092, "data_wait_seconds": 1.6629695892333984e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20762, "cosmopedia_v2": 0.20046, "finemath": 0.08234}}
{"step": 10770, "total_steps": 72479, "tokens": 5646581760, "loss": 3.5366057753562927, "l_mix": 2.1733908653259277, "l_a": 2.3009465038776398, "l_c": 2.2378515899181366, "z_loss": 15.755099415779114, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21080787479877472, "tokens_per_second": 1413513.0129254402, "eta_seconds": 22888.56762983799, "data_wait_seconds": 2.586841583251953e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20761, "cosmopedia_v2": 0.20046, "finemath": 0.08236}}
{"step": 10780, "total_steps": 72479, "tokens": 5651824640, "loss": 3.525793880224228, "l_mix": 2.1659722924232483, "l_a": 2.296471744775772, "l_c": 2.2310807406902313, "z_loss": 15.558813452720642, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18078649044036865, "tokens_per_second": 1453191.9565995166, "eta_seconds": 22259.994741296767, "data_wait_seconds": 2.142786979675293e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20761, "cosmopedia_v2": 0.20046, "finemath": 0.08237}}
{"step": 10790, "total_steps": 72479, "tokens": 5657067520, "loss": 5.099962294101715, "l_mix": 3.159302145242691, "l_a": 3.2459908723831177, "l_c": 3.2175916135311127, "z_loss": 15.853316187858582, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1971147209405899, "tokens_per_second": 1390637.4823188859, "eta_seconds": 23257.536808276178, "data_wait_seconds": 2.0384788513183594e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20769, "cosmopedia_v2": 0.20037, "finemath": 0.08239}}
{"step": 10800, "total_steps": 72479, "tokens": 5662310400, "loss": 3.564980924129486, "l_mix": 2.1916965544223785, "l_a": 2.3157885670661926, "l_c": 2.2567766904830933, "z_loss": 15.147506594657898, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.25382113456726074, "tokens_per_second": 1416886.8416495894, "eta_seconds": 22822.96553361416, "data_wait_seconds": 2.7120113372802734e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20769, "cosmopedia_v2": 0.20046, "finemath": 0.08231}, "gates": {"gate_a2b/L1_absmean": 0.20862909, "gate_a2b/L1_absmax": 0.62159437, "gate_a2b/L1_rms": 0.24173209, "gate_a2b/L3_absmean": 0.29720768, "gate_a2b/L3_absmax": 0.47331277, "gate_a2b/L3_rms": 0.30288392, "gate_a2b/L5_absmean": 0.48586527, "gate_a2b/L5_absmax": 0.67373937, "gate_a2b/L5_rms": 0.49346828, "gate_c2b/L1_absmean": 0.01549478, "gate_c2b/L1_absmax": 0.52226263, "gate_c2b/L1_rms": 0.06037964, "gate_c2b/L3_absmean": 0.21349235, "gate_c2b/L3_absmax": 0.43294099, "gate_c2b/L3_rms": 0.21885842, "gate_c2b/L5_absmean": 0.54042596, "gate_c2b/L5_absmax": 0.75442678, "gate_c2b/L5_rms": 0.55373037, "gate_b2a/L7_absmean": 0.0742455, "gate_b2a/L7_absmax": 0.24458271, "gate_b2a/L7_rms": 0.07812001, "gate_b2a/L11_absmean": 0.12022586, "gate_b2a/L11_absmax": 0.3267954, "gate_b2a/L11_rms": 0.12235478, "gate_b2a/L14_absmean": 0.2945019, "gate_b2a/L14_absmax": 0.45942587, "gate_b2a/L14_rms": 0.2955648, "gate_b2c/L3_absmean": 0.06449679, "gate_b2c/L3_absmax": 0.50395054, "gate_b2c/L3_rms": 0.08600334, "gate_b2c/L5_absmean": 0.1047186, "gate_b2c/L5_absmax": 0.58386856, "gate_b2c/L5_rms": 0.10990868, "gate_b2c/L6_absmean": 0.15660836, "gate_b2c/L6_absmax": 0.42189634, "gate_b2c/L6_rms": 0.15902536}, "streams": {"cos_a2b/round0": 0.362059, "cos_c2b/round0": 0.220931, "cos_b2a/round0": 0.167489, "cos_b2c/round0": 0.080188, "rms_a/round0": 1.003106, "rms_b/round0": 1.827122, "rms_c/round0": 1.304838, "cos_a2b/round1": 0.393894, "cos_c2b/round1": 0.27058, "cos_b2a/round1": 0.210274, "cos_b2c/round1": 0.151078, "rms_a/round1": 1.78997, "rms_b/round1": 3.445738, "rms_c/round1": 2.15686, "cos_a2b/round2": 0.492158, "cos_c2b/round2": 0.45294, "cos_b2a/round2": 0.044733, "cos_b2c/round2": 0.120685, "rms_a/round2": 3.692979, "rms_b/round2": 7.339885, "rms_c/round2": 3.43409}, "alpha_mean": 0.439502862161234, "alpha_hist": [0.03072, 0.11932, 0.16965, 0.15677, 0.13039, 0.12068, 0.1183, 0.10564, 0.04567, 0.00287], "alpha_by_type": {"special": {"share": 0.00107, "mean": 0.65551, "std": 0.0958}, "whitespace": {"share": 0.03487, "mean": 0.5987, "std": 0.1649}, "digit": {"share": 0.01294, "mean": 0.39241, "std": 0.15422}, "word_start": {"share": 0.57059, "mean": 0.45527, "std": 0.20515}, "word_cont": {"share": 0.25491, "mean": 0.29423, "std": 0.15929}, "punct": {"share": 0.12563, "mean": 0.62152, "std": 0.17978}}, "alpha_p05": 0.12169, "alpha_p50": 0.39981, "alpha_p95": 0.78431}
{"step": 10810, "total_steps": 72479, "tokens": 5667553280, "loss": 3.5977509021759033, "l_mix": 2.2111203372478485, "l_a": 2.340058773756027, "l_c": 2.276798903942108, "z_loss": 15.732262134552002, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2505835294723511, "tokens_per_second": 1360150.1230814601, "eta_seconds": 23771.138290786745, "data_wait_seconds": 2.9325485229492188e-05, "mix": {"fineweb_hq": 0.50953, "dclm": 0.20768, "cosmopedia_v2": 0.20046, "finemath": 0.08233}}
{"step": 10820, "total_steps": 72479, "tokens": 5672796160, "loss": 3.5756386816501617, "l_mix": 2.197866678237915, "l_a": 2.323616921901703, "l_c": 2.263849586248398, "z_loss": 15.319939136505127, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22878849506378174, "tokens_per_second": 1380306.5638827244, "eta_seconds": 23420.21304388046, "data_wait_seconds": 2.7060508728027344e-05, "mix": {"fineweb_hq": 0.50952, "dclm": 0.20767, "cosmopedia_v2": 0.20046, "finemath": 0.08235}}
{"step": 10830, "total_steps": 72479, "tokens": 5678039040, "loss": 3.608812242746353, "l_mix": 2.2177999317646027, "l_a": 2.348881781101227, "l_c": 2.2826784253120422, "z_loss": 15.44213318824768, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20211723446846008, "tokens_per_second": 1381573.023656502, "eta_seconds": 23394.949350166324, "data_wait_seconds": 2.110004425048828e-05, "mix": {"fineweb_hq": 0.50951, "dclm": 0.20766, "cosmopedia_v2": 0.20046, "finemath": 0.08236}}
{"step": 10840, "total_steps": 72479, "tokens": 5683281920, "loss": 4.228858560323715, "l_mix": 2.611438751220703, "l_a": 2.7254310846328735, "l_c": 2.6604428589344025, "z_loss": 16.57687282562256, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16510921716690063, "tokens_per_second": 1434530.6169903774, "eta_seconds": 22527.63911013603, "data_wait_seconds": 2.0891427993774414e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20766, "cosmopedia_v2": 0.20037, "finemath": 0.08238}}
{"step": 10850, "total_steps": 72479, "tokens": 5688524800, "loss": 4.290588319301605, "l_mix": 2.649024397134781, "l_a": 2.7626431584358215, "l_c": 2.7036437690258026, "z_loss": 16.77770733833313, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17834055423736572, "tokens_per_second": 1392518.824326331, "eta_seconds": 23203.5248554945, "data_wait_seconds": 1.6927719116210938e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20765, "cosmopedia_v2": 0.20046, "finemath": 0.0823}}
{"step": 10860, "total_steps": 72479, "tokens": 5693767680, "loss": 4.147843956947327, "l_mix": 2.5604403913021088, "l_a": 2.6769540905952454, "l_c": 2.6087015569210052, "z_loss": 17.067188262939453, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.175014927983284, "tokens_per_second": 1442478.6753213597, "eta_seconds": 22396.24254050255, "data_wait_seconds": 3.108382225036621e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20046, "finemath": 0.08232}}
{"step": 10870, "total_steps": 72479, "tokens": 5699010560, "loss": 3.5167422592639923, "l_mix": 2.1605391800403595, "l_a": 2.290233761072159, "l_c": 2.225188434123993, "z_loss": 15.763933420181274, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16417786478996277, "tokens_per_second": 1398274.2436042666, "eta_seconds": 23100.518041968346, "data_wait_seconds": 2.3812055587768555e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20764, "cosmopedia_v2": 0.20046, "finemath": 0.08234}}
{"step": 10880, "total_steps": 72479, "tokens": 5704253440, "loss": 3.5256170332431793, "l_mix": 2.167149692773819, "l_a": 2.2930099964141846, "l_c": 2.229916989803314, "z_loss": 15.89130973815918, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20084671676158905, "tokens_per_second": 1391395.1004156345, "eta_seconds": 23210.960353642702, "data_wait_seconds": 2.5987625122070312e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20763, "cosmopedia_v2": 0.20046, "finemath": 0.08235}}
{"step": 10890, "total_steps": 72479, "tokens": 5709496320, "loss": 3.572834312915802, "l_mix": 2.196210354566574, "l_a": 2.3213289380073547, "l_c": 2.262171983718872, "z_loss": 15.736204385757446, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2127590924501419, "tokens_per_second": 1393806.6552340658, "eta_seconds": 23167.03935279846, "data_wait_seconds": 2.828240394592285e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20762, "cosmopedia_v2": 0.20046, "finemath": 0.08237}}
{"step": 10900, "total_steps": 72479, "tokens": 5714739200, "loss": 3.539584666490555, "l_mix": 2.1749876737594604, "l_a": 2.302412062883377, "l_c": 2.241061955690384, "z_loss": 15.547429323196411, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18101459741592407, "tokens_per_second": 1320937.1523803528, "eta_seconds": 24441.080102729797, "data_wait_seconds": 2.0772218704223633e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20761, "cosmopedia_v2": 0.20046, "finemath": 0.08239}, "gates": {"gate_a2b/L1_absmean": 0.20915668, "gate_a2b/L1_absmax": 0.61605978, "gate_a2b/L1_rms": 0.2420059, "gate_a2b/L3_absmean": 0.29718697, "gate_a2b/L3_absmax": 0.48026145, "gate_a2b/L3_rms": 0.3029519, "gate_a2b/L5_absmean": 0.48575184, "gate_a2b/L5_absmax": 0.67478979, "gate_a2b/L5_rms": 0.49337339, "gate_c2b/L1_absmean": 0.01492224, "gate_c2b/L1_absmax": 0.52753645, "gate_c2b/L1_rms": 0.06056542, "gate_c2b/L3_absmean": 0.21390474, "gate_c2b/L3_absmax": 0.43642917, "gate_c2b/L3_rms": 0.21934286, "gate_c2b/L5_absmean": 0.53958148, "gate_c2b/L5_absmax": 0.75852424, "gate_c2b/L5_rms": 0.55279022, "gate_b2a/L7_absmean": 0.07400008, "gate_b2a/L7_absmax": 0.24629129, "gate_b2a/L7_rms": 0.07800551, "gate_b2a/L11_absmean": 0.12030792, "gate_b2a/L11_absmax": 0.33103999, "gate_b2a/L11_rms": 0.12248173, "gate_b2a/L14_absmean": 0.29454786, "gate_b2a/L14_absmax": 0.46192232, "gate_b2a/L14_rms": 0.2956036, "gate_b2c/L3_absmean": 0.06495495, "gate_b2c/L3_absmax": 0.50577414, "gate_b2c/L3_rms": 0.08628099, "gate_b2c/L5_absmean": 0.10472067, "gate_b2c/L5_absmax": 0.58360666, "gate_b2c/L5_rms": 0.10991202, "gate_b2c/L6_absmean": 0.15650418, "gate_b2c/L6_absmax": 0.4237164, "gate_b2c/L6_rms": 0.15893541}, "streams": {"cos_a2b/round0": 0.356999, "cos_c2b/round0": 0.221017, "cos_b2a/round0": 0.167719, "cos_b2c/round0": 0.077939, "rms_a/round0": 0.994341, "rms_b/round0": 1.838373, "rms_c/round0": 1.313736, "cos_a2b/round1": 0.38836, "cos_c2b/round1": 0.27175, "cos_b2a/round1": 0.212092, "cos_b2c/round1": 0.150142, "rms_a/round1": 1.775768, "rms_b/round1": 3.462936, "rms_c/round1": 2.184342, "cos_a2b/round2": 0.482966, "cos_c2b/round2": 0.452761, "cos_b2a/round2": 0.034406, "cos_b2c/round2": 0.117572, "rms_a/round2": 3.660326, "rms_b/round2": 7.393526, "rms_c/round2": 3.485353}, "alpha_mean": 0.4412595179234202, "alpha_hist": [0.04697, 0.14107, 0.16235, 0.13913, 0.11272, 0.10049, 0.10216, 0.10689, 0.07997, 0.00826], "alpha_by_type": {"special": {"share": 0.00114, "mean": 0.65244, "std": 0.10263}, "whitespace": {"share": 0.03133, "mean": 0.66891, "std": 0.16928}, "digit": {"share": 0.00938, "mean": 0.49202, "std": 0.2096}, "word_start": {"share": 0.58099, "mean": 0.45243, "std": 0.22421}, "word_cont": {"share": 0.26163, "mean": 0.28539, "std": 0.16993}, "punct": {"share": 0.11552, "mean": 0.67011, "std": 0.19563}}, "alpha_p05": 0.09947, "alpha_p50": 0.40192, "alpha_p95": 0.82104}
{"step": 10910, "total_steps": 72479, "tokens": 5719982080, "loss": 4.9690505266189575, "l_mix": 3.0775819420814514, "l_a": 3.166743814945221, "l_c": 3.1326803863048553, "z_loss": 16.412461519241333, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1850072741508484, "tokens_per_second": 1296091.6651589961, "eta_seconds": 24905.55933637619, "data_wait_seconds": 1.9729137420654297e-05, "mix": {"fineweb_hq": 0.50953, "dclm": 0.2077, "cosmopedia_v2": 0.20046, "finemath": 0.08231}}
{"step": 10920, "total_steps": 72479, "tokens": 5725224960, "loss": 5.099954009056091, "l_mix": 3.158850461244583, "l_a": 3.250043213367462, "l_c": 3.214906930923462, "z_loss": 16.184956312179565, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1772298812866211, "tokens_per_second": 1407245.371947209, "eta_seconds": 22934.62507348061, "data_wait_seconds": 1.6987323760986328e-05, "mix": {"fineweb_hq": 0.50952, "dclm": 0.20769, "cosmopedia_v2": 0.20046, "finemath": 0.08233}}
{"step": 10930, "total_steps": 72479, "tokens": 5730467840, "loss": 3.5408331751823425, "l_mix": 2.1759350299835205, "l_a": 2.3032150864601135, "l_c": 2.241188406944275, "z_loss": 15.770105838775635, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20082341134548187, "tokens_per_second": 1446045.0432362237, "eta_seconds": 22315.627208805086, "data_wait_seconds": 2.1696090698242188e-05, "mix": {"fineweb_hq": 0.50952, "dclm": 0.20769, "cosmopedia_v2": 0.20046, "finemath": 0.08234}}
{"step": 10940, "total_steps": 72479, "tokens": 5735710720, "loss": 3.5493742525577545, "l_mix": 2.18126580119133, "l_a": 2.3093323707580566, "l_c": 2.2458887696266174, "z_loss": 15.420153737068176, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18191222846508026, "tokens_per_second": 1362164.2805723979, "eta_seconds": 23685.953076410293, "data_wait_seconds": 1.621246337890625e-05, "mix": {"fineweb_hq": 0.50951, "dclm": 0.20768, "cosmopedia_v2": 0.20046, "finemath": 0.08236}}
{"step": 10950, "total_steps": 72479, "tokens": 5740953600, "loss": 4.144906133413315, "l_mix": 2.559937745332718, "l_a": 2.6690458059310913, "l_c": 2.608679920434952, "z_loss": 16.505747079849243, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.170701801776886, "tokens_per_second": 1286129.2373398615, "eta_seconds": 25082.173249340056, "data_wait_seconds": 1.5616416931152344e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20767, "cosmopedia_v2": 0.20037, "finemath": 0.08237}}
{"step": 10960, "total_steps": 72479, "tokens": 5746196480, "loss": 4.171533405780792, "l_mix": 2.574546992778778, "l_a": 2.685598224401474, "l_c": 2.6323655247688293, "z_loss": 15.972215414047241, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.257691890001297, "tokens_per_second": 1405064.014111064, "eta_seconds": 22955.30534415245, "data_wait_seconds": 2.9206275939941406e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20046, "finemath": 0.0823}}
{"step": 10970, "total_steps": 72479, "tokens": 5751439360, "loss": 4.120025008916855, "l_mix": 2.5436317920684814, "l_a": 2.6558078825473785, "l_c": 2.59318545460701, "z_loss": 16.950478076934814, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20035532116889954, "tokens_per_second": 1217760.2925694925, "eta_seconds": 26481.755718898774, "data_wait_seconds": 1.9222497940063477e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20046, "finemath": 0.08232}}
{"step": 10980, "total_steps": 72479, "tokens": 5756682240, "loss": 3.5487270951271057, "l_mix": 2.1819920539855957, "l_a": 2.303287237882614, "l_c": 2.247197449207306, "z_loss": 15.896056771278381, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2291100025177002, "tokens_per_second": 1374666.9596958037, "eta_seconds": 23455.272191262244, "data_wait_seconds": 2.7239322662353516e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20046, "finemath": 0.08233}}
{"step": 10990, "total_steps": 72479, "tokens": 5761925120, "loss": 3.553925544023514, "l_mix": 2.1842562556266785, "l_a": 2.3088571429252625, "l_c": 2.25160613656044, "z_loss": 15.301699757575989, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1849072128534317, "tokens_per_second": 1414863.1998436782, "eta_seconds": 22785.20272176266, "data_wait_seconds": 2.6434659957885742e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20764, "cosmopedia_v2": 0.20045, "finemath": 0.08235}}
{"step": 11000, "total_steps": 72479, "tokens": 5767168000, "loss": 3.5582175552845, "l_mix": 2.187218517065048, "l_a": 2.312654137611389, "l_c": 2.252180904150009, "z_loss": 15.484564542770386, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2068551480770111, "tokens_per_second": 1408848.2909082707, "eta_seconds": 22878.76001980305, "data_wait_seconds": 2.1249055862426758e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20764, "cosmopedia_v2": 0.20045, "finemath": 0.08236}, "gates": {"gate_a2b/L1_absmean": 0.21113209, "gate_a2b/L1_absmax": 0.6199798, "gate_a2b/L1_rms": 0.24354206, "gate_a2b/L3_absmean": 0.2980276, "gate_a2b/L3_absmax": 0.48022148, "gate_a2b/L3_rms": 0.30373043, "gate_a2b/L5_absmean": 0.48712197, "gate_a2b/L5_absmax": 0.67635703, "gate_a2b/L5_rms": 0.49471182, "gate_c2b/L1_absmean": 0.01531225, "gate_c2b/L1_absmax": 0.52405971, "gate_c2b/L1_rms": 0.06045104, "gate_c2b/L3_absmean": 0.21325965, "gate_c2b/L3_absmax": 0.43393371, "gate_c2b/L3_rms": 0.21865879, "gate_c2b/L5_absmean": 0.53888696, "gate_c2b/L5_absmax": 0.75936884, "gate_c2b/L5_rms": 0.55211103, "gate_b2a/L7_absmean": 0.07417725, "gate_b2a/L7_absmax": 0.24513228, "gate_b2a/L7_rms": 0.07807632, "gate_b2a/L11_absmean": 0.12047824, "gate_b2a/L11_absmax": 0.33079249, "gate_b2a/L11_rms": 0.12258328, "gate_b2a/L14_absmean": 0.29502848, "gate_b2a/L14_absmax": 0.46227473, "gate_b2a/L14_rms": 0.2960926, "gate_b2c/L3_absmean": 0.06585193, "gate_b2c/L3_absmax": 0.51013005, "gate_b2c/L3_rms": 0.08706323, "gate_b2c/L5_absmean": 0.10522461, "gate_b2c/L5_absmax": 0.5937196, "gate_b2c/L5_rms": 0.11052636, "gate_b2c/L6_absmean": 0.15725893, "gate_b2c/L6_absmax": 0.42543286, "gate_b2c/L6_rms": 0.15972272}, "streams": {"cos_a2b/round0": 0.364066, "cos_c2b/round0": 0.227971, "cos_b2a/round0": 0.169733, "cos_b2c/round0": 0.085617, "rms_a/round0": 0.993033, "rms_b/round0": 1.83544, "rms_c/round0": 1.303992, "cos_a2b/round1": 0.389569, "cos_c2b/round1": 0.269269, "cos_b2a/round1": 0.21257, "cos_b2c/round1": 0.150785, "rms_a/round1": 1.782378, "rms_b/round1": 3.454783, "rms_c/round1": 2.188858, "cos_a2b/round2": 0.492677, "cos_c2b/round2": 0.449073, "cos_b2a/round2": 0.037555, "cos_b2c/round2": 0.118055, "rms_a/round2": 3.693736, "rms_b/round2": 7.387897, "rms_c/round2": 3.49179}, "alpha_mean": 0.442545149749229, "alpha_hist": [0.04234, 0.12663, 0.16467, 0.147, 0.12067, 0.10692, 0.1109, 0.11165, 0.06195, 0.00726], "alpha_by_type": {"special": {"share": 0.00108, "mean": 0.64374, "std": 0.07301}, "whitespace": {"share": 0.03717, "mean": 0.63463, "std": 0.16858}, "digit": {"share": 0.01306, "mean": 0.4354, "std": 0.16159}, "word_start": {"share": 0.57442, "mean": 0.46038, "std": 0.2156}, "word_cont": {"share": 0.25308, "mean": 0.27671, "std": 0.16164}, "punct": {"share": 0.12119, "mean": 0.64441, "std": 0.1864}}, "alpha_p05": 0.11717, "alpha_p50": 0.4031, "alpha_p95": 0.81523}
{"step": 11010, "total_steps": 72479, "tokens": 5772410880, "loss": 3.525237739086151, "l_mix": 2.1654478311538696, "l_a": 2.293123036623001, "l_c": 2.2343006134033203, "z_loss": 15.627699613571167, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16688191890716553, "tokens_per_second": 1422251.5896691687, "eta_seconds": 22659.464265036582, "data_wait_seconds": 1.576542854309082e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20763, "cosmopedia_v2": 0.20045, "finemath": 0.08238}}
{"step": 11020, "total_steps": 72479, "tokens": 5777653760, "loss": 4.185792803764343, "l_mix": 2.584526628255844, "l_a": 2.700947254896164, "l_c": 2.6310680508613586, "z_loss": 16.614673137664795, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19658085703849792, "tokens_per_second": 1370150.801945332, "eta_seconds": 23517.27718310356, "data_wait_seconds": 2.8431415557861328e-05, "mix": {"fineweb_hq": 0.50962, "dclm": 0.20762, "cosmopedia_v2": 0.20045, "finemath": 0.0823}}
{"step": 11030, "total_steps": 72479, "tokens": 5782896640, "loss": 4.26007342338562, "l_mix": 2.6306899189949036, "l_a": 2.7428474128246307, "l_c": 2.6828328371047974, "z_loss": 16.793447256088257, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18315207958221436, "tokens_per_second": 1338236.2461330616, "eta_seconds": 24074.204689264297, "data_wait_seconds": 1.570582389831543e-05, "mix": {"fineweb_hq": 0.50961, "dclm": 0.20762, "cosmopedia_v2": 0.20045, "finemath": 0.08232}}
{"step": 11040, "total_steps": 72479, "tokens": 5788139520, "loss": 4.164109647274017, "l_mix": 2.5698926746845245, "l_a": 2.686247706413269, "l_c": 2.6221665143966675, "z_loss": 16.92525863647461, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22039368748664856, "tokens_per_second": 1259957.1547775392, "eta_seconds": 25565.734763169286, "data_wait_seconds": 1.6689300537109375e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20761, "cosmopedia_v2": 0.20045, "finemath": 0.08234}}
{"step": 11050, "total_steps": 72479, "tokens": 5793382400, "loss": 4.260259926319122, "l_mix": 2.6300903260707855, "l_a": 2.7458525002002716, "l_c": 2.6826645731925964, "z_loss": 16.144270658493042, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21508900821208954, "tokens_per_second": 1458182.2458885696, "eta_seconds": 22086.73685529232, "data_wait_seconds": 1.665949821472168e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.2076, "cosmopedia_v2": 0.20045, "finemath": 0.08235}}
{"step": 11060, "total_steps": 72479, "tokens": 5798625280, "loss": 4.168664366006851, "l_mix": 2.573832482099533, "l_a": 2.685594230890274, "l_c": 2.624880611896515, "z_loss": 16.892736434936523, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2016795575618744, "tokens_per_second": 1412191.518599727, "eta_seconds": 22802.321248841286, "data_wait_seconds": 4.023313522338867e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20759, "cosmopedia_v2": 0.20045, "finemath": 0.08237}}
{"step": 11070, "total_steps": 72479, "tokens": 5803868160, "loss": 4.283173561096191, "l_mix": 2.6461486518383026, "l_a": 2.7561845183372498, "l_c": 2.695053815841675, "z_loss": 16.533384323120117, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19662447273731232, "tokens_per_second": 1273788.7263602894, "eta_seconds": 25275.77856965065, "data_wait_seconds": 2.8967857360839844e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20768, "cosmopedia_v2": 0.20045, "finemath": 0.08229}}
{"step": 11080, "total_steps": 72479, "tokens": 5809111040, "loss": 4.172479927539825, "l_mix": 2.576634109020233, "l_a": 2.6878595650196075, "l_c": 2.626062721014023, "z_loss": 16.690396547317505, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1777806431055069, "tokens_per_second": 1317025.3434872378, "eta_seconds": 24442.019336366655, "data_wait_seconds": 1.773238182067871e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20045, "finemath": 0.08231}}
{"step": 11090, "total_steps": 72479, "tokens": 5814353920, "loss": 3.546478658914566, "l_mix": 2.1798903048038483, "l_a": 2.3065869212150574, "l_c": 2.243659794330597, "z_loss": 15.143152832984924, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.23781950771808624, "tokens_per_second": 1482005.5532175894, "eta_seconds": 21717.540775823592, "data_wait_seconds": 1.6808509826660156e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20045, "finemath": 0.08233}}
{"step": 11100, "total_steps": 72479, "tokens": 5819596800, "loss": 3.5137637853622437, "l_mix": 2.159925192594528, "l_a": 2.2837183475494385, "l_c": 2.223844736814499, "z_loss": 15.695999026298523, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18687506020069122, "tokens_per_second": 1431525.8469068627, "eta_seconds": 22479.701097631456, "data_wait_seconds": 2.0802021026611328e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20766, "cosmopedia_v2": 0.20045, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.2129488, "gate_a2b/L1_absmax": 0.62514251, "gate_a2b/L1_rms": 0.24506186, "gate_a2b/L3_absmean": 0.29915601, "gate_a2b/L3_absmax": 0.48068106, "gate_a2b/L3_rms": 0.30477419, "gate_a2b/L5_absmean": 0.48801589, "gate_a2b/L5_absmax": 0.67496359, "gate_a2b/L5_rms": 0.49552613, "gate_c2b/L1_absmean": 0.01517024, "gate_c2b/L1_absmax": 0.51887494, "gate_c2b/L1_rms": 0.06028552, "gate_c2b/L3_absmean": 0.2127599, "gate_c2b/L3_absmax": 0.43886015, "gate_c2b/L3_rms": 0.21815753, "gate_c2b/L5_absmean": 0.53894526, "gate_c2b/L5_absmax": 0.75967503, "gate_c2b/L5_rms": 0.55211031, "gate_b2a/L7_absmean": 0.07443941, "gate_b2a/L7_absmax": 0.24650811, "gate_b2a/L7_rms": 0.07830854, "gate_b2a/L11_absmean": 0.12067454, "gate_b2a/L11_absmax": 0.33228168, "gate_b2a/L11_rms": 0.12273907, "gate_b2a/L14_absmean": 0.29524964, "gate_b2a/L14_absmax": 0.46306467, "gate_b2a/L14_rms": 0.29629824, "gate_b2c/L3_absmean": 0.06609198, "gate_b2c/L3_absmax": 0.51309413, "gate_b2c/L3_rms": 0.08760238, "gate_b2c/L5_absmean": 0.10608183, "gate_b2c/L5_absmax": 0.59704274, "gate_b2c/L5_rms": 0.11135755, "gate_b2c/L6_absmean": 0.15800524, "gate_b2c/L6_absmax": 0.42736387, "gate_b2c/L6_rms": 0.16048957}, "streams": {"cos_a2b/round0": 0.361567, "cos_c2b/round0": 0.221648, "cos_b2a/round0": 0.167456, "cos_b2c/round0": 0.079344, "rms_a/round0": 0.993872, "rms_b/round0": 1.824856, "rms_c/round0": 1.306935, "cos_a2b/round1": 0.390906, "cos_c2b/round1": 0.264885, "cos_b2a/round1": 0.214957, "cos_b2c/round1": 0.149302, "rms_a/round1": 1.771917, "rms_b/round1": 3.457353, "rms_c/round1": 2.157978, "cos_a2b/round2": 0.483996, "cos_c2b/round2": 0.452509, "cos_b2a/round2": 0.035761, "cos_b2c/round2": 0.120357, "rms_a/round2": 3.637754, "rms_b/round2": 7.339642, "rms_c/round2": 3.435968}, "alpha_mean": 0.4143656030574192, "alpha_hist": [0.05217, 0.14804, 0.17867, 0.14615, 0.11574, 0.106, 0.11235, 0.09908, 0.0386, 0.0032], "alpha_by_type": {"special": {"share": 0.0011, "mean": 0.61021, "std": 0.09413}, "whitespace": {"share": 0.03198, "mean": 0.64968, "std": 0.18212}, "digit": {"share": 0.01009, "mean": 0.35646, "std": 0.16454}, "word_start": {"share": 0.57864, "mean": 0.43413, "std": 0.20942}, "word_cont": {"share": 0.26552, "mean": 0.26324, "std": 0.15908}, "punct": {"share": 0.11267, "mean": 0.60549, "std": 0.1868}}, "alpha_p05": 0.09236, "alpha_p50": 0.38468, "alpha_p95": 0.79182}
{"step": 11110, "total_steps": 72479, "tokens": 5824839680, "loss": 3.4977957606315613, "l_mix": 2.1490468978881836, "l_a": 2.2763685286045074, "l_c": 2.214284986257553, "z_loss": 15.528297662734985, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1655883491039276, "tokens_per_second": 1338322.8220529654, "eta_seconds": 24041.307330203057, "data_wait_seconds": 2.2858381271362305e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20765, "cosmopedia_v2": 0.20045, "finemath": 0.08236}}
{"step": 11120, "total_steps": 72479, "tokens": 5830082560, "loss": 3.5622947216033936, "l_mix": 2.1894024908542633, "l_a": 2.3173094987869263, "l_c": 2.2538094222545624, "z_loss": 15.564828157424927, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19497331976890564, "tokens_per_second": 1438914.6965240599, "eta_seconds": 22356.980208563804, "data_wait_seconds": 3.522634506225586e-05, "mix": {"fineweb_hq": 0.50953, "dclm": 0.20764, "cosmopedia_v2": 0.20045, "finemath": 0.08237}}
{"step": 11130, "total_steps": 72479, "tokens": 5835325440, "loss": 4.29577824473381, "l_mix": 2.654051274061203, "l_a": 2.7650636434555054, "l_c": 2.701782464981079, "z_loss": 16.73021388053894, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21940240263938904, "tokens_per_second": 1408689.4500089972, "eta_seconds": 22832.95620038509, "data_wait_seconds": 2.9206275939941406e-05, "mix": {"fineweb_hq": 0.50961, "dclm": 0.20764, "cosmopedia_v2": 0.20045, "finemath": 0.0823}}
{"step": 11140, "total_steps": 72479, "tokens": 5840568320, "loss": 4.231263518333435, "l_mix": 2.614556163549423, "l_a": 2.7229265570640564, "l_c": 2.6605877578258514, "z_loss": 16.5304958820343, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2573870122432709, "tokens_per_second": 1323554.4552045055, "eta_seconds": 24297.679257202148, "data_wait_seconds": 1.659989356994629e-05, "mix": {"fineweb_hq": 0.50961, "dclm": 0.20763, "cosmopedia_v2": 0.20045, "finemath": 0.08232}}
{"step": 11150, "total_steps": 72479, "tokens": 5845811200, "loss": 4.294850051403046, "l_mix": 2.653059333562851, "l_a": 2.7632083892822266, "l_c": 2.7039240896701813, "z_loss": 16.5081729888916, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.23179633915424347, "tokens_per_second": 1253599.503053101, "eta_seconds": 25649.38696424961, "data_wait_seconds": 1.5050172805786133e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20762, "cosmopedia_v2": 0.20045, "finemath": 0.08233}}
{"step": 11160, "total_steps": 72479, "tokens": 5851054080, "loss": 4.25023552775383, "l_mix": 2.6253385841846466, "l_a": 2.737727850675583, "l_c": 2.6730529069900513, "z_loss": 16.625393629074097, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19727276265621185, "tokens_per_second": 1484736.947806073, "eta_seconds": 21652.86983630657, "data_wait_seconds": 2.0205974578857422e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20762, "cosmopedia_v2": 0.20045, "finemath": 0.08235}}
{"step": 11170, "total_steps": 72479, "tokens": 5856296960, "loss": 4.127796024084091, "l_mix": 2.548135817050934, "l_a": 2.660552501678467, "l_c": 2.599493831396103, "z_loss": 16.462803840637207, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18149858713150024, "tokens_per_second": 1381299.1383347348, "eta_seconds": 23270.537206554414, "data_wait_seconds": 1.6123056411743164e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20761, "cosmopedia_v2": 0.20045, "finemath": 0.08236}}
{"step": 11180, "total_steps": 72479, "tokens": 5861539840, "loss": 4.987789213657379, "l_mix": 3.0903465151786804, "l_a": 3.1753394305706024, "l_c": 3.1441740095615387, "z_loss": 15.886397004127502, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18808415532112122, "tokens_per_second": 1429621.4400849, "eta_seconds": 22480.307870936394, "data_wait_seconds": 2.3126602172851562e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20769, "cosmopedia_v2": 0.20045, "finemath": 0.08229}}
{"step": 11190, "total_steps": 72479, "tokens": 5866782720, "loss": 5.069154679775238, "l_mix": 3.1393001079559326, "l_a": 3.231976270675659, "l_c": 3.195289671421051, "z_loss": 16.747761011123657, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19881954789161682, "tokens_per_second": 1441772.2948005677, "eta_seconds": 22287.21369378567, "data_wait_seconds": 1.9997358322143555e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20769, "cosmopedia_v2": 0.20045, "finemath": 0.08231}}
{"step": 11200, "total_steps": 72479, "tokens": 5872025600, "loss": 5.1358500719070435, "l_mix": 3.1818427443504333, "l_a": 3.2716534435749054, "l_c": 3.236228734254837, "z_loss": 16.426522731781006, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18297231197357178, "tokens_per_second": 1436259.5569535552, "eta_seconds": 22369.107447504997, "data_wait_seconds": 2.5033950805664062e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20768, "cosmopedia_v2": 0.20045, "finemath": 0.08232}, "gates": {"gate_a2b/L1_absmean": 0.21502458, "gate_a2b/L1_absmax": 0.63292843, "gate_a2b/L1_rms": 0.24691334, "gate_a2b/L3_absmean": 0.30080226, "gate_a2b/L3_absmax": 0.48254213, "gate_a2b/L3_rms": 0.30631754, "gate_a2b/L5_absmean": 0.48964, "gate_a2b/L5_absmax": 0.67736942, "gate_a2b/L5_rms": 0.49715388, "gate_c2b/L1_absmean": 0.01622734, "gate_c2b/L1_absmax": 0.52298689, "gate_c2b/L1_rms": 0.06025797, "gate_c2b/L3_absmean": 0.21339007, "gate_c2b/L3_absmax": 0.43574446, "gate_c2b/L3_rms": 0.21873961, "gate_c2b/L5_absmean": 0.53865635, "gate_c2b/L5_absmax": 0.75765496, "gate_c2b/L5_rms": 0.55176604, "gate_b2a/L7_absmean": 0.07397076, "gate_b2a/L7_absmax": 0.2473157, "gate_b2a/L7_rms": 0.07786936, "gate_b2a/L11_absmean": 0.12130688, "gate_b2a/L11_absmax": 0.33475602, "gate_b2a/L11_rms": 0.12339543, "gate_b2a/L14_absmean": 0.29549384, "gate_b2a/L14_absmax": 0.45865825, "gate_b2a/L14_rms": 0.29657847, "gate_b2c/L3_absmean": 0.06631412, "gate_b2c/L3_absmax": 0.51351768, "gate_b2c/L3_rms": 0.08753111, "gate_b2c/L5_absmean": 0.1068761, "gate_b2c/L5_absmax": 0.59915209, "gate_b2c/L5_rms": 0.11208069, "gate_b2c/L6_absmean": 0.15894774, "gate_b2c/L6_absmax": 0.42067215, "gate_b2c/L6_rms": 0.16139096}, "streams": {"cos_a2b/round0": 0.395365, "cos_c2b/round0": 0.257237, "cos_b2a/round0": 0.174356, "cos_b2c/round0": 0.120732, "rms_a/round0": 0.968925, "rms_b/round0": 1.850364, "rms_c/round0": 1.270927, "cos_a2b/round1": 0.386666, "cos_c2b/round1": 0.270572, "cos_b2a/round1": 0.222752, "cos_b2c/round1": 0.144757, "rms_a/round1": 1.718376, "rms_b/round1": 3.343736, "rms_c/round1": 2.095019, "cos_a2b/round2": 0.505333, "cos_c2b/round2": 0.474144, "cos_b2a/round2": 0.105492, "cos_b2c/round2": 0.139884, "rms_a/round2": 3.581925, "rms_b/round2": 7.122711, "rms_c/round2": 3.319871}, "alpha_mean": 0.45335258010684854, "alpha_hist": [0.00862, 0.05237, 0.13028, 0.19418, 0.21573, 0.20039, 0.12653, 0.06174, 0.00998, 0.00018], "alpha_by_type": {"special": {"share": 0.0005, "mean": 0.59845, "std": 0.13181}, "whitespace": {"share": 0.07037, "mean": 0.555, "std": 0.11354}, "digit": {"share": 0.04396, "mean": 0.50524, "std": 0.14968}, "word_start": {"share": 0.50958, "mean": 0.4702, "std": 0.15206}, "word_cont": {"share": 0.26927, "mean": 0.33982, "std": 0.12699}, "punct": {"share": 0.10631, "mean": 0.57074, "std": 0.157}}, "alpha_p05": 0.18684, "alpha_p50": 0.47293, "alpha_p95": 0.73602}
{"step": 11210, "total_steps": 72479, "tokens": 5877268480, "loss": 3.5671686828136444, "l_mix": 2.1934106051921844, "l_a": 2.3168904781341553, "l_c": 2.2571649849414825, "z_loss": 15.414067268371582, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.24922597408294678, "tokens_per_second": 1141858.1806118062, "eta_seconds": 28131.866126132012, "data_wait_seconds": 3.129243850708008e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20045, "finemath": 0.08234}}
{"step": 11220, "total_steps": 72479, "tokens": 5882511360, "loss": 3.583886444568634, "l_mix": 2.2011325657367706, "l_a": 2.332174301147461, "l_c": 2.2718305587768555, "z_loss": 15.52463686466217, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2201111912727356, "tokens_per_second": 1425442.1922874604, "eta_seconds": 22531.505497574806, "data_wait_seconds": 3.585219383239746e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20766, "cosmopedia_v2": 0.20045, "finemath": 0.08235}}
{"step": 11230, "total_steps": 72479, "tokens": 5887754240, "loss": 3.5402660071849823, "l_mix": 2.1761149168014526, "l_a": 2.3032308220863342, "l_c": 2.2388449609279633, "z_loss": 15.283391833305359, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.198417529463768, "tokens_per_second": 1428169.2100573743, "eta_seconds": 22484.811663675307, "data_wait_seconds": 2.1785497665405273e-05, "mix": {"fineweb_hq": 0.50953, "dclm": 0.20766, "cosmopedia_v2": 0.20045, "finemath": 0.08237}}
{"step": 11240, "total_steps": 72479, "tokens": 5892997120, "loss": 4.102882832288742, "l_mix": 2.532695323228836, "l_a": 2.6453309059143066, "l_c": 2.5832047760486603, "z_loss": 16.267147064208984, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22146451473236084, "tokens_per_second": 1122987.972079311, "eta_seconds": 28590.57588350773, "data_wait_seconds": 0.1073998212814331, "mix": {"fineweb_hq": 0.50961, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.0823}}
{"step": 11250, "total_steps": 72479, "tokens": 5898240000, "loss": 4.3158198595047, "l_mix": 2.665926694869995, "l_a": 2.7760041654109955, "l_c": 2.7180802524089813, "z_loss": 16.677276372909546, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19342955946922302, "tokens_per_second": 1476291.278354026, "eta_seconds": 21744.780601692197, "data_wait_seconds": 2.110004425048828e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08231}}
{"step": 11260, "total_steps": 72479, "tokens": 5903482880, "loss": 4.255350828170776, "l_mix": 2.6280778646469116, "l_a": 2.7382305562496185, "l_c": 2.6804808378219604, "z_loss": 16.59355902671814, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15390589833259583, "tokens_per_second": 1391168.1529689296, "eta_seconds": 23071.536681962014, "data_wait_seconds": 3.781914710998535e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 11270, "total_steps": 72479, "tokens": 5908725760, "loss": 4.47335821390152, "l_mix": 2.7653075754642487, "l_a": 2.8698717951774597, "l_c": 2.8183068335056305, "z_loss": 15.969916224479675, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20519228279590607, "tokens_per_second": 1272245.9929315015, "eta_seconds": 25224.00885543823, "data_wait_seconds": 1.9252300262451172e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20763, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 11280, "total_steps": 72479, "tokens": 5913968640, "loss": 4.321103274822235, "l_mix": 2.669772297143936, "l_a": 2.7768503725528717, "l_c": 2.7221789956092834, "z_loss": 16.221484422683716, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20082509517669678, "tokens_per_second": 1481514.4161496973, "eta_seconds": 21657.501919817925, "data_wait_seconds": 1.7076730728149414e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20762, "cosmopedia_v2": 0.20044, "finemath": 0.08236}}
{"step": 11290, "total_steps": 72479, "tokens": 5919211520, "loss": 4.314731031656265, "l_mix": 2.6650788486003876, "l_a": 2.7802218198776245, "l_c": 2.713155895471573, "z_loss": 16.388668417930603, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16836147010326385, "tokens_per_second": 1167349.7663822854, "eta_seconds": 27481.6163551569, "data_wait_seconds": 3.0517578125e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20762, "cosmopedia_v2": 0.20044, "finemath": 0.08237}}
{"step": 11300, "total_steps": 72479, "tokens": 5924454400, "loss": 5.101576745510101, "l_mix": 3.15988352894783, "l_a": 3.2494603097438812, "l_c": 3.217266231775284, "z_loss": 16.751718521118164, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19692015647888184, "tokens_per_second": 1194854.6351920255, "eta_seconds": 26844.61741812229, "data_wait_seconds": 3.5762786865234375e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.2077, "cosmopedia_v2": 0.20044, "finemath": 0.0823}, "gates": {"gate_a2b/L1_absmean": 0.21753493, "gate_a2b/L1_absmax": 0.62864447, "gate_a2b/L1_rms": 0.24855074, "gate_a2b/L3_absmean": 0.30134287, "gate_a2b/L3_absmax": 0.48437616, "gate_a2b/L3_rms": 0.30690461, "gate_a2b/L5_absmean": 0.49046716, "gate_a2b/L5_absmax": 0.67805356, "gate_a2b/L5_rms": 0.49802992, "gate_c2b/L1_absmean": 0.01575801, "gate_c2b/L1_absmax": 0.52148038, "gate_c2b/L1_rms": 0.06055087, "gate_c2b/L3_absmean": 0.2131104, "gate_c2b/L3_absmax": 0.43687475, "gate_c2b/L3_rms": 0.21841036, "gate_c2b/L5_absmean": 0.53905278, "gate_c2b/L5_absmax": 0.75760949, "gate_c2b/L5_rms": 0.55218273, "gate_b2a/L7_absmean": 0.07498991, "gate_b2a/L7_absmax": 0.24604405, "gate_b2a/L7_rms": 0.07877591, "gate_b2a/L11_absmean": 0.12179875, "gate_b2a/L11_absmax": 0.33497664, "gate_b2a/L11_rms": 0.12390617, "gate_b2a/L14_absmean": 0.29546106, "gate_b2a/L14_absmax": 0.45996138, "gate_b2a/L14_rms": 0.2965107, "gate_b2c/L3_absmean": 0.06717684, "gate_b2c/L3_absmax": 0.51333749, "gate_b2c/L3_rms": 0.08847315, "gate_b2c/L5_absmean": 0.10694076, "gate_b2c/L5_absmax": 0.60454762, "gate_b2c/L5_rms": 0.11235359, "gate_b2c/L6_absmean": 0.15944046, "gate_b2c/L6_absmax": 0.43124419, "gate_b2c/L6_rms": 0.16194396}, "streams": {"cos_a2b/round0": 0.378968, "cos_c2b/round0": 0.268844, "cos_b2a/round0": 0.182342, "cos_b2c/round0": 0.137965, "rms_a/round0": 0.94109, "rms_b/round0": 1.818569, "rms_c/round0": 1.260605, "cos_a2b/round1": 0.401069, "cos_c2b/round1": 0.278358, "cos_b2a/round1": 0.23102, "cos_b2c/round1": 0.164169, "rms_a/round1": 1.696686, "rms_b/round1": 3.310654, "rms_c/round1": 2.061719, "cos_a2b/round2": 0.53347, "cos_c2b/round2": 0.459356, "cos_b2a/round2": 0.097014, "cos_b2c/round2": 0.162443, "rms_a/round2": 3.594772, "rms_b/round2": 6.958319, "rms_c/round2": 3.250655}, "alpha_mean": 0.4634114169624155, "alpha_hist": [0.00986, 0.04996, 0.12138, 0.17517, 0.21254, 0.21417, 0.13451, 0.07191, 0.01038, 0.00012], "alpha_by_type": {"special": {"share": 0.00029, "mean": 0.56242, "std": 0.09441}, "whitespace": {"share": 0.06848, "mean": 0.55166, "std": 0.09238}, "digit": {"share": 0.0238, "mean": 0.50984, "std": 0.10977}, "word_start": {"share": 0.55966, "mean": 0.47749, "std": 0.15251}, "word_cont": {"share": 0.23701, "mean": 0.33977, "std": 0.14001}, "punct": {"share": 0.11075, "mean": 0.59204, "std": 0.1534}}, "alpha_p05": 0.17442, "alpha_p50": 0.47147, "alpha_p95": 0.71919}
{"step": 11310, "total_steps": 72479, "tokens": 5929697280, "loss": 5.084276914596558, "l_mix": 3.149636536836624, "l_a": 3.2378731667995453, "l_c": 3.205538749694824, "z_loss": 16.1678147315979, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18905390799045563, "tokens_per_second": 1352417.8167643915, "eta_seconds": 23713.213678836823, "data_wait_seconds": 2.714991569519043e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20769, "cosmopedia_v2": 0.20044, "finemath": 0.08232}}
{"step": 11320, "total_steps": 72479, "tokens": 5934940160, "loss": 5.076777279376984, "l_mix": 3.1466324627399445, "l_a": 3.227193772792816, "l_c": 3.2012976706027985, "z_loss": 15.973429083824158, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21579290926456451, "tokens_per_second": 1379786.3101622665, "eta_seconds": 23239.054885411264, "data_wait_seconds": 1.755356788635254e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20769, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 11330, "total_steps": 72479, "tokens": 5940183040, "loss": 5.033819913864136, "l_mix": 3.1178849041461945, "l_a": 3.2111775875091553, "l_c": 3.1699755489826202, "z_loss": 15.8905611038208, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18915453553199768, "tokens_per_second": 1400998.1675418087, "eta_seconds": 22883.46098856926, "data_wait_seconds": 6.628036499023438e-05, "mix": {"fineweb_hq": 0.50953, "dclm": 0.20768, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 11340, "total_steps": 72479, "tokens": 5945425920, "loss": 5.0518679022789, "l_mix": 3.1297229528427124, "l_a": 3.2149981260299683, "l_c": 3.1867586970329285, "z_loss": 16.17712390422821, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.198835089802742, "tokens_per_second": 1379106.8559148351, "eta_seconds": 23242.90093586445, "data_wait_seconds": 2.1010637283325195e-05, "mix": {"fineweb_hq": 0.50952, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08236}}
{"step": 11350, "total_steps": 72479, "tokens": 5950668800, "loss": 4.628506004810333, "l_mix": 2.8622650802135468, "l_a": 2.963475376367569, "l_c": 2.9186538457870483, "z_loss": 16.021655559539795, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22716017067432404, "tokens_per_second": 1453689.4768022634, "eta_seconds": 22046.799996447564, "data_wait_seconds": 1.6391277313232422e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08229}}
{"step": 11360, "total_steps": 72479, "tokens": 5955911680, "loss": 4.643532454967499, "l_mix": 2.8721064627170563, "l_a": 2.9747455418109894, "l_c": 2.924592822790146, "z_loss": 16.245696306228638, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19565722346305847, "tokens_per_second": 1451576.5747406168, "eta_seconds": 22075.279271936415, "data_wait_seconds": 1.6123056411743164e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08231}}
{"step": 11370, "total_steps": 72479, "tokens": 5961154560, "loss": 4.693672776222229, "l_mix": 2.9034388065338135, "l_a": 3.0076789557933807, "l_c": 2.954255223274231, "z_loss": 16.536115646362305, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2194908708333969, "tokens_per_second": 1412560.3587287245, "eta_seconds": 22681.307169651984, "data_wait_seconds": 2.1964311599731445e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08232}}
{"step": 11380, "total_steps": 72479, "tokens": 5966397440, "loss": 4.826155185699463, "l_mix": 2.9866748452186584, "l_a": 3.0864934027194977, "l_c": 3.0397901237010956, "z_loss": 15.952406287193298, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1934829205274582, "tokens_per_second": 1419183.7640589026, "eta_seconds": 22571.758022642138, "data_wait_seconds": 2.6047229766845703e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 11390, "total_steps": 72479, "tokens": 5971640320, "loss": 4.747088253498077, "l_mix": 2.936364084482193, "l_a": 3.04318106174469, "l_c": 2.987202972173691, "z_loss": 16.088252544403076, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22249247133731842, "tokens_per_second": 1417490.9151989066, "eta_seconds": 22595.015804743765, "data_wait_seconds": 2.995133399963379e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 11400, "total_steps": 72479, "tokens": 5976883200, "loss": 4.7616947889328, "l_mix": 2.945873349905014, "l_a": 3.0434534549713135, "l_c": 3.003967523574829, "z_loss": 15.950299859046936, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.26024821400642395, "tokens_per_second": 1356617.255157389, "eta_seconds": 23605.026863884927, "data_wait_seconds": 3.0487775802612305e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20763, "cosmopedia_v2": 0.20044, "finemath": 0.08237}, "gates": {"gate_a2b/L1_absmean": 0.21761744, "gate_a2b/L1_absmax": 0.62385726, "gate_a2b/L1_rms": 0.24864732, "gate_a2b/L3_absmean": 0.30095676, "gate_a2b/L3_absmax": 0.48356777, "gate_a2b/L3_rms": 0.3065249, "gate_a2b/L5_absmean": 0.49002954, "gate_a2b/L5_absmax": 0.67469013, "gate_a2b/L5_rms": 0.49748263, "gate_c2b/L1_absmean": 0.01505713, "gate_c2b/L1_absmax": 0.51757842, "gate_c2b/L1_rms": 0.05983172, "gate_c2b/L3_absmean": 0.21210511, "gate_c2b/L3_absmax": 0.43111402, "gate_c2b/L3_rms": 0.21738362, "gate_c2b/L5_absmean": 0.53933877, "gate_c2b/L5_absmax": 0.75183165, "gate_c2b/L5_rms": 0.55245966, "gate_b2a/L7_absmean": 0.07486455, "gate_b2a/L7_absmax": 0.24635741, "gate_b2a/L7_rms": 0.07855112, "gate_b2a/L11_absmean": 0.12112942, "gate_b2a/L11_absmax": 0.33981109, "gate_b2a/L11_rms": 0.12327451, "gate_b2a/L14_absmean": 0.29501054, "gate_b2a/L14_absmax": 0.45784578, "gate_b2a/L14_rms": 0.29609165, "gate_b2c/L3_absmean": 0.06705716, "gate_b2c/L3_absmax": 0.51473796, "gate_b2c/L3_rms": 0.08857328, "gate_b2c/L5_absmean": 0.1065551, "gate_b2c/L5_absmax": 0.60290796, "gate_b2c/L5_rms": 0.11192705, "gate_b2c/L6_absmean": 0.15906893, "gate_b2c/L6_absmax": 0.42708015, "gate_b2c/L6_rms": 0.16159445}, "streams": {"cos_a2b/round0": 0.376184, "cos_c2b/round0": 0.260751, "cos_b2a/round0": 0.174621, "cos_b2c/round0": 0.123025, "rms_a/round0": 0.956321, "rms_b/round0": 1.823756, "rms_c/round0": 1.287101, "cos_a2b/round1": 0.397884, "cos_c2b/round1": 0.266611, "cos_b2a/round1": 0.217664, "cos_b2c/round1": 0.145847, "rms_a/round1": 1.721412, "rms_b/round1": 3.343189, "rms_c/round1": 2.088168, "cos_a2b/round2": 0.517547, "cos_c2b/round2": 0.467193, "cos_b2a/round2": 0.079996, "cos_b2c/round2": 0.141824, "rms_a/round2": 3.603306, "rms_b/round2": 7.206393, "rms_c/round2": 3.321941}, "alpha_mean": 0.45994870420161504, "alpha_hist": [0.00613, 0.04538, 0.13432, 0.19965, 0.20529, 0.18961, 0.13443, 0.07295, 0.01219, 5e-05], "alpha_by_type": {"special": {"share": 0.00127, "mean": 0.55602, "std": 0.10505}, "whitespace": {"share": 0.02858, "mean": 0.53459, "std": 0.1142}, "digit": {"share": 0.0365, "mean": 0.51295, "std": 0.10819}, "word_start": {"share": 0.5602, "mean": 0.4722, "std": 0.16348}, "word_cont": {"share": 0.26802, "mean": 0.36751, "std": 0.13018}, "punct": {"share": 0.10544, "mean": 0.59007, "std": 0.15362}}, "alpha_p05": 0.19315, "alpha_p50": 0.46587, "alpha_p95": 0.74466}
{"step": 11410, "total_steps": 72479, "tokens": 5982126080, "loss": 3.352155238389969, "l_mix": 2.0716842114925385, "l_a": 2.152942642569542, "l_c": 2.109309658408165, "z_loss": 17.952837467193604, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.23319067060947418, "tokens_per_second": 1456519.3857116627, "eta_seconds": 21982.36713228226, "data_wait_seconds": 2.911686897277832e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20762, "cosmopedia_v2": 0.20044, "finemath": 0.08238}}
{"step": 11420, "total_steps": 72479, "tokens": 5987368960, "loss": 5.184964895248413, "l_mix": 3.2134514153003693, "l_a": 3.3017578423023224, "l_c": 3.264481157064438, "z_loss": 16.416130304336548, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16954012215137482, "tokens_per_second": 1388776.127015275, "eta_seconds": 23050.872181105617, "data_wait_seconds": 4.228949546813965e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20771, "cosmopedia_v2": 0.20044, "finemath": 0.08231}}
{"step": 11430, "total_steps": 72479, "tokens": 5992611840, "loss": 5.076255023479462, "l_mix": 3.1457656621932983, "l_a": 3.2321536540985107, "l_c": 3.1973996460437775, "z_loss": 16.234140276908875, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2055525928735733, "tokens_per_second": 1426386.585564355, "eta_seconds": 22439.399273610117, "data_wait_seconds": 1.7940998077392578e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.2077, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 11440, "total_steps": 72479, "tokens": 5997854720, "loss": 5.118274688720703, "l_mix": 3.1711373031139374, "l_a": 3.2612936794757843, "l_c": 3.2237369120121, "z_loss": 16.28061056137085, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20155614614486694, "tokens_per_second": 1396189.6397551368, "eta_seconds": 22920.965978240965, "data_wait_seconds": 1.7702579498291016e-05, "mix": {"fineweb_hq": 0.50953, "dclm": 0.20769, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 11450, "total_steps": 72479, "tokens": 6003097600, "loss": 5.247280418872833, "l_mix": 3.252342641353607, "l_a": 3.3411786258220673, "l_c": 3.303279221057892, "z_loss": 16.00194013118744, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2084924876689911, "tokens_per_second": 1427185.0338821437, "eta_seconds": 22419.49823770523, "data_wait_seconds": 2.6494264602661133e-05, "mix": {"fineweb_hq": 0.50952, "dclm": 0.20769, "cosmopedia_v2": 0.20044, "finemath": 0.08236}}
{"step": 11460, "total_steps": 72479, "tokens": 6008340480, "loss": 5.1156856417655945, "l_mix": 3.16960746049881, "l_a": 3.257805287837982, "l_c": 3.223777621984482, "z_loss": 16.03200125694275, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.170012429356575, "tokens_per_second": 1403587.040051338, "eta_seconds": 22792.69369060993, "data_wait_seconds": 1.7702579498291016e-05, "mix": {"fineweb_hq": 0.50951, "dclm": 0.20768, "cosmopedia_v2": 0.20044, "finemath": 0.08237}}
{"step": 11470, "total_steps": 72479, "tokens": 6013583360, "loss": 4.762696385383606, "l_mix": 2.9467363357543945, "l_a": 3.0468506515026093, "l_c": 3.000791549682617, "z_loss": 16.67278480529785, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.14061957597732544, "tokens_per_second": 1410608.675730218, "eta_seconds": 22675.52095937729, "data_wait_seconds": 2.110004425048828e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.0823}}
{"step": 11480, "total_steps": 72479, "tokens": 6018826240, "loss": 4.672735214233398, "l_mix": 2.8908194601535797, "l_a": 2.9910526275634766, "l_c": 2.943120002746582, "z_loss": 16.638750076293945, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1568901687860489, "tokens_per_second": 1429161.7106683291, "eta_seconds": 22377.484278559685, "data_wait_seconds": 3.0875205993652344e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08232}}
{"step": 11490, "total_steps": 72479, "tokens": 6024069120, "loss": 4.8372045159339905, "l_mix": 2.994385749101639, "l_a": 3.0939436852931976, "l_c": 3.043484926223755, "z_loss": 15.899749755859375, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.23572273552417755, "tokens_per_second": 1439606.7810667327, "eta_seconds": 22211.482505178454, "data_wait_seconds": 2.5719404220581055e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 11500, "total_steps": 72479, "tokens": 6029312000, "loss": 4.68662703037262, "l_mix": 2.89821258187294, "l_a": 3.005012661218643, "l_c": 2.9509397447109222, "z_loss": 16.2864887714386, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2739968001842499, "tokens_per_second": 1391546.7847458723, "eta_seconds": 22974.835127687456, "data_wait_seconds": 1.6897916793823242e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.21965408, "gate_a2b/L1_absmax": 0.62887257, "gate_a2b/L1_rms": 0.2504341, "gate_a2b/L3_absmean": 0.30171219, "gate_a2b/L3_absmax": 0.48469868, "gate_a2b/L3_rms": 0.307235, "gate_a2b/L5_absmean": 0.49065048, "gate_a2b/L5_absmax": 0.68163031, "gate_a2b/L5_rms": 0.4980967, "gate_c2b/L1_absmean": 0.01513897, "gate_c2b/L1_absmax": 0.51199418, "gate_c2b/L1_rms": 0.05954353, "gate_c2b/L3_absmean": 0.21224622, "gate_c2b/L3_absmax": 0.42664638, "gate_c2b/L3_rms": 0.21751669, "gate_c2b/L5_absmean": 0.53969389, "gate_c2b/L5_absmax": 0.75338703, "gate_c2b/L5_rms": 0.55271363, "gate_b2a/L7_absmean": 0.07465041, "gate_b2a/L7_absmax": 0.24517581, "gate_b2a/L7_rms": 0.07835019, "gate_b2a/L11_absmean": 0.12164548, "gate_b2a/L11_absmax": 0.34122223, "gate_b2a/L11_rms": 0.12379462, "gate_b2a/L14_absmean": 0.29559219, "gate_b2a/L14_absmax": 0.46037373, "gate_b2a/L14_rms": 0.29664284, "gate_b2c/L3_absmean": 0.06767993, "gate_b2c/L3_absmax": 0.51591694, "gate_b2c/L3_rms": 0.08904319, "gate_b2c/L5_absmean": 0.10711667, "gate_b2c/L5_absmax": 0.60750574, "gate_b2c/L5_rms": 0.11246902, "gate_b2c/L6_absmean": 0.15966055, "gate_b2c/L6_absmax": 0.43430302, "gate_b2c/L6_rms": 0.16221602}, "streams": {"cos_a2b/round0": 0.375242, "cos_c2b/round0": 0.252253, "cos_b2a/round0": 0.169513, "cos_b2c/round0": 0.11803, "rms_a/round0": 0.94717, "rms_b/round0": 1.831421, "rms_c/round0": 1.283912, "cos_a2b/round1": 0.380669, "cos_c2b/round1": 0.260293, "cos_b2a/round1": 0.212964, "cos_b2c/round1": 0.146763, "rms_a/round1": 1.704473, "rms_b/round1": 3.38677, "rms_c/round1": 2.100761, "cos_a2b/round2": 0.506924, "cos_c2b/round2": 0.463739, "cos_b2a/round2": 0.074326, "cos_b2c/round2": 0.142142, "rms_a/round2": 3.552246, "rms_b/round2": 7.22098, "rms_c/round2": 3.329649}, "alpha_mean": 0.45665542017074756, "alpha_hist": [0.02142, 0.06168, 0.12558, 0.18605, 0.19055, 0.18329, 0.12852, 0.07867, 0.02361, 0.00063], "alpha_by_type": {"special": {"share": 0.00117, "mean": 0.61139, "std": 0.10321}, "whitespace": {"share": 0.02808, "mean": 0.53046, "std": 0.1355}, "digit": {"share": 0.03719, "mean": 0.4778, "std": 0.14009}, "word_start": {"share": 0.54674, "mean": 0.48702, "std": 0.16466}, "word_cont": {"share": 0.28317, "mean": 0.33143, "std": 0.1511}, "punct": {"share": 0.10365, "mean": 0.60929, "std": 0.16443}}, "alpha_p05": 0.16532, "alpha_p50": 0.46563, "alpha_p95": 0.76734}
{"step": 11510, "total_steps": 72479, "tokens": 6034554880, "loss": 4.816574573516846, "l_mix": 2.9811037182807922, "l_a": 3.082271546125412, "l_c": 3.0306360125541687, "z_loss": 15.985652446746826, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.23907674849033356, "tokens_per_second": 1331283.8935313486, "eta_seconds": 24010.89296379089, "data_wait_seconds": 2.396106719970703e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08236}}
{"step": 11520, "total_steps": 72479, "tokens": 6039797760, "loss": 3.4349162876605988, "l_mix": 2.1234899908304214, "l_a": 2.209202140569687, "l_c": 2.1562371104955673, "z_loss": 17.943762063980103, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2996414303779602, "tokens_per_second": 1477594.029018995, "eta_seconds": 21629.805998349188, "data_wait_seconds": 1.7076730728149414e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20764, "cosmopedia_v2": 0.20043, "finemath": 0.08238}}
{"step": 11530, "total_steps": 72479, "tokens": 6045040640, "loss": 3.527010977268219, "l_mix": 2.1813501119613647, "l_a": 2.255554974079132, "l_c": 2.2239414900541306, "z_loss": 18.11898446083069, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.3898594081401825, "tokens_per_second": 1400652.1119827037, "eta_seconds": 22814.251332378386, "data_wait_seconds": 1.722574234008789e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20763, "cosmopedia_v2": 0.20043, "finemath": 0.08239}}
{"step": 11540, "total_steps": 72479, "tokens": 6050283520, "loss": 4.714987516403198, "l_mix": 2.917752683162689, "l_a": 3.0169231593608856, "l_c": 2.968397080898285, "z_loss": 16.387067556381226, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1652318388223648, "tokens_per_second": 1406664.1243045514, "eta_seconds": 22713.01718723774, "data_wait_seconds": 2.5480985641479492e-05, "mix": {"fineweb_hq": 0.50962, "dclm": 0.20763, "cosmopedia_v2": 0.20043, "finemath": 0.08232}}
{"step": 11550, "total_steps": 72479, "tokens": 6055526400, "loss": 4.74240243434906, "l_mix": 2.9338265657424927, "l_a": 3.040448248386383, "l_c": 2.982804000377655, "z_loss": 16.001709818840027, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16594800353050232, "tokens_per_second": 1404366.4385337215, "eta_seconds": 22746.444713783265, "data_wait_seconds": 1.6450881958007812e-05, "mix": {"fineweb_hq": 0.50961, "dclm": 0.20762, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 11560, "total_steps": 72479, "tokens": 6060769280, "loss": 4.761282384395599, "l_mix": 2.9458945989608765, "l_a": 3.045014590024948, "l_c": 3.0008193254470825, "z_loss": 16.374474048614502, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21730805933475494, "tokens_per_second": 1345563.3626074404, "eta_seconds": 23736.601010084152, "data_wait_seconds": 1.767277717590332e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20761, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 11570, "total_steps": 72479, "tokens": 6066012160, "loss": 4.777119338512421, "l_mix": 2.955860882997513, "l_a": 3.0590800046920776, "l_c": 3.006360024213791, "z_loss": 16.26410746574402, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21201090514659882, "tokens_per_second": 1453198.4868074895, "eta_seconds": 21974.876853990554, "data_wait_seconds": 1.6868114471435547e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20761, "cosmopedia_v2": 0.20043, "finemath": 0.08237}}
{"step": 11580, "total_steps": 72479, "tokens": 6071255040, "loss": 4.6194539070129395, "l_mix": 2.85746893286705, "l_a": 2.9559330344200134, "l_c": 2.911844104528427, "z_loss": 16.51810932159424, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1577267348766327, "tokens_per_second": 1487015.93582218, "eta_seconds": 21471.6023835659, "data_wait_seconds": 2.2411346435546875e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20769, "cosmopedia_v2": 0.20043, "finemath": 0.0823}}
{"step": 11590, "total_steps": 72479, "tokens": 6076497920, "loss": 4.631671369075775, "l_mix": 2.865258812904358, "l_a": 2.9659608006477356, "l_c": 2.916565716266632, "z_loss": 16.545541524887085, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1593557447195053, "tokens_per_second": 1396641.703237037, "eta_seconds": 22857.23815779686, "data_wait_seconds": 1.84476375579834e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20768, "cosmopedia_v2": 0.20043, "finemath": 0.08231}}
{"step": 11600, "total_steps": 72479, "tokens": 6081740800, "loss": 4.653079807758331, "l_mix": 2.879245400428772, "l_a": 2.9762814939022064, "l_c": 2.9310791194438934, "z_loss": 16.260398864746094, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22096046805381775, "tokens_per_second": 1387626.8005841728, "eta_seconds": 23001.954948234557, "data_wait_seconds": 3.0487775802612305e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.22001739, "gate_a2b/L1_absmax": 0.62760013, "gate_a2b/L1_rms": 0.25068697, "gate_a2b/L3_absmean": 0.30141857, "gate_a2b/L3_absmax": 0.48698738, "gate_a2b/L3_rms": 0.30698186, "gate_a2b/L5_absmean": 0.4910987, "gate_a2b/L5_absmax": 0.67853487, "gate_a2b/L5_rms": 0.49846697, "gate_c2b/L1_absmean": 0.01518533, "gate_c2b/L1_absmax": 0.51089889, "gate_c2b/L1_rms": 0.05942268, "gate_c2b/L3_absmean": 0.21144052, "gate_c2b/L3_absmax": 0.42658827, "gate_c2b/L3_rms": 0.21667702, "gate_c2b/L5_absmean": 0.53868389, "gate_c2b/L5_absmax": 0.75021321, "gate_c2b/L5_rms": 0.55145329, "gate_b2a/L7_absmean": 0.07503255, "gate_b2a/L7_absmax": 0.2470814, "gate_b2a/L7_rms": 0.07868867, "gate_b2a/L11_absmean": 0.12169331, "gate_b2a/L11_absmax": 0.34341663, "gate_b2a/L11_rms": 0.12393409, "gate_b2a/L14_absmean": 0.29503679, "gate_b2a/L14_absmax": 0.46531084, "gate_b2a/L14_rms": 0.29610735, "gate_b2c/L3_absmean": 0.06808782, "gate_b2c/L3_absmax": 0.51935357, "gate_b2c/L3_rms": 0.08966355, "gate_b2c/L5_absmean": 0.10715272, "gate_b2c/L5_absmax": 0.60530239, "gate_b2c/L5_rms": 0.11249674, "gate_b2c/L6_absmean": 0.16017424, "gate_b2c/L6_absmax": 0.43519452, "gate_b2c/L6_rms": 0.1628048}, "streams": {"cos_a2b/round0": 0.382869, "cos_c2b/round0": 0.271019, "cos_b2a/round0": 0.177462, "cos_b2c/round0": 0.133187, "rms_a/round0": 0.95145, "rms_b/round0": 1.816872, "rms_c/round0": 1.2879, "cos_a2b/round1": 0.402089, "cos_c2b/round1": 0.267661, "cos_b2a/round1": 0.219676, "cos_b2c/round1": 0.13681, "rms_a/round1": 1.71901, "rms_b/round1": 3.371001, "rms_c/round1": 2.120779, "cos_a2b/round2": 0.509997, "cos_c2b/round2": 0.469391, "cos_b2a/round2": 0.087119, "cos_b2c/round2": 0.129897, "rms_a/round2": 3.609968, "rms_b/round2": 7.338086, "rms_c/round2": 3.381941}, "alpha_mean": 0.4433489216464963, "alpha_hist": [0.01633, 0.07164, 0.15036, 0.18797, 0.19258, 0.16389, 0.12912, 0.07436, 0.0134, 0.00035], "alpha_by_type": {"special": {"share": 0.00098, "mean": 0.59408, "std": 0.11775}, "whitespace": {"share": 0.0295, "mean": 0.52521, "std": 0.12356}, "digit": {"share": 0.04372, "mean": 0.45484, "std": 0.12206}, "word_start": {"share": 0.58003, "mean": 0.46477, "std": 0.1702}, "word_cont": {"share": 0.24928, "mean": 0.32496, "std": 0.14376}, "punct": {"share": 0.0965, "mean": 0.58871, "std": 0.16456}}, "alpha_p05": 0.17189, "alpha_p50": 0.43915, "alpha_p95": 0.73412}
{"step": 11610, "total_steps": 72479, "tokens": 6086983680, "loss": 4.648954749107361, "l_mix": 2.8765194416046143, "l_a": 2.9750228822231293, "l_c": 2.927601933479309, "z_loss": 16.477225065231323, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18283900618553162, "tokens_per_second": 1480782.7578320897, "eta_seconds": 21551.36268517971, "data_wait_seconds": 1.7583370208740234e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 11620, "total_steps": 72479, "tokens": 6092226560, "loss": 4.698475956916809, "l_mix": 2.907284677028656, "l_a": 3.0086847245693207, "l_c": 2.956407904624939, "z_loss": 16.63486075401306, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1940024346113205, "tokens_per_second": 1406524.3976327146, "eta_seconds": 22685.453196334838, "data_wait_seconds": 2.3543834686279297e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08236}}
{"step": 11630, "total_steps": 72479, "tokens": 6097469440, "loss": 4.848797678947449, "l_mix": 3.001855880022049, "l_a": 3.0987505316734314, "l_c": 3.0523044168949127, "z_loss": 16.252022862434387, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21415962278842926, "tokens_per_second": 1231761.4167515093, "eta_seconds": 25899.821246337888, "data_wait_seconds": 1.817941665649414e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08237}}
{"step": 11640, "total_steps": 72479, "tokens": 6102712320, "loss": 3.3965838253498077, "l_mix": 2.0995238721370697, "l_a": 2.178675264120102, "l_c": 2.1388823837041855, "z_loss": 17.926599502563477, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.23146750032901764, "tokens_per_second": 1471226.9819987682, "eta_seconds": 21680.650247907637, "data_wait_seconds": 2.0831823348999023e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08239}}
{"step": 11650, "total_steps": 72479, "tokens": 6107955200, "loss": 4.832154035568237, "l_mix": 2.9910886585712433, "l_a": 3.090630531311035, "l_c": 3.0406731367111206, "z_loss": 16.741584300994873, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2154606431722641, "tokens_per_second": 1468857.6453211869, "eta_seconds": 21712.05280075073, "data_wait_seconds": 2.3633241653442383e-05, "mix": {"fineweb_hq": 0.50961, "dclm": 0.20764, "cosmopedia_v2": 0.20043, "finemath": 0.08232}}
{"step": 11660, "total_steps": 72479, "tokens": 6113198080, "loss": 4.606775641441345, "l_mix": 2.849868059158325, "l_a": 2.947097033262253, "l_c": 2.903775691986084, "z_loss": 16.45657706260681, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17675882577896118, "tokens_per_second": 1337896.6476655714, "eta_seconds": 23833.434314703944, "data_wait_seconds": 2.7298927307128906e-05, "mix": {"fineweb_hq": 0.50961, "dclm": 0.20763, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 11670, "total_steps": 72479, "tokens": 6118440960, "loss": 4.6481799483299255, "l_mix": 2.8763445615768433, "l_a": 2.9745775759220123, "l_c": 2.9259786009788513, "z_loss": 16.684773206710815, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.23136281967163086, "tokens_per_second": 1356320.129336925, "eta_seconds": 23505.828972387313, "data_wait_seconds": 1.7434358596801758e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20763, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 11680, "total_steps": 72479, "tokens": 6123683840, "loss": 4.704269111156464, "l_mix": 2.9102622866630554, "l_a": 3.0098352432250977, "l_c": 2.964808851480484, "z_loss": 16.13535726070404, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20389503240585327, "tokens_per_second": 1275928.078773191, "eta_seconds": 24982.745220756533, "data_wait_seconds": 2.7507543563842773e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20762, "cosmopedia_v2": 0.20043, "finemath": 0.08236}}
{"step": 11690, "total_steps": 72479, "tokens": 6128926720, "loss": 4.689857006072998, "l_mix": 2.902632176876068, "l_a": 2.999509811401367, "l_c": 2.9524028599262238, "z_loss": 16.511298418045044, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1742851734161377, "tokens_per_second": 1436405.5357477972, "eta_seconds": 22187.98413040638, "data_wait_seconds": 2.5957822799682617e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20761, "cosmopedia_v2": 0.20043, "finemath": 0.08238}}
{"step": 11700, "total_steps": 72479, "tokens": 6134169600, "loss": 5.130802154541016, "l_mix": 3.178808808326721, "l_a": 3.270289659500122, "l_c": 3.230871021747589, "z_loss": 16.45141899585724, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.174424409866333, "tokens_per_second": 1421489.0645414498, "eta_seconds": 22417.12662227154, "data_wait_seconds": 1.6123056411743164e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20769, "cosmopedia_v2": 0.20043, "finemath": 0.08231}, "gates": {"gate_a2b/L1_absmean": 0.22262016, "gate_a2b/L1_absmax": 0.63284642, "gate_a2b/L1_rms": 0.2530618, "gate_a2b/L3_absmean": 0.3023465, "gate_a2b/L3_absmax": 0.48430339, "gate_a2b/L3_rms": 0.307818, "gate_a2b/L5_absmean": 0.49153978, "gate_a2b/L5_absmax": 0.67444324, "gate_a2b/L5_rms": 0.49892282, "gate_c2b/L1_absmean": 0.01500861, "gate_c2b/L1_absmax": 0.50898755, "gate_c2b/L1_rms": 0.05930627, "gate_c2b/L3_absmean": 0.21107601, "gate_c2b/L3_absmax": 0.42879474, "gate_c2b/L3_rms": 0.21637319, "gate_c2b/L5_absmean": 0.53830492, "gate_c2b/L5_absmax": 0.75162584, "gate_c2b/L5_rms": 0.55111367, "gate_b2a/L7_absmean": 0.07521878, "gate_b2a/L7_absmax": 0.24790119, "gate_b2a/L7_rms": 0.07901292, "gate_b2a/L11_absmean": 0.12179844, "gate_b2a/L11_absmax": 0.34596774, "gate_b2a/L11_rms": 0.124047, "gate_b2a/L14_absmean": 0.29504508, "gate_b2a/L14_absmax": 0.46103045, "gate_b2a/L14_rms": 0.29612833, "gate_b2c/L3_absmean": 0.06840527, "gate_b2c/L3_absmax": 0.52270716, "gate_b2c/L3_rms": 0.09007202, "gate_b2c/L5_absmean": 0.10737912, "gate_b2c/L5_absmax": 0.60907364, "gate_b2c/L5_rms": 0.11283143, "gate_b2c/L6_absmean": 0.16025296, "gate_b2c/L6_absmax": 0.43517849, "gate_b2c/L6_rms": 0.16287544}, "streams": {"cos_a2b/round0": 0.375708, "cos_c2b/round0": 0.254635, "cos_b2a/round0": 0.17126, "cos_b2c/round0": 0.121391, "rms_a/round0": 0.938628, "rms_b/round0": 1.813758, "rms_c/round0": 1.273745, "cos_a2b/round1": 0.39637, "cos_c2b/round1": 0.275554, "cos_b2a/round1": 0.228387, "cos_b2c/round1": 0.157362, "rms_a/round1": 1.687483, "rms_b/round1": 3.329679, "rms_c/round1": 2.098155, "cos_a2b/round2": 0.529669, "cos_c2b/round2": 0.465797, "cos_b2a/round2": 0.088522, "cos_b2c/round2": 0.155563, "rms_a/round2": 3.629939, "rms_b/round2": 7.065282, "rms_c/round2": 3.310745}, "alpha_mean": 0.4763517611851995, "alpha_hist": [0.0106, 0.04619, 0.10941, 0.16493, 0.19441, 0.2226, 0.1626, 0.07892, 0.01016, 0.00018], "alpha_by_type": {"special": {"share": 0.00073, "mean": 0.57283, "std": 0.13597}, "whitespace": {"share": 0.04665, "mean": 0.57791, "std": 0.106}, "digit": {"share": 0.02167, "mean": 0.55891, "std": 0.12372}, "word_start": {"share": 0.57899, "mean": 0.49883, "std": 0.14858}, "word_cont": {"share": 0.25278, "mean": 0.34864, "std": 0.14226}, "punct": {"share": 0.09918, "mean": 0.6041, "std": 0.15296}}, "alpha_p05": 0.1956, "alpha_p50": 0.47969, "alpha_p95": 0.72213}
{"step": 11710, "total_steps": 72479, "tokens": 6139412480, "loss": 4.987504661083221, "l_mix": 3.090160071849823, "l_a": 3.180796444416046, "l_c": 3.1382523477077484, "z_loss": 16.299495458602905, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19838939607143402, "tokens_per_second": 1408715.078730581, "eta_seconds": 22616.679520964622, "data_wait_seconds": 1.8149614334106445e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20769, "cosmopedia_v2": 0.20043, "finemath": 0.08232}}
{"step": 11720, "total_steps": 72479, "tokens": 6144655360, "loss": 5.09375524520874, "l_mix": 3.1552012860774994, "l_a": 3.2452794015407562, "l_c": 3.211153417825699, "z_loss": 16.239511609077454, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2053237408399582, "tokens_per_second": 1472443.8006834371, "eta_seconds": 21634.24816431999, "data_wait_seconds": 3.4302473068237305e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 11730, "total_steps": 72479, "tokens": 6149898240, "loss": 5.447739064693451, "l_mix": 3.376704156398773, "l_a": 3.470469146966934, "l_c": 3.4276584088802338, "z_loss": 15.966323375701904, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2329282909631729, "tokens_per_second": 1466837.5194472317, "eta_seconds": 21713.360402727125, "data_wait_seconds": 1.96993350982666e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 11740, "total_steps": 72479, "tokens": 6155141120, "loss": 5.410310864448547, "l_mix": 3.3540078699588776, "l_a": 3.43055322766304, "l_c": 3.418507844209671, "z_loss": 15.846357464790344, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.3076701760292053, "tokens_per_second": 1395704.1173800773, "eta_seconds": 22816.24624836445, "data_wait_seconds": 3.1054019927978516e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08237}}
{"step": 11750, "total_steps": 72479, "tokens": 6160384000, "loss": 3.520023435354233, "l_mix": 2.1750869750976562, "l_a": 2.26365202665329, "l_c": 2.213290959596634, "z_loss": 18.534733772277832, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.3009311258792877, "tokens_per_second": 1436657.3156782931, "eta_seconds": 22162.19943652153, "data_wait_seconds": 1.633167266845703e-05, "mix": {"fineweb_hq": 0.50953, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08238}}
{"step": 11760, "total_steps": 72479, "tokens": 6165626880, "loss": 4.6221818923950195, "l_mix": 2.8599046170711517, "l_a": 2.9616808593273163, "l_c": 2.9070132076740265, "z_loss": 16.690969944000244, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19167429208755493, "tokens_per_second": 1426985.6389606178, "eta_seconds": 22308.734021449087, "data_wait_seconds": 2.434849739074707e-05, "mix": {"fineweb_hq": 0.50961, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08231}}
{"step": 11770, "total_steps": 72479, "tokens": 6170869760, "loss": 4.831089079380035, "l_mix": 2.990612030029297, "l_a": 3.0873868465423584, "l_c": 3.042191594839096, "z_loss": 16.033130526542664, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1777144968509674, "tokens_per_second": 1415200.1023206527, "eta_seconds": 22490.81252877712, "data_wait_seconds": 2.321600914001465e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 11780, "total_steps": 72479, "tokens": 6176112640, "loss": 4.7463154792785645, "l_mix": 2.9370318055152893, "l_a": 3.0391171872615814, "l_c": 2.9863489270210266, "z_loss": 16.436139345169067, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21202561259269714, "tokens_per_second": 1420272.4114078123, "eta_seconds": 22406.79820039272, "data_wait_seconds": 1.9729137420654297e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20764, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 11790, "total_steps": 72479, "tokens": 6181355520, "loss": 4.578827083110809, "l_mix": 2.8323396742343903, "l_a": 2.9321381747722626, "l_c": 2.884156495332718, "z_loss": 15.988412976264954, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1958901733160019, "tokens_per_second": 1465542.5242773315, "eta_seconds": 21711.082349991797, "data_wait_seconds": 2.300739288330078e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20763, "cosmopedia_v2": 0.20042, "finemath": 0.08236}}
{"step": 11800, "total_steps": 72479, "tokens": 6186598400, "loss": 4.663103401660919, "l_mix": 2.8846964836120605, "l_a": 2.9848030507564545, "l_c": 2.937695860862732, "z_loss": 16.57085943222046, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18980823457241058, "tokens_per_second": 1357916.983275135, "eta_seconds": 23427.994453144074, "data_wait_seconds": 1.9431114196777344e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20763, "cosmopedia_v2": 0.20042, "finemath": 0.08237}, "gates": {"gate_a2b/L1_absmean": 0.22355647, "gate_a2b/L1_absmax": 0.63464433, "gate_a2b/L1_rms": 0.25371897, "gate_a2b/L3_absmean": 0.30240682, "gate_a2b/L3_absmax": 0.486256, "gate_a2b/L3_rms": 0.30786157, "gate_a2b/L5_absmean": 0.49177155, "gate_a2b/L5_absmax": 0.67804599, "gate_a2b/L5_rms": 0.49917468, "gate_c2b/L1_absmean": 0.01527798, "gate_c2b/L1_absmax": 0.50740349, "gate_c2b/L1_rms": 0.05925743, "gate_c2b/L3_absmean": 0.21109711, "gate_c2b/L3_absmax": 0.42724115, "gate_c2b/L3_rms": 0.21625835, "gate_c2b/L5_absmean": 0.53852123, "gate_c2b/L5_absmax": 0.75679225, "gate_c2b/L5_rms": 0.5513413, "gate_b2a/L7_absmean": 0.07525897, "gate_b2a/L7_absmax": 0.24693485, "gate_b2a/L7_rms": 0.07885426, "gate_b2a/L11_absmean": 0.12194562, "gate_b2a/L11_absmax": 0.34738708, "gate_b2a/L11_rms": 0.12417991, "gate_b2a/L14_absmean": 0.29526585, "gate_b2a/L14_absmax": 0.46339732, "gate_b2a/L14_rms": 0.29636338, "gate_b2c/L3_absmean": 0.06885584, "gate_b2c/L3_absmax": 0.52418095, "gate_b2c/L3_rms": 0.09013101, "gate_b2c/L5_absmean": 0.10713726, "gate_b2c/L5_absmax": 0.61291718, "gate_b2c/L5_rms": 0.1126105, "gate_b2c/L6_absmean": 0.16013849, "gate_b2c/L6_absmax": 0.43821171, "gate_b2c/L6_rms": 0.16276175}, "streams": {"cos_a2b/round0": 0.375097, "cos_c2b/round0": 0.266091, "cos_b2a/round0": 0.172906, "cos_b2c/round0": 0.126899, "rms_a/round0": 0.936714, "rms_b/round0": 1.804868, "rms_c/round0": 1.277064, "cos_a2b/round1": 0.403336, "cos_c2b/round1": 0.266907, "cos_b2a/round1": 0.211978, "cos_b2c/round1": 0.149334, "rms_a/round1": 1.709107, "rms_b/round1": 3.344987, "rms_c/round1": 2.098675, "cos_a2b/round2": 0.528229, "cos_c2b/round2": 0.454563, "cos_b2a/round2": 0.072058, "cos_b2c/round2": 0.139727, "rms_a/round2": 3.597225, "rms_b/round2": 7.154154, "rms_c/round2": 3.345662}, "alpha_mean": 0.4681643144310783, "alpha_hist": [0.0182, 0.05402, 0.12846, 0.17365, 0.18225, 0.17679, 0.15045, 0.09663, 0.01884, 0.0007], "alpha_by_type": {"special": {"share": 0.0015, "mean": 0.56673, "std": 0.12994}, "whitespace": {"share": 0.02681, "mean": 0.53281, "std": 0.13364}, "digit": {"share": 0.02937, "mean": 0.49571, "std": 0.13064}, "word_start": {"share": 0.58066, "mean": 0.49559, "std": 0.16982}, "word_cont": {"share": 0.25992, "mean": 0.33945, "std": 0.14793}, "punct": {"share": 0.10175, "mean": 0.61404, "std": 0.16023}}, "alpha_p05": 0.16532, "alpha_p50": 0.47037, "alpha_p95": 0.75958}
{"step": 11810, "total_steps": 72479, "tokens": 6191841280, "loss": 5.00956603884697, "l_mix": 3.1051718294620514, "l_a": 3.188283860683441, "l_c": 3.1543931365013123, "z_loss": 15.910953283309937, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19067592918872833, "tokens_per_second": 1366483.6581107716, "eta_seconds": 23277.284351849554, "data_wait_seconds": 1.8328428268432617e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20771, "cosmopedia_v2": 0.20042, "finemath": 0.0823}}
{"step": 11820, "total_steps": 72479, "tokens": 6197084160, "loss": 5.035413593053818, "l_mix": 3.1200985312461853, "l_a": 3.2065256237983704, "l_c": 3.1725362837314606, "z_loss": 15.96402883529663, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22054645419120789, "tokens_per_second": 1414326.6725310115, "eta_seconds": 22486.1670289278, "data_wait_seconds": 3.0368566513061523e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.2077, "cosmopedia_v2": 0.20042, "finemath": 0.08232}}
{"step": 11830, "total_steps": 72479, "tokens": 6202327040, "loss": 5.105761289596558, "l_mix": 3.1638735234737396, "l_a": 3.2515147626399994, "l_c": 3.21611887216568, "z_loss": 15.976243257522583, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1846114695072174, "tokens_per_second": 1301196.4226956079, "eta_seconds": 24437.158262491226, "data_wait_seconds": 1.8656253814697266e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20769, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 11840, "total_steps": 72479, "tokens": 6207569920, "loss": 5.06915545463562, "l_mix": 3.1401133835315704, "l_a": 3.230482041835785, "l_c": 3.1942382752895355, "z_loss": 16.257750749588013, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19317476451396942, "tokens_per_second": 1454555.93026037, "eta_seconds": 21857.04885635376, "data_wait_seconds": 2.1904706954956055e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20769, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 11850, "total_steps": 72479, "tokens": 6212812800, "loss": 5.056112110614777, "l_mix": 3.13381490111351, "l_a": 3.2199297547340393, "l_c": 3.1822784543037415, "z_loss": 16.347113370895386, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1787145882844925, "tokens_per_second": 1471538.383459575, "eta_seconds": 21601.242284464835, "data_wait_seconds": 2.5719404220581055e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20768, "cosmopedia_v2": 0.20042, "finemath": 0.08236}}
{"step": 11860, "total_steps": 72479, "tokens": 6218055680, "loss": 3.3843152821063995, "l_mix": 2.0908851474523544, "l_a": 2.1748142540454865, "l_c": 2.1307286769151688, "z_loss": 17.6721830368042, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.24192224442958832, "tokens_per_second": 1414303.9318918562, "eta_seconds": 22471.70042827129, "data_wait_seconds": 3.55839729309082e-05, "mix": {"fineweb_hq": 0.50953, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08238}}
{"step": 11870, "total_steps": 72479, "tokens": 6223298560, "loss": 4.62191504240036, "l_mix": 2.859871953725815, "l_a": 2.9581658840179443, "l_c": 2.9098883867263794, "z_loss": 16.267082452774048, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1900227665901184, "tokens_per_second": 1459756.5541825169, "eta_seconds": 21768.404670596123, "data_wait_seconds": 1.811981201171875e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08231}}
{"step": 11880, "total_steps": 72479, "tokens": 6228541440, "loss": 4.737868547439575, "l_mix": 2.931540846824646, "l_a": 3.032436728477478, "l_c": 2.9832371175289154, "z_loss": 16.25474762916565, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18018896877765656, "tokens_per_second": 1423897.5630513174, "eta_seconds": 22312.93130660057, "data_wait_seconds": 1.9311904907226562e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08232}}
{"step": 11890, "total_steps": 72479, "tokens": 6233784320, "loss": 4.590140879154205, "l_mix": 2.8390206396579742, "l_a": 2.94296458363533, "l_c": 2.8885706961154938, "z_loss": 16.597778797149658, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15815377235412598, "tokens_per_second": 1452878.6716112029, "eta_seconds": 21864.23839285374, "data_wait_seconds": 2.0265579223632812e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 11900, "total_steps": 72479, "tokens": 6239027200, "loss": 4.649309933185577, "l_mix": 2.8759749233722687, "l_a": 2.9793314039707184, "l_c": 2.9261681735515594, "z_loss": 16.850745677947998, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17753516137599945, "tokens_per_second": 1411949.0563072779, "eta_seconds": 22494.326271986964, "data_wait_seconds": 1.5437602996826172e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.22482248, "gate_a2b/L1_absmax": 0.64018595, "gate_a2b/L1_rms": 0.25484788, "gate_a2b/L3_absmean": 0.30295166, "gate_a2b/L3_absmax": 0.48721078, "gate_a2b/L3_rms": 0.30840516, "gate_a2b/L5_absmean": 0.49322072, "gate_a2b/L5_absmax": 0.68223768, "gate_a2b/L5_rms": 0.50057453, "gate_c2b/L1_absmean": 0.01555406, "gate_c2b/L1_absmax": 0.50830501, "gate_c2b/L1_rms": 0.05936113, "gate_c2b/L3_absmean": 0.21022491, "gate_c2b/L3_absmax": 0.42597079, "gate_c2b/L3_rms": 0.21540743, "gate_c2b/L5_absmean": 0.53797144, "gate_c2b/L5_absmax": 0.75549841, "gate_c2b/L5_rms": 0.55076456, "gate_b2a/L7_absmean": 0.07521954, "gate_b2a/L7_absmax": 0.24820304, "gate_b2a/L7_rms": 0.07905415, "gate_b2a/L11_absmean": 0.12101419, "gate_b2a/L11_absmax": 0.3478654, "gate_b2a/L11_rms": 0.1232602, "gate_b2a/L14_absmean": 0.296193, "gate_b2a/L14_absmax": 0.46435794, "gate_b2a/L14_rms": 0.297297, "gate_b2c/L3_absmean": 0.06889969, "gate_b2c/L3_absmax": 0.52639884, "gate_b2c/L3_rms": 0.09028837, "gate_b2c/L5_absmean": 0.10712292, "gate_b2c/L5_absmax": 0.61824054, "gate_b2c/L5_rms": 0.11270018, "gate_b2c/L6_absmean": 0.160478, "gate_b2c/L6_absmax": 0.43852842, "gate_b2c/L6_rms": 0.163176}, "streams": {"cos_a2b/round0": 0.381304, "cos_c2b/round0": 0.267259, "cos_b2a/round0": 0.176135, "cos_b2c/round0": 0.128539, "rms_a/round0": 0.946627, "rms_b/round0": 1.805924, "rms_c/round0": 1.275257, "cos_a2b/round1": 0.39364, "cos_c2b/round1": 0.269733, "cos_b2a/round1": 0.217394, "cos_b2c/round1": 0.151367, "rms_a/round1": 1.701836, "rms_b/round1": 3.323755, "rms_c/round1": 2.093095, "cos_a2b/round2": 0.525076, "cos_c2b/round2": 0.459844, "cos_b2a/round2": 0.086351, "cos_b2c/round2": 0.153275, "rms_a/round2": 3.59496, "rms_b/round2": 7.085229, "rms_c/round2": 3.300063}, "alpha_mean": 0.4582149811878651, "alpha_hist": [0.02161, 0.05678, 0.11983, 0.17336, 0.20081, 0.20006, 0.14252, 0.0739, 0.01105, 0.00011], "alpha_by_type": {"special": {"share": 0.00122, "mean": 0.50105, "std": 0.11328}, "whitespace": {"share": 0.03778, "mean": 0.51918, "std": 0.122}, "digit": {"share": 0.0377, "mean": 0.47329, "std": 0.13386}, "word_start": {"share": 0.53906, "mean": 0.48164, "std": 0.16176}, "word_cont": {"share": 0.26917, "mean": 0.3464, "std": 0.1525}, "punct": {"share": 0.11507, "mean": 0.58464, "std": 0.1631}}, "alpha_p05": 0.15355, "alpha_p50": 0.45205, "alpha_p95": 0.73106}
{"step": 11910, "total_steps": 72479, "tokens": 6244270080, "loss": 4.75127637386322, "l_mix": 2.938763976097107, "l_a": 3.0459216833114624, "l_c": 2.990274727344513, "z_loss": 16.534533500671387, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21047021448612213, "tokens_per_second": 1468721.1818625606, "eta_seconds": 21621.258183074, "data_wait_seconds": 4.7594308853149414e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20764, "cosmopedia_v2": 0.20042, "finemath": 0.08237}}
{"step": 11920, "total_steps": 72479, "tokens": 6249512960, "loss": 3.3904176354408264, "l_mix": 2.0959023535251617, "l_a": 2.1713112592697144, "l_c": 2.1378981173038483, "z_loss": 17.523894548416138, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.275412917137146, "tokens_per_second": 1225181.0176349543, "eta_seconds": 25914.829347658157, "data_wait_seconds": 1.8388032913208008e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20763, "cosmopedia_v2": 0.20042, "finemath": 0.08238}}
{"step": 11930, "total_steps": 72479, "tokens": 6254755840, "loss": 4.893033146858215, "l_mix": 3.029380649328232, "l_a": 3.101525843143463, "l_c": 3.1054973900318146, "z_loss": 15.455060720443726, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.6938550472259521, "tokens_per_second": 1472276.3096321672, "eta_seconds": 21561.92686407566, "data_wait_seconds": 2.008676528930664e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20771, "cosmopedia_v2": 0.20042, "finemath": 0.08231}}
{"step": 11940, "total_steps": 72479, "tokens": 6259998720, "loss": 5.764131665229797, "l_mix": 3.557904750108719, "l_a": 3.6345143914222717, "l_c": 3.7143622040748596, "z_loss": 15.63853669166565, "lr": 0.0023, "wd": 0.1, "grad_norm": 3.0987792015075684, "tokens_per_second": 1375594.8137008294, "eta_seconds": 23073.561281180384, "data_wait_seconds": 3.2007694244384766e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20771, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 11950, "total_steps": 72479, "tokens": 6265241600, "loss": 5.1660043597221375, "l_mix": 3.200512409210205, "l_a": 3.2921945452690125, "l_c": 3.254123032093048, "z_loss": 15.966692209243774, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.24902240931987762, "tokens_per_second": 1417621.8626516224, "eta_seconds": 22385.820357370376, "data_wait_seconds": 1.817941665649414e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.2077, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 11960, "total_steps": 72479, "tokens": 6270484480, "loss": 5.075562000274658, "l_mix": 3.1446419954299927, "l_a": 3.234465628862381, "l_c": 3.196537107229233, "z_loss": 16.191198110580444, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21237802505493164, "tokens_per_second": 1425521.5678367359, "eta_seconds": 22258.088679885863, "data_wait_seconds": 1.6629695892333984e-05, "mix": {"fineweb_hq": 0.50953, "dclm": 0.20769, "cosmopedia_v2": 0.20042, "finemath": 0.08236}}
{"step": 11970, "total_steps": 72479, "tokens": 6275727360, "loss": 5.105059564113617, "l_mix": 3.1620555222034454, "l_a": 3.2548587322235107, "l_c": 3.2163690328598022, "z_loss": 16.3565776348114, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22141021490097046, "tokens_per_second": 1317519.9393671632, "eta_seconds": 24078.68119797707, "data_wait_seconds": 1.6242265701293945e-05, "mix": {"fineweb_hq": 0.50952, "dclm": 0.20769, "cosmopedia_v2": 0.20042, "finemath": 0.08237}}
{"step": 11980, "total_steps": 72479, "tokens": 6280970240, "loss": 4.668689072132111, "l_mix": 2.8874451816082, "l_a": 2.991156369447708, "l_c": 2.9408543407917023, "z_loss": 16.405366897583008, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17129890620708466, "tokens_per_second": 1438697.7976589447, "eta_seconds": 22046.950904917718, "data_wait_seconds": 1.6421079635620117e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20768, "cosmopedia_v2": 0.20042, "finemath": 0.0823}}
{"step": 11990, "total_steps": 72479, "tokens": 6286213120, "loss": 4.715150594711304, "l_mix": 2.917142629623413, "l_a": 3.0178647935390472, "l_c": 2.969981849193573, "z_loss": 16.53903293609619, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15015889704227448, "tokens_per_second": 1370852.8201223854, "eta_seconds": 23134.25363137722, "data_wait_seconds": 2.3752450942993164e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08232}}
{"step": 12000, "total_steps": 72479, "tokens": 6291456000, "loss": 4.662815809249878, "l_mix": 2.8834208250045776, "l_a": 2.9876644909381866, "l_c": 2.9381581246852875, "z_loss": 16.480913877487183, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16567645967006683, "tokens_per_second": 1477578.143707099, "eta_seconds": 21459.720480465887, "data_wait_seconds": 1.5735626220703125e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.22347458, "gate_a2b/L1_absmax": 0.63753742, "gate_a2b/L1_rms": 0.25286421, "gate_a2b/L3_absmean": 0.30296054, "gate_a2b/L3_absmax": 0.49298307, "gate_a2b/L3_rms": 0.30837709, "gate_a2b/L5_absmean": 0.49221811, "gate_a2b/L5_absmax": 0.68195778, "gate_a2b/L5_rms": 0.49959376, "gate_c2b/L1_absmean": 0.01496129, "gate_c2b/L1_absmax": 0.51017839, "gate_c2b/L1_rms": 0.05914421, "gate_c2b/L3_absmean": 0.21066275, "gate_c2b/L3_absmax": 0.43168214, "gate_c2b/L3_rms": 0.2158681, "gate_c2b/L5_absmean": 0.53785956, "gate_c2b/L5_absmax": 0.75585198, "gate_c2b/L5_rms": 0.5507403, "gate_b2a/L7_absmean": 0.07600829, "gate_b2a/L7_absmax": 0.24697161, "gate_b2a/L7_rms": 0.07969047, "gate_b2a/L11_absmean": 0.12246405, "gate_b2a/L11_absmax": 0.34776279, "gate_b2a/L11_rms": 0.12476011, "gate_b2a/L14_absmean": 0.29572642, "gate_b2a/L14_absmax": 0.47183719, "gate_b2a/L14_rms": 0.29687533, "gate_b2c/L3_absmean": 0.06910998, "gate_b2c/L3_absmax": 0.53643954, "gate_b2c/L3_rms": 0.09005289, "gate_b2c/L5_absmean": 0.10680433, "gate_b2c/L5_absmax": 0.61119056, "gate_b2c/L5_rms": 0.11238813, "gate_b2c/L6_absmean": 0.16033098, "gate_b2c/L6_absmax": 0.44702244, "gate_b2c/L6_rms": 0.16303141}, "streams": {"cos_a2b/round0": 0.375572, "cos_c2b/round0": 0.25988, "cos_b2a/round0": 0.166275, "cos_b2c/round0": 0.127071, "rms_a/round0": 0.954717, "rms_b/round0": 1.811191, "rms_c/round0": 1.286213, "cos_a2b/round1": 0.3997, "cos_c2b/round1": 0.266291, "cos_b2a/round1": 0.217829, "cos_b2c/round1": 0.147794, "rms_a/round1": 1.709798, "rms_b/round1": 3.356444, "rms_c/round1": 2.107051, "cos_a2b/round2": 0.523071, "cos_c2b/round2": 0.459251, "cos_b2a/round2": 0.078601, "cos_b2c/round2": 0.131802, "rms_a/round2": 3.606112, "rms_b/round2": 7.201326, "rms_c/round2": 3.325507}, "alpha_mean": 0.4466256529255048, "alpha_hist": [0.01627, 0.06734, 0.14972, 0.18501, 0.18234, 0.1772, 0.13718, 0.07242, 0.01225, 0.00027], "alpha_by_type": {"special": {"share": 0.00108, "mean": 0.56789, "std": 0.11717}, "whitespace": {"share": 0.0285, "mean": 0.50442, "std": 0.12685}, "digit": {"share": 0.04012, "mean": 0.4354, "std": 0.13336}, "word_start": {"share": 0.56688, "mean": 0.47921, "std": 0.1624}, "word_cont": {"share": 0.25511, "mean": 0.31545, "std": 0.14498}, "punct": {"share": 0.10831, "mean": 0.57278, "std": 0.1693}}, "alpha_p05": 0.17639, "alpha_p50": 0.45568, "alpha_p95": 0.73526}
{"step": 12010, "total_steps": 72479, "tokens": 6296698880, "loss": 4.607975900173187, "l_mix": 2.850834012031555, "l_a": 2.95094096660614, "l_c": 2.900756686925888, "z_loss": 16.324317455291748, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15220078825950623, "tokens_per_second": 1368550.3580143396, "eta_seconds": 23165.512972426415, "data_wait_seconds": 1.800060272216797e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 12020, "total_steps": 72479, "tokens": 6301941760, "loss": 4.629299163818359, "l_mix": 2.8631394803524017, "l_a": 2.9665406346321106, "l_c": 2.9152182936668396, "z_loss": 16.320987224578857, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17953979969024658, "tokens_per_second": 1385456.3871585743, "eta_seconds": 22879.051614904405, "data_wait_seconds": 2.619624137878418e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08236}}
{"step": 12030, "total_steps": 72479, "tokens": 6307184640, "loss": 3.2933073937892914, "l_mix": 2.0334616154432297, "l_a": 2.125834971666336, "l_c": 2.067562460899353, "z_loss": 18.264760971069336, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2139347493648529, "tokens_per_second": 1424608.8670838436, "eta_seconds": 22246.58714702129, "data_wait_seconds": 4.464387893676758e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08238}}
{"step": 12040, "total_steps": 72479, "tokens": 6312427520, "loss": 3.509102076292038, "l_mix": 2.1701616644859314, "l_a": 2.2479888796806335, "l_c": 2.2090001702308655, "z_loss": 18.43672251701355, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.27858832478523254, "tokens_per_second": 1344405.181922264, "eta_seconds": 23569.86037995815, "data_wait_seconds": 2.1278858184814453e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20764, "cosmopedia_v2": 0.20042, "finemath": 0.08239}}
{"step": 12050, "total_steps": 72479, "tokens": 6317670400, "loss": 4.733036518096924, "l_mix": 2.92817285656929, "l_a": 3.031819075345993, "l_c": 2.9788475930690765, "z_loss": 16.634573936462402, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1736145168542862, "tokens_per_second": 1357346.60590213, "eta_seconds": 23341.274376225472, "data_wait_seconds": 2.148747444152832e-05, "mix": {"fineweb_hq": 0.50963, "dclm": 0.20763, "cosmopedia_v2": 0.20041, "finemath": 0.08232}}
{"step": 12060, "total_steps": 72479, "tokens": 6322913280, "loss": 4.658906519412994, "l_mix": 2.8821361660957336, "l_a": 2.980604976415634, "l_c": 2.936512917280197, "z_loss": 16.349941730499268, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17368252575397491, "tokens_per_second": 1479576.4133603463, "eta_seconds": 21409.476648831365, "data_wait_seconds": 2.5957822799682617e-05, "mix": {"fineweb_hq": 0.50962, "dclm": 0.20763, "cosmopedia_v2": 0.20041, "finemath": 0.08234}}
{"step": 12070, "total_steps": 72479, "tokens": 6328156160, "loss": 4.812387704849243, "l_mix": 2.979056030511856, "l_a": 3.0727968215942383, "l_c": 3.032840847969055, "z_loss": 16.4023654460907, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18206533789634705, "tokens_per_second": 1395601.3671240031, "eta_seconds": 22693.954404234886, "data_wait_seconds": 2.6881694793701172e-05, "mix": {"fineweb_hq": 0.50961, "dclm": 0.20762, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 12080, "total_steps": 72479, "tokens": 6333399040, "loss": 4.695455014705658, "l_mix": 2.902789980173111, "l_a": 3.0114466845989227, "l_c": 2.958689033985138, "z_loss": 16.242518424987793, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2337571084499359, "tokens_per_second": 1458006.7697157362, "eta_seconds": 21719.015007162092, "data_wait_seconds": 3.001093864440918e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20762, "cosmopedia_v2": 0.20041, "finemath": 0.08237}}
{"step": 12090, "total_steps": 72479, "tokens": 6338641920, "loss": 4.558620870113373, "l_mix": 2.82066610455513, "l_a": 2.921038120985031, "l_c": 2.8666070997714996, "z_loss": 16.610763788223267, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18506933748722076, "tokens_per_second": 1441236.233063851, "eta_seconds": 21968.10439929962, "data_wait_seconds": 1.5944242477416992e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20761, "cosmopedia_v2": 0.20041, "finemath": 0.08238}}
{"step": 12100, "total_steps": 72479, "tokens": 6343884800, "loss": 4.615798234939575, "l_mix": 2.8549392223358154, "l_a": 2.9577519297599792, "l_c": 2.9063332080841064, "z_loss": 16.334672212600708, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1809391975402832, "tokens_per_second": 1368964.3290838408, "eta_seconds": 23124.03945045471, "data_wait_seconds": 2.148747444152832e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20769, "cosmopedia_v2": 0.20041, "finemath": 0.08231}, "gates": {"gate_a2b/L1_absmean": 0.22624843, "gate_a2b/L1_absmax": 0.64531666, "gate_a2b/L1_rms": 0.25545242, "gate_a2b/L3_absmean": 0.30345818, "gate_a2b/L3_absmax": 0.48816779, "gate_a2b/L3_rms": 0.30876765, "gate_a2b/L5_absmean": 0.49219981, "gate_a2b/L5_absmax": 0.68036681, "gate_a2b/L5_rms": 0.49955237, "gate_c2b/L1_absmean": 0.01506891, "gate_c2b/L1_absmax": 0.50630462, "gate_c2b/L1_rms": 0.05875681, "gate_c2b/L3_absmean": 0.21036315, "gate_c2b/L3_absmax": 0.43010291, "gate_c2b/L3_rms": 0.21559788, "gate_c2b/L5_absmean": 0.53819621, "gate_c2b/L5_absmax": 0.76053548, "gate_c2b/L5_rms": 0.5510838, "gate_b2a/L7_absmean": 0.07603108, "gate_b2a/L7_absmax": 0.24647696, "gate_b2a/L7_rms": 0.07971458, "gate_b2a/L11_absmean": 0.12225266, "gate_b2a/L11_absmax": 0.34985951, "gate_b2a/L11_rms": 0.1245619, "gate_b2a/L14_absmean": 0.29636741, "gate_b2a/L14_absmax": 0.46589977, "gate_b2a/L14_rms": 0.2974745, "gate_b2c/L3_absmean": 0.06925137, "gate_b2c/L3_absmax": 0.53468007, "gate_b2c/L3_rms": 0.09039448, "gate_b2c/L5_absmean": 0.10728556, "gate_b2c/L5_absmax": 0.61544847, "gate_b2c/L5_rms": 0.11287231, "gate_b2c/L6_absmean": 0.1610193, "gate_b2c/L6_absmax": 0.44369295, "gate_b2c/L6_rms": 0.16371706}, "streams": {"cos_a2b/round0": 0.374281, "cos_c2b/round0": 0.260366, "cos_b2a/round0": 0.167945, "cos_b2c/round0": 0.123373, "rms_a/round0": 0.945103, "rms_b/round0": 1.799767, "rms_c/round0": 1.281314, "cos_a2b/round1": 0.388159, "cos_c2b/round1": 0.257823, "cos_b2a/round1": 0.209612, "cos_b2c/round1": 0.147634, "rms_a/round1": 1.708128, "rms_b/round1": 3.380094, "rms_c/round1": 2.112595, "cos_a2b/round2": 0.515848, "cos_c2b/round2": 0.450851, "cos_b2a/round2": 0.073096, "cos_b2c/round2": 0.135116, "rms_a/round2": 3.584857, "rms_b/round2": 7.233971, "rms_c/round2": 3.343152}, "alpha_mean": 0.44548297514053914, "alpha_hist": [0.02939, 0.07899, 0.14317, 0.17009, 0.17303, 0.1656, 0.13802, 0.08354, 0.01723, 0.00093], "alpha_by_type": {"special": {"share": 0.0016, "mean": 0.57508, "std": 0.11308}, "whitespace": {"share": 0.03299, "mean": 0.53818, "std": 0.13017}, "digit": {"share": 0.04538, "mean": 0.60359, "std": 0.14387}, "word_start": {"share": 0.54341, "mean": 0.46654, "std": 0.17084}, "word_cont": {"share": 0.26584, "mean": 0.30364, "std": 0.15429}, "punct": {"share": 0.11078, "mean": 0.58831, "std": 0.1656}}, "alpha_p05": 0.12421, "alpha_p50": 0.43927, "alpha_p95": 0.73048}
{"step": 12110, "total_steps": 72479, "tokens": 6349127680, "loss": 4.670256674289703, "l_mix": 2.888404279947281, "l_a": 2.9922133684158325, "l_c": 2.9418409168720245, "z_loss": 16.36078715324402, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.14667566120624542, "tokens_per_second": 1463874.733291368, "eta_seconds": 21621.20948753357, "data_wait_seconds": 1.9311904907226562e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20768, "cosmopedia_v2": 0.20041, "finemath": 0.08233}}
{"step": 12120, "total_steps": 72479, "tokens": 6354370560, "loss": 4.723681211471558, "l_mix": 2.921604573726654, "l_a": 3.0278216898441315, "l_c": 2.9736730754375458, "z_loss": 16.280711889266968, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18896934390068054, "tokens_per_second": 1431352.9074590337, "eta_seconds": 22108.802956342697, "data_wait_seconds": 1.5556812286376953e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08234}}
{"step": 12130, "total_steps": 72479, "tokens": 6359613440, "loss": 4.736102640628815, "l_mix": 2.9296391308307648, "l_a": 3.0341672003269196, "l_c": 2.9819009006023407, "z_loss": 16.430367946624756, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18674971163272858, "tokens_per_second": 1436576.6952043083, "eta_seconds": 22024.7597066164, "data_wait_seconds": 2.4139881134033203e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08236}}
{"step": 12140, "total_steps": 72479, "tokens": 6364856320, "loss": 3.4072558283805847, "l_mix": 2.1045663356781006, "l_a": 2.1802382469177246, "l_c": 2.1562876403331757, "z_loss": 17.316365957260132, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.3197164237499237, "tokens_per_second": 1431175.7251421919, "eta_seconds": 22104.2133934021, "data_wait_seconds": 2.816319465637207e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20041, "finemath": 0.08237}}
{"step": 12150, "total_steps": 72479, "tokens": 6370099200, "loss": 3.4034945964813232, "l_mix": 2.1016600281000137, "l_a": 2.201061725616455, "l_c": 2.1322858333587646, "z_loss": 18.302236795425415, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.34640783071517944, "tokens_per_second": 1377091.991685433, "eta_seconds": 22968.52421114445, "data_wait_seconds": 3.445148468017578e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20765, "cosmopedia_v2": 0.20041, "finemath": 0.08239}}
{"step": 12160, "total_steps": 72479, "tokens": 6375342080, "loss": 4.53828763961792, "l_mix": 2.8058096170425415, "l_a": 2.914068102836609, "l_c": 2.85527640581131, "z_loss": 16.74619174003601, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1659955531358719, "tokens_per_second": 1382782.9407463968, "eta_seconds": 22870.20394895077, "data_wait_seconds": 2.1338462829589844e-05, "mix": {"fineweb_hq": 0.50962, "dclm": 0.20765, "cosmopedia_v2": 0.20041, "finemath": 0.08232}}
{"step": 12170, "total_steps": 72479, "tokens": 6380584960, "loss": 4.750240623950958, "l_mix": 2.9392496049404144, "l_a": 3.0411201417446136, "l_c": 2.9900616109371185, "z_loss": 16.36416792869568, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16807310283184052, "tokens_per_second": 1377625.3182879111, "eta_seconds": 22952.020823264123, "data_wait_seconds": 2.1725893020629883e-05, "mix": {"fineweb_hq": 0.50961, "dclm": 0.20764, "cosmopedia_v2": 0.20041, "finemath": 0.08233}}
{"step": 12180, "total_steps": 72479, "tokens": 6385827840, "loss": 4.638425588607788, "l_mix": 2.8690294921398163, "l_a": 2.9698341488838196, "l_c": 2.922544628381729, "z_loss": 16.823423862457275, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18481189012527466, "tokens_per_second": 1448805.7200748522, "eta_seconds": 21820.76014330387, "data_wait_seconds": 1.800060272216797e-05, "mix": {"fineweb_hq": 0.50961, "dclm": 0.20764, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 12190, "total_steps": 72479, "tokens": 6391070720, "loss": 4.682795941829681, "l_mix": 2.8962822556495667, "l_a": 2.999492973089218, "l_c": 2.9500337839126587, "z_loss": 16.55541968345642, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18866272270679474, "tokens_per_second": 1393776.1773478081, "eta_seconds": 22678.53314306736, "data_wait_seconds": 2.5480985641479492e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20763, "cosmopedia_v2": 0.20041, "finemath": 0.08236}}
{"step": 12200, "total_steps": 72479, "tokens": 6396313600, "loss": 4.606896817684174, "l_mix": 2.850258082151413, "l_a": 2.9481579661369324, "l_c": 2.901825577020645, "z_loss": 16.434978365898132, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21735194325447083, "tokens_per_second": 1462379.0544525946, "eta_seconds": 21611.056487560272, "data_wait_seconds": 1.9848346710205078e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20762, "cosmopedia_v2": 0.20041, "finemath": 0.08238}, "gates": {"gate_a2b/L1_absmean": 0.22688042, "gate_a2b/L1_absmax": 0.63979584, "gate_a2b/L1_rms": 0.25617966, "gate_a2b/L3_absmean": 0.30387339, "gate_a2b/L3_absmax": 0.48582405, "gate_a2b/L3_rms": 0.30923668, "gate_a2b/L5_absmean": 0.49270913, "gate_a2b/L5_absmax": 0.67800289, "gate_a2b/L5_rms": 0.50001347, "gate_c2b/L1_absmean": 0.01465504, "gate_c2b/L1_absmax": 0.5037967, "gate_c2b/L1_rms": 0.058507, "gate_c2b/L3_absmean": 0.21028577, "gate_c2b/L3_absmax": 0.42791858, "gate_c2b/L3_rms": 0.21540935, "gate_c2b/L5_absmean": 0.53828681, "gate_c2b/L5_absmax": 0.75070059, "gate_c2b/L5_rms": 0.55107737, "gate_b2a/L7_absmean": 0.07573248, "gate_b2a/L7_absmax": 0.24679996, "gate_b2a/L7_rms": 0.07939199, "gate_b2a/L11_absmean": 0.12166712, "gate_b2a/L11_absmax": 0.35645106, "gate_b2a/L11_rms": 0.12403312, "gate_b2a/L14_absmean": 0.29660797, "gate_b2a/L14_absmax": 0.46619898, "gate_b2a/L14_rms": 0.29770401, "gate_b2c/L3_absmean": 0.06959464, "gate_b2c/L3_absmax": 0.53389269, "gate_b2c/L3_rms": 0.09083749, "gate_b2c/L5_absmean": 0.10783508, "gate_b2c/L5_absmax": 0.61806035, "gate_b2c/L5_rms": 0.11344237, "gate_b2c/L6_absmean": 0.16116785, "gate_b2c/L6_absmax": 0.44042277, "gate_b2c/L6_rms": 0.16386054}, "streams": {"cos_a2b/round0": 0.374015, "cos_c2b/round0": 0.263657, "cos_b2a/round0": 0.164414, "cos_b2c/round0": 0.122978, "rms_a/round0": 0.929101, "rms_b/round0": 1.788948, "rms_c/round0": 1.280156, "cos_a2b/round1": 0.384259, "cos_c2b/round1": 0.257559, "cos_b2a/round1": 0.211491, "cos_b2c/round1": 0.144826, "rms_a/round1": 1.679471, "rms_b/round1": 3.38278, "rms_c/round1": 2.102405, "cos_a2b/round2": 0.515003, "cos_c2b/round2": 0.458048, "cos_b2a/round2": 0.074152, "cos_b2c/round2": 0.139624, "rms_a/round2": 3.564945, "rms_b/round2": 7.207357, "rms_c/round2": 3.338252}, "alpha_mean": 0.4330316936104879, "alpha_hist": [0.01669, 0.06128, 0.1595, 0.21188, 0.20613, 0.16045, 0.11278, 0.0611, 0.00984, 0.00035], "alpha_by_type": {"special": {"share": 0.00124, "mean": 0.54374, "std": 0.11571}, "whitespace": {"share": 0.03067, "mean": 0.48781, "std": 0.12775}, "digit": {"share": 0.05009, "mean": 0.4348, "std": 0.11362}, "word_start": {"share": 0.52168, "mean": 0.46024, "std": 0.16359}, "word_cont": {"share": 0.28378, "mean": 0.32607, "std": 0.13723}, "punct": {"share": 0.11253, "mean": 0.5597, "std": 0.15852}}, "alpha_p05": 0.17667, "alpha_p50": 0.4225, "alpha_p95": 0.72739}
{"step": 12210, "total_steps": 72479, "tokens": 6401556480, "loss": 4.96272873878479, "l_mix": 3.073912560939789, "l_a": 3.1678108274936676, "l_c": 3.1227089762687683, "z_loss": 16.60192060470581, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17213581502437592, "tokens_per_second": 1435388.8041718917, "eta_seconds": 22013.766151833534, "data_wait_seconds": 1.6391277313232422e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.2077, "cosmopedia_v2": 0.20041, "finemath": 0.08231}}
{"step": 12220, "total_steps": 72479, "tokens": 6406799360, "loss": 5.124770760536194, "l_mix": 3.1751688420772552, "l_a": 3.2648263573646545, "l_c": 3.228469729423523, "z_loss": 16.130752086639404, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16049079596996307, "tokens_per_second": 1359584.1729209556, "eta_seconds": 23237.303891325, "data_wait_seconds": 2.2590160369873047e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20769, "cosmopedia_v2": 0.20041, "finemath": 0.08232}}
{"step": 12230, "total_steps": 72479, "tokens": 6412042240, "loss": 5.005920231342316, "l_mix": 3.1010686457157135, "l_a": 3.190859943628311, "l_c": 3.1531797349452972, "z_loss": 16.39516854286194, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1789102554321289, "tokens_per_second": 1436294.3602574826, "eta_seconds": 21992.586329126356, "data_wait_seconds": 3.007054328918457e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20769, "cosmopedia_v2": 0.20041, "finemath": 0.08234}}
{"step": 12240, "total_steps": 72479, "tokens": 6417285120, "loss": 5.029397904872894, "l_mix": 3.1156605780124664, "l_a": 3.2041282951831818, "l_c": 3.169511139392853, "z_loss": 16.454208374023438, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17949174344539642, "tokens_per_second": 1435451.2067303997, "eta_seconds": 22001.851880383492, "data_wait_seconds": 2.9921531677246094e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 12250, "total_steps": 72479, "tokens": 6422528000, "loss": 4.960464894771576, "l_mix": 3.073576182126999, "l_a": 3.1609511971473694, "l_c": 3.1231409311294556, "z_loss": 16.608911514282227, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17660681903362274, "tokens_per_second": 1367839.9749598983, "eta_seconds": 23085.552791309357, "data_wait_seconds": 2.142786979675293e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08237}}
{"step": 12260, "total_steps": 72479, "tokens": 6427770880, "loss": 3.3013811707496643, "l_mix": 2.0398793667554855, "l_a": 2.1239394545555115, "l_c": 2.0749762654304504, "z_loss": 18.27047300338745, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.27904707193374634, "tokens_per_second": 1419372.0978226843, "eta_seconds": 22243.70841193199, "data_wait_seconds": 1.8209218978881836e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08238}}
{"step": 12270, "total_steps": 72479, "tokens": 6433013760, "loss": 4.492754578590393, "l_mix": 2.7783932089805603, "l_a": 2.8810263574123383, "l_c": 2.8279656171798706, "z_loss": 16.637157440185547, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1428453028202057, "tokens_per_second": 1374324.941447752, "eta_seconds": 22968.99025840759, "data_wait_seconds": 2.0384788513183594e-05, "mix": {"fineweb_hq": 0.50962, "dclm": 0.20766, "cosmopedia_v2": 0.20041, "finemath": 0.08231}}
{"step": 12280, "total_steps": 72479, "tokens": 6438256640, "loss": 4.675567328929901, "l_mix": 2.8932669162750244, "l_a": 2.9917201101779938, "l_c": 2.943770855665207, "z_loss": 16.528770685195923, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.14482112228870392, "tokens_per_second": 1401312.603441776, "eta_seconds": 22522.892632579802, "data_wait_seconds": 2.092123031616211e-05, "mix": {"fineweb_hq": 0.50961, "dclm": 0.20765, "cosmopedia_v2": 0.20041, "finemath": 0.08233}}
{"step": 12290, "total_steps": 72479, "tokens": 6443499520, "loss": 4.72667670249939, "l_mix": 2.925116389989853, "l_a": 3.023439198732376, "l_c": 2.976275712251663, "z_loss": 16.457388401031494, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17479732632637024, "tokens_per_second": 1444881.226008507, "eta_seconds": 21840.11381971836, "data_wait_seconds": 2.765655517578125e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20765, "cosmopedia_v2": 0.20041, "finemath": 0.08234}}
{"step": 12300, "total_steps": 72479, "tokens": 6448742400, "loss": 4.745453476905823, "l_mix": 2.9374575912952423, "l_a": 3.0364846289157867, "l_c": 2.9846825897693634, "z_loss": 16.456071376800537, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1686871498823166, "tokens_per_second": 1359772.4062813525, "eta_seconds": 23203.241517663002, "data_wait_seconds": 1.9311904907226562e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20764, "cosmopedia_v2": 0.20041, "finemath": 0.08236}, "gates": {"gate_a2b/L1_absmean": 0.22909084, "gate_a2b/L1_absmax": 0.65051389, "gate_a2b/L1_rms": 0.25804538, "gate_a2b/L3_absmean": 0.30446482, "gate_a2b/L3_absmax": 0.4903236, "gate_a2b/L3_rms": 0.30984682, "gate_a2b/L5_absmean": 0.49367499, "gate_a2b/L5_absmax": 0.67679673, "gate_a2b/L5_rms": 0.50095296, "gate_c2b/L1_absmean": 0.01544065, "gate_c2b/L1_absmax": 0.5023188, "gate_c2b/L1_rms": 0.05861074, "gate_c2b/L3_absmean": 0.21085964, "gate_c2b/L3_absmax": 0.42768481, "gate_c2b/L3_rms": 0.21600251, "gate_c2b/L5_absmean": 0.53900623, "gate_c2b/L5_absmax": 0.75299996, "gate_c2b/L5_rms": 0.55185944, "gate_b2a/L7_absmean": 0.07600848, "gate_b2a/L7_absmax": 0.24594453, "gate_b2a/L7_rms": 0.07968689, "gate_b2a/L11_absmean": 0.12275721, "gate_b2a/L11_absmax": 0.35573632, "gate_b2a/L11_rms": 0.12503031, "gate_b2a/L14_absmean": 0.29804772, "gate_b2a/L14_absmax": 0.45991415, "gate_b2a/L14_rms": 0.29914188, "gate_b2c/L3_absmean": 0.07025036, "gate_b2c/L3_absmax": 0.53484994, "gate_b2c/L3_rms": 0.0914009, "gate_b2c/L5_absmean": 0.10831121, "gate_b2c/L5_absmax": 0.62294698, "gate_b2c/L5_rms": 0.11394373, "gate_b2c/L6_absmean": 0.16168177, "gate_b2c/L6_absmax": 0.44205523, "gate_b2c/L6_rms": 0.16440022}, "streams": {"cos_a2b/round0": 0.373581, "cos_c2b/round0": 0.261376, "cos_b2a/round0": 0.168825, "cos_b2c/round0": 0.121576, "rms_a/round0": 0.935067, "rms_b/round0": 1.802993, "rms_c/round0": 1.277389, "cos_a2b/round1": 0.401755, "cos_c2b/round1": 0.272568, "cos_b2a/round1": 0.215822, "cos_b2c/round1": 0.147111, "rms_a/round1": 1.702108, "rms_b/round1": 3.329392, "rms_c/round1": 2.094845, "cos_a2b/round2": 0.532862, "cos_c2b/round2": 0.456002, "cos_b2a/round2": 0.079546, "cos_b2c/round2": 0.145029, "rms_a/round2": 3.591535, "rms_b/round2": 7.086291, "rms_c/round2": 3.3035}, "alpha_mean": 0.45623383763399517, "alpha_hist": [0.01752, 0.05559, 0.13068, 0.18958, 0.20178, 0.17377, 0.13445, 0.08119, 0.01509, 0.00037], "alpha_by_type": {"special": {"share": 0.00099, "mean": 0.5195, "std": 0.11194}, "whitespace": {"share": 0.02036, "mean": 0.48413, "std": 0.11421}, "digit": {"share": 0.01837, "mean": 0.4466, "std": 0.11454}, "word_start": {"share": 0.58115, "mean": 0.48671, "std": 0.16217}, "word_cont": {"share": 0.26738, "mean": 0.33342, "std": 0.14411}, "punct": {"share": 0.11176, "mean": 0.58753, "std": 0.1651}}, "alpha_p05": 0.18865, "alpha_p50": 0.44879, "alpha_p95": 0.75274}
{"step": 12310, "total_steps": 72479, "tokens": 6453985280, "loss": 4.767132043838501, "l_mix": 2.950808882713318, "l_a": 3.0477954149246216, "l_c": 3.0012601912021637, "z_loss": 16.06358027458191, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19885030388832092, "tokens_per_second": 1408812.8193512328, "eta_seconds": 22391.82114095688, "data_wait_seconds": 2.6941299438476562e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20764, "cosmopedia_v2": 0.20041, "finemath": 0.08237}}
{"step": 12320, "total_steps": 72479, "tokens": 6459228160, "loss": 3.3432829678058624, "l_mix": 2.0660519748926163, "l_a": 2.1422090232372284, "l_c": 2.1092844754457474, "z_loss": 17.82863712310791, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2764757573604584, "tokens_per_second": 1409235.525402179, "eta_seconds": 22381.384249448776, "data_wait_seconds": 3.30507755279541e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20763, "cosmopedia_v2": 0.20041, "finemath": 0.08239}}
{"step": 12330, "total_steps": 72479, "tokens": 6464471040, "loss": 4.989136040210724, "l_mix": 3.0912832617759705, "l_a": 3.183363139629364, "l_c": 3.137485295534134, "z_loss": 15.981057167053223, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2299703061580658, "tokens_per_second": 1452825.2067938382, "eta_seconds": 21706.25809941292, "data_wait_seconds": 1.7881393432617188e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.2077, "cosmopedia_v2": 0.20041, "finemath": 0.08232}}
{"step": 12340, "total_steps": 72479, "tokens": 6469713920, "loss": 5.087246894836426, "l_mix": 3.1522383987903595, "l_a": 3.2409943342208862, "l_c": 3.2036893367767334, "z_loss": 16.031452894210815, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17098647356033325, "tokens_per_second": 1444825.9749183755, "eta_seconds": 21822.80536158085, "data_wait_seconds": 1.627206802368164e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.2077, "cosmopedia_v2": 0.20041, "finemath": 0.08233}}
{"step": 12350, "total_steps": 72479, "tokens": 6474956800, "loss": 4.833872199058533, "l_mix": 2.9936591386795044, "l_a": 3.085211843252182, "l_c": 3.043317884206772, "z_loss": 16.540396213531494, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15325628221035004, "tokens_per_second": 1444156.2729682615, "eta_seconds": 21829.29489147663, "data_wait_seconds": 2.7567148208618164e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20769, "cosmopedia_v2": 0.2004, "finemath": 0.08235}}
{"step": 12360, "total_steps": 72479, "tokens": 6480199680, "loss": 5.0517847537994385, "l_mix": 3.1292612552642822, "l_a": 3.2220072746276855, "l_c": 3.180962949991226, "z_loss": 16.323893547058105, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.151945561170578, "tokens_per_second": 1426183.3438541617, "eta_seconds": 22100.714054632186, "data_wait_seconds": 2.4825334548950195e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20769, "cosmopedia_v2": 0.2004, "finemath": 0.08236}}
{"step": 12370, "total_steps": 72479, "tokens": 6485442560, "loss": 5.03278249502182, "l_mix": 3.1178510189056396, "l_a": 3.2073669135570526, "l_c": 3.1702679693698883, "z_loss": 16.40877652168274, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1903984695672989, "tokens_per_second": 1393852.3303880617, "eta_seconds": 22609.58833653927, "data_wait_seconds": 3.358721733093262e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20768, "cosmopedia_v2": 0.2004, "finemath": 0.08238}}
{"step": 12380, "total_steps": 72479, "tokens": 6490685440, "loss": 4.694715321063995, "l_mix": 2.9046445786952972, "l_a": 3.0051496028900146, "l_c": 2.9563251435756683, "z_loss": 16.281121373176575, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1800391525030136, "tokens_per_second": 1373664.7548537203, "eta_seconds": 22938.045400571824, "data_wait_seconds": 2.6047229766845703e-05, "mix": {"fineweb_hq": 0.50961, "dclm": 0.20767, "cosmopedia_v2": 0.2004, "finemath": 0.08231}}
{"step": 12390, "total_steps": 72479, "tokens": 6495928320, "loss": 4.708315551280975, "l_mix": 2.9134195148944855, "l_a": 3.016128897666931, "l_c": 2.96133890748024, "z_loss": 16.55636763572693, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1490405946969986, "tokens_per_second": 1198501.5649038416, "eta_seconds": 26286.108049035072, "data_wait_seconds": 2.2917985916137695e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20767, "cosmopedia_v2": 0.2004, "finemath": 0.08232}}
{"step": 12400, "total_steps": 72479, "tokens": 6501171200, "loss": 4.616450011730194, "l_mix": 2.8575611114501953, "l_a": 2.952648550271988, "l_c": 2.904875159263611, "z_loss": 16.317918062210083, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16749602556228638, "tokens_per_second": 1366213.1758044278, "eta_seconds": 23055.478683590893, "data_wait_seconds": 2.0802021026611328e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20766, "cosmopedia_v2": 0.2004, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.23084311, "gate_a2b/L1_absmax": 0.64566523, "gate_a2b/L1_rms": 0.25965297, "gate_a2b/L3_absmean": 0.30537406, "gate_a2b/L3_absmax": 0.49254826, "gate_a2b/L3_rms": 0.31065583, "gate_a2b/L5_absmean": 0.49438697, "gate_a2b/L5_absmax": 0.67718321, "gate_a2b/L5_rms": 0.50155801, "gate_c2b/L1_absmean": 0.01511379, "gate_c2b/L1_absmax": 0.50143945, "gate_c2b/L1_rms": 0.05855309, "gate_c2b/L3_absmean": 0.21062227, "gate_c2b/L3_absmax": 0.42750153, "gate_c2b/L3_rms": 0.21565585, "gate_c2b/L5_absmean": 0.53870314, "gate_c2b/L5_absmax": 0.7491709, "gate_c2b/L5_rms": 0.55147123, "gate_b2a/L7_absmean": 0.07603787, "gate_b2a/L7_absmax": 0.24780785, "gate_b2a/L7_rms": 0.07969511, "gate_b2a/L11_absmean": 0.1220771, "gate_b2a/L11_absmax": 0.35876605, "gate_b2a/L11_rms": 0.12440106, "gate_b2a/L14_absmean": 0.29785535, "gate_b2a/L14_absmax": 0.46296901, "gate_b2a/L14_rms": 0.29896691, "gate_b2c/L3_absmean": 0.07087478, "gate_b2c/L3_absmax": 0.53563344, "gate_b2c/L3_rms": 0.09189391, "gate_b2c/L5_absmean": 0.10863896, "gate_b2c/L5_absmax": 0.62985682, "gate_b2c/L5_rms": 0.11438508, "gate_b2c/L6_absmean": 0.16228637, "gate_b2c/L6_absmax": 0.44059205, "gate_b2c/L6_rms": 0.1650022}, "streams": {"cos_a2b/round0": 0.392341, "cos_c2b/round0": 0.274587, "cos_b2a/round0": 0.171486, "cos_b2c/round0": 0.125577, "rms_a/round0": 0.9308, "rms_b/round0": 1.765116, "rms_c/round0": 1.296647, "cos_a2b/round1": 0.399408, "cos_c2b/round1": 0.257636, "cos_b2a/round1": 0.224579, "cos_b2c/round1": 0.12901, "rms_a/round1": 1.646269, "rms_b/round1": 3.301316, "rms_c/round1": 2.083584, "cos_a2b/round2": 0.495321, "cos_c2b/round2": 0.483024, "cos_b2a/round2": 0.103688, "cos_b2c/round2": 0.130176, "rms_a/round2": 3.492062, "rms_b/round2": 7.333945, "rms_c/round2": 3.298015}, "alpha_mean": 0.42768608622309046, "alpha_hist": [0.01056, 0.06502, 0.17258, 0.22194, 0.20409, 0.14885, 0.10863, 0.06071, 0.00757, 6e-05], "alpha_by_type": {"whitespace": {"share": 0.0327, "mean": 0.51089, "std": 0.09364}, "digit": {"share": 0.0959, "mean": 0.40843, "std": 0.14834}, "word_start": {"share": 0.50888, "mean": 0.44283, "std": 0.1628}, "word_cont": {"share": 0.24692, "mean": 0.32591, "std": 0.12461}, "punct": {"share": 0.1156, "mean": 0.57083, "std": 0.15471}}, "alpha_p05": 0.18507, "alpha_p50": 0.41182, "alpha_p95": 0.71721}
{"step": 12410, "total_steps": 72479, "tokens": 6506414080, "loss": 4.7301440834999084, "l_mix": 2.9269660115242004, "l_a": 3.0285145342350006, "l_c": 2.976578027009964, "z_loss": 16.502086400985718, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1501847356557846, "tokens_per_second": 1287235.2601279437, "eta_seconds": 24465.967370152473, "data_wait_seconds": 2.703070640563965e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20766, "cosmopedia_v2": 0.2004, "finemath": 0.08235}}
{"step": 12420, "total_steps": 72479, "tokens": 6511656960, "loss": 4.768436968326569, "l_mix": 2.949887692928314, "l_a": 3.0526562333106995, "l_c": 3.003640502691269, "z_loss": 16.60327458381653, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21428105235099792, "tokens_per_second": 1345516.3516150094, "eta_seconds": 23402.326515173914, "data_wait_seconds": 1.6957521438598633e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.2004, "finemath": 0.08237}}
{"step": 12430, "total_steps": 72479, "tokens": 6516899840, "loss": 3.51114958524704, "l_mix": 2.1698913127183914, "l_a": 2.2560744285583496, "l_c": 2.208857476711273, "z_loss": 17.7868549823761, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.3027056157588959, "tokens_per_second": 1416531.7983882371, "eta_seconds": 22225.388902544975, "data_wait_seconds": 2.41696834564209e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20764, "cosmopedia_v2": 0.2004, "finemath": 0.08238}}
{"step": 12440, "total_steps": 72479, "tokens": 6522142720, "loss": 3.5317427814006805, "l_mix": 2.1703631579875946, "l_a": 2.2995811998844147, "l_c": 2.233167141675949, "z_loss": 15.55055844783783, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19198453426361084, "tokens_per_second": 1429332.6345246804, "eta_seconds": 22022.67440879345, "data_wait_seconds": 3.516674041748047e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20764, "cosmopedia_v2": 0.20048, "finemath": 0.08232}}
{"step": 12450, "total_steps": 72479, "tokens": 6527385600, "loss": 5.082468211650848, "l_mix": 3.1495690643787384, "l_a": 3.238319009542465, "l_c": 3.1992179453372955, "z_loss": 16.3783917427063, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1606467217206955, "tokens_per_second": 1426268.5373560037, "eta_seconds": 22066.310465168954, "data_wait_seconds": 2.5898218154907227e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20771, "cosmopedia_v2": 0.2004, "finemath": 0.08233}}
{"step": 12460, "total_steps": 72479, "tokens": 6532628480, "loss": 5.075918674468994, "l_mix": 3.144798308610916, "l_a": 3.234047293663025, "l_c": 3.197688549757004, "z_loss": 15.99591338634491, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16875088214874268, "tokens_per_second": 1469028.382979599, "eta_seconds": 21420.444857692717, "data_wait_seconds": 1.5854835510253906e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.2077, "cosmopedia_v2": 0.2004, "finemath": 0.08234}}
{"step": 12470, "total_steps": 72479, "tokens": 6537871360, "loss": 4.981937050819397, "l_mix": 3.086186498403549, "l_a": 3.174725115299225, "l_c": 3.1389972269535065, "z_loss": 16.337239503860474, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1572405993938446, "tokens_per_second": 1431701.810951472, "eta_seconds": 21975.24536976814, "data_wait_seconds": 1.5974044799804688e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.2077, "cosmopedia_v2": 0.2004, "finemath": 0.08236}}
{"step": 12480, "total_steps": 72479, "tokens": 6543114240, "loss": 4.920750379562378, "l_mix": 3.0475584268569946, "l_a": 3.1379033625125885, "l_c": 3.100398302078247, "z_loss": 17.01344656944275, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18876171112060547, "tokens_per_second": 1473659.868660029, "eta_seconds": 21346.00824856758, "data_wait_seconds": 1.7404556274414062e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20769, "cosmopedia_v2": 0.2004, "finemath": 0.08237}}
{"step": 12490, "total_steps": 72479, "tokens": 6548357120, "loss": 4.670332252979279, "l_mix": 2.8896729946136475, "l_a": 2.986800342798233, "l_c": 2.943275809288025, "z_loss": 16.363872051239014, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18372811377048492, "tokens_per_second": 1477824.1064850052, "eta_seconds": 21282.31140227318, "data_wait_seconds": 1.6927719116210938e-05, "mix": {"fineweb_hq": 0.50961, "dclm": 0.20769, "cosmopedia_v2": 0.2004, "finemath": 0.08231}}
{"step": 12500, "total_steps": 72479, "tokens": 6553600000, "loss": 4.6372429728508, "l_mix": 2.868214040994644, "l_a": 2.972070097923279, "l_c": 2.9192156195640564, "z_loss": 16.431336641311646, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1881178617477417, "tokens_per_second": 1409243.3824431407, "eta_seconds": 22314.293147492408, "data_wait_seconds": 2.390146255493164e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20768, "cosmopedia_v2": 0.2004, "finemath": 0.08232}, "gates": {"gate_a2b/L1_absmean": 0.23256712, "gate_a2b/L1_absmax": 0.64913493, "gate_a2b/L1_rms": 0.26132733, "gate_a2b/L3_absmean": 0.30661702, "gate_a2b/L3_absmax": 0.49580652, "gate_a2b/L3_rms": 0.31182075, "gate_a2b/L5_absmean": 0.49604231, "gate_a2b/L5_absmax": 0.67857569, "gate_a2b/L5_rms": 0.50315559, "gate_c2b/L1_absmean": 0.01479441, "gate_c2b/L1_absmax": 0.50079328, "gate_c2b/L1_rms": 0.05830886, "gate_c2b/L3_absmean": 0.21045284, "gate_c2b/L3_absmax": 0.43209204, "gate_c2b/L3_rms": 0.2154765, "gate_c2b/L5_absmean": 0.53899366, "gate_c2b/L5_absmax": 0.74243647, "gate_c2b/L5_rms": 0.55178392, "gate_b2a/L7_absmean": 0.07584058, "gate_b2a/L7_absmax": 0.24785814, "gate_b2a/L7_rms": 0.07937197, "gate_b2a/L11_absmean": 0.12227228, "gate_b2a/L11_absmax": 0.35836008, "gate_b2a/L11_rms": 0.12462483, "gate_b2a/L14_absmean": 0.2977286, "gate_b2a/L14_absmax": 0.46128851, "gate_b2a/L14_rms": 0.29883996, "gate_b2c/L3_absmean": 0.07165308, "gate_b2c/L3_absmax": 0.53630769, "gate_b2c/L3_rms": 0.09276457, "gate_b2c/L5_absmean": 0.10887738, "gate_b2c/L5_absmax": 0.63138831, "gate_b2c/L5_rms": 0.11461324, "gate_b2c/L6_absmean": 0.16288193, "gate_b2c/L6_absmax": 0.44238666, "gate_b2c/L6_rms": 0.16565721}, "streams": {"cos_a2b/round0": 0.381531, "cos_c2b/round0": 0.261664, "cos_b2a/round0": 0.167267, "cos_b2c/round0": 0.122086, "rms_a/round0": 0.923214, "rms_b/round0": 1.769549, "rms_c/round0": 1.275255, "cos_a2b/round1": 0.395338, "cos_c2b/round1": 0.264843, "cos_b2a/round1": 0.221023, "cos_b2c/round1": 0.141784, "rms_a/round1": 1.663924, "rms_b/round1": 3.322812, "rms_c/round1": 2.08496, "cos_a2b/round2": 0.516706, "cos_c2b/round2": 0.46564, "cos_b2a/round2": 0.086263, "cos_b2c/round2": 0.135131, "rms_a/round2": 3.551539, "rms_b/round2": 7.176592, "rms_c/round2": 3.292944}, "alpha_mean": 0.4427190469886355, "alpha_hist": [0.01485, 0.06566, 0.15633, 0.19792, 0.18835, 0.16498, 0.1245, 0.07054, 0.01643, 0.00044], "alpha_by_type": {"special": {"share": 0.00093, "mean": 0.55561, "std": 0.12204}, "whitespace": {"share": 0.03238, "mean": 0.52692, "std": 0.13357}, "digit": {"share": 0.0486, "mean": 0.5263, "std": 0.11287}, "word_start": {"share": 0.53607, "mean": 0.46229, "std": 0.16543}, "word_cont": {"share": 0.28032, "mean": 0.32696, "std": 0.14324}, "punct": {"share": 0.1017, "mean": 0.59085, "std": 0.16948}}, "alpha_p05": 0.17357, "alpha_p50": 0.44119, "alpha_p95": 0.75672}
{"step": 12510, "total_steps": 72479, "tokens": 6558842880, "loss": 4.707796931266785, "l_mix": 2.9132376611232758, "l_a": 3.011697381734848, "l_c": 2.964631348848343, "z_loss": 16.60565996170044, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15959729254245758, "tokens_per_second": 1340360.7900718532, "eta_seconds": 23457.137290859224, "data_wait_seconds": 2.703070640563965e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20767, "cosmopedia_v2": 0.2004, "finemath": 0.08233}}
{"step": 12520, "total_steps": 72479, "tokens": 6564085760, "loss": 4.684935986995697, "l_mix": 2.8979999721050262, "l_a": 3.000780165195465, "l_c": 2.950135588645935, "z_loss": 16.61231756210327, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20563584566116333, "tokens_per_second": 1467163.90179273, "eta_seconds": 21426.22521831989, "data_wait_seconds": 1.576542854309082e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.2004, "finemath": 0.08235}}
{"step": 12530, "total_steps": 72479, "tokens": 6569328640, "loss": 4.640364944934845, "l_mix": 2.8710907101631165, "l_a": 2.972076177597046, "l_c": 2.9201036989688873, "z_loss": 16.201453804969788, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15394550561904907, "tokens_per_second": 1344984.475380867, "eta_seconds": 23368.701934719084, "data_wait_seconds": 2.5480985641479492e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.2004, "finemath": 0.08236}}
{"step": 12540, "total_steps": 72479, "tokens": 6574571520, "loss": 3.2116423547267914, "l_mix": 1.9847961068153381, "l_a": 2.0622499883174896, "l_c": 2.0212587267160416, "z_loss": 17.935503244400024, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19900557398796082, "tokens_per_second": 1433709.3486262453, "eta_seconds": 21918.876697087286, "data_wait_seconds": 2.5033950805664062e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.2004, "finemath": 0.08238}}
{"step": 12550, "total_steps": 72479, "tokens": 6579814400, "loss": 3.5605377554893494, "l_mix": 2.1880014836788177, "l_a": 2.3166973292827606, "l_c": 2.253223031759262, "z_loss": 15.60137403011322, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19891729950904846, "tokens_per_second": 1364901.2530467885, "eta_seconds": 23020.021032190325, "data_wait_seconds": 3.662705421447754e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20048, "finemath": 0.08231}}
{"step": 12560, "total_steps": 72479, "tokens": 6585057280, "loss": 3.5885148644447327, "l_mix": 2.205668419599533, "l_a": 2.3350091576576233, "l_c": 2.269258588552475, "z_loss": 15.661304116249084, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2163931429386139, "tokens_per_second": 1462644.303600774, "eta_seconds": 21478.094567942622, "data_wait_seconds": 2.5331974029541016e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20764, "cosmopedia_v2": 0.20048, "finemath": 0.08232}}
{"step": 12570, "total_steps": 72479, "tokens": 6590300160, "loss": 3.529144912958145, "l_mix": 2.1688115298748016, "l_a": 2.2950240671634674, "l_c": 2.2341721951961517, "z_loss": 15.744526505470276, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2072148323059082, "tokens_per_second": 1312731.2063245396, "eta_seconds": 23926.88590068817, "data_wait_seconds": 1.9758939743041992e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20764, "cosmopedia_v2": 0.20048, "finemath": 0.08234}}
{"step": 12580, "total_steps": 72479, "tokens": 6595543040, "loss": 3.54906165599823, "l_mix": 2.1819292306900024, "l_a": 2.3073295950889587, "l_c": 2.2445991337299347, "z_loss": 15.537839889526367, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22623112797737122, "tokens_per_second": 1435162.2891320859, "eta_seconds": 21882.07365105152, "data_wait_seconds": 1.7523765563964844e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20763, "cosmopedia_v2": 0.20048, "finemath": 0.08235}}
{"step": 12590, "total_steps": 72479, "tokens": 6600785920, "loss": 4.697317540645599, "l_mix": 2.9072678685188293, "l_a": 3.003975421190262, "l_c": 2.957344800233841, "z_loss": 16.533552885055542, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16442488133907318, "tokens_per_second": 1330112.3395025337, "eta_seconds": 23606.339930462836, "data_wait_seconds": 2.250075340270996e-05, "mix": {"fineweb_hq": 0.50961, "dclm": 0.20763, "cosmopedia_v2": 0.2004, "finemath": 0.08237}}
{"step": 12600, "total_steps": 72479, "tokens": 6606028800, "loss": 4.793183505535126, "l_mix": 2.965747445821762, "l_a": 3.0634235739707947, "l_c": 3.0225991904735565, "z_loss": 16.29202365875244, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.26045429706573486, "tokens_per_second": 1330741.3836780335, "eta_seconds": 23591.241346406936, "data_wait_seconds": 2.631545066833496e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20762, "cosmopedia_v2": 0.2004, "finemath": 0.08238}, "gates": {"gate_a2b/L1_absmean": 0.23464261, "gate_a2b/L1_absmax": 0.64766008, "gate_a2b/L1_rms": 0.26295307, "gate_a2b/L3_absmean": 0.30726811, "gate_a2b/L3_absmax": 0.49987662, "gate_a2b/L3_rms": 0.31251314, "gate_a2b/L5_absmean": 0.49687424, "gate_a2b/L5_absmax": 0.67793739, "gate_a2b/L5_rms": 0.50400448, "gate_c2b/L1_absmean": 0.01496181, "gate_c2b/L1_absmax": 0.50236017, "gate_c2b/L1_rms": 0.05851612, "gate_c2b/L3_absmean": 0.21050893, "gate_c2b/L3_absmax": 0.43337345, "gate_c2b/L3_rms": 0.21548675, "gate_c2b/L5_absmean": 0.5389744, "gate_c2b/L5_absmax": 0.74896336, "gate_c2b/L5_rms": 0.55173504, "gate_b2a/L7_absmean": 0.07612656, "gate_b2a/L7_absmax": 0.24843599, "gate_b2a/L7_rms": 0.07967792, "gate_b2a/L11_absmean": 0.12287246, "gate_b2a/L11_absmax": 0.36163929, "gate_b2a/L11_rms": 0.12517907, "gate_b2a/L14_absmean": 0.29797649, "gate_b2a/L14_absmax": 0.46287861, "gate_b2a/L14_rms": 0.29909787, "gate_b2c/L3_absmean": 0.07222756, "gate_b2c/L3_absmax": 0.54001468, "gate_b2c/L3_rms": 0.09335987, "gate_b2c/L5_absmean": 0.1093256, "gate_b2c/L5_absmax": 0.63398784, "gate_b2c/L5_rms": 0.11504827, "gate_b2c/L6_absmean": 0.16355431, "gate_b2c/L6_absmax": 0.44491214, "gate_b2c/L6_rms": 0.16636845}, "streams": {"cos_a2b/round0": 0.378142, "cos_c2b/round0": 0.269418, "cos_b2a/round0": 0.166424, "cos_b2c/round0": 0.123406, "rms_a/round0": 0.919094, "rms_b/round0": 1.759821, "rms_c/round0": 1.267596, "cos_a2b/round1": 0.39434, "cos_c2b/round1": 0.263312, "cos_b2a/round1": 0.216226, "cos_b2c/round1": 0.143365, "rms_a/round1": 1.655474, "rms_b/round1": 3.299231, "rms_c/round1": 2.0694, "cos_a2b/round2": 0.519269, "cos_c2b/round2": 0.469614, "cos_b2a/round2": 0.072751, "cos_b2c/round2": 0.137464, "rms_a/round2": 3.512758, "rms_b/round2": 7.105286, "rms_c/round2": 3.306171}, "alpha_mean": 0.4435415045873867, "alpha_hist": [0.01476, 0.07027, 0.15688, 0.19476, 0.18201, 0.15759, 0.13115, 0.077, 0.01514, 0.00046], "alpha_by_type": {"special": {"share": 0.00116, "mean": 0.49833, "std": 0.1285}, "whitespace": {"share": 0.0262, "mean": 0.47145, "std": 0.1394}, "digit": {"share": 0.04675, "mean": 0.49159, "std": 0.11859}, "word_start": {"share": 0.57533, "mean": 0.46256, "std": 0.1738}, "word_cont": {"share": 0.2523, "mean": 0.32885, "std": 0.1424}, "punct": {"share": 0.09825, "mean": 0.59578, "std": 0.16405}}, "alpha_p05": 0.16132, "alpha_p50": 0.43063, "alpha_p95": 0.73564}
{"step": 12610, "total_steps": 72479, "tokens": 6611271680, "loss": 4.680850803852081, "l_mix": 2.8958489298820496, "l_a": 2.9981052577495575, "l_c": 2.9463759064674377, "z_loss": 16.574569702148438, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17970414459705353, "tokens_per_second": 1460969.8009986177, "eta_seconds": 21484.768713593483, "data_wait_seconds": 1.5854835510253906e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20769, "cosmopedia_v2": 0.2004, "finemath": 0.08232}}
{"step": 12620, "total_steps": 72479, "tokens": 6616514560, "loss": 4.807486295700073, "l_mix": 2.9747602939605713, "l_a": 3.0757346749305725, "l_c": 3.027906209230423, "z_loss": 16.33640766143799, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1643233746290207, "tokens_per_second": 1449633.5811694919, "eta_seconds": 21649.164174771307, "data_wait_seconds": 2.0563602447509766e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20769, "cosmopedia_v2": 0.2004, "finemath": 0.08233}}
{"step": 12630, "total_steps": 72479, "tokens": 6621757440, "loss": 4.699360966682434, "l_mix": 2.9066454470157623, "l_a": 3.0085807144641876, "l_c": 2.961645931005478, "z_loss": 16.474726915359497, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18536552786827087, "tokens_per_second": 1433176.3691932776, "eta_seconds": 21894.104024100303, "data_wait_seconds": 1.6391277313232422e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20768, "cosmopedia_v2": 0.2004, "finemath": 0.08234}}
{"step": 12640, "total_steps": 72479, "tokens": 6627000320, "loss": 4.857459664344788, "l_mix": 3.006218910217285, "l_a": 3.1072813868522644, "l_c": 3.0578958988189697, "z_loss": 16.875312566757202, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18173399567604065, "tokens_per_second": 1447515.0073938337, "eta_seconds": 21673.605780768394, "data_wait_seconds": 1.8298625946044922e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.2004, "finemath": 0.08236}}
{"step": 12650, "total_steps": 72479, "tokens": 6632243200, "loss": 3.785920023918152, "l_mix": 2.340878427028656, "l_a": 2.430193543434143, "l_c": 2.380674719810486, "z_loss": 17.810437202453613, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.4903238117694855, "tokens_per_second": 1232818.7890764105, "eta_seconds": 25443.825994491577, "data_wait_seconds": 2.4259090423583984e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.2004, "finemath": 0.08237}}
{"step": 12660, "total_steps": 72479, "tokens": 6637486080, "loss": 3.593781739473343, "l_mix": 2.2089993357658386, "l_a": 2.3406906723976135, "l_c": 2.2703618109226227, "z_loss": 14.665800333023071, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2687758505344391, "tokens_per_second": 1418338.252456766, "eta_seconds": 22112.062350201606, "data_wait_seconds": 2.92360782623291e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20047, "finemath": 0.08231}}
{"step": 12670, "total_steps": 72479, "tokens": 6642728960, "loss": 3.569004535675049, "l_mix": 2.193053215742111, "l_a": 2.326241195201874, "l_c": 2.2550970911979675, "z_loss": 15.49855923652649, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18652354180812836, "tokens_per_second": 1406159.8714301325, "eta_seconds": 22299.84060070515, "data_wait_seconds": 2.5391578674316406e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20766, "cosmopedia_v2": 0.20047, "finemath": 0.08232}}
{"step": 12680, "total_steps": 72479, "tokens": 6647971840, "loss": 3.5662912726402283, "l_mix": 2.1914689540863037, "l_a": 2.3233164846897125, "l_c": 2.2541919946670532, "z_loss": 15.697225451469421, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21619141101837158, "tokens_per_second": 1324292.3816065623, "eta_seconds": 23674.45327591896, "data_wait_seconds": 3.662705421447754e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20765, "cosmopedia_v2": 0.20047, "finemath": 0.08233}}
{"step": 12690, "total_steps": 72479, "tokens": 6653214720, "loss": 3.501952588558197, "l_mix": 2.1524103581905365, "l_a": 2.277202785015106, "l_c": 2.2161026895046234, "z_loss": 15.5053790807724, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20274418592453003, "tokens_per_second": 1474719.494788843, "eta_seconds": 21256.011968898776, "data_wait_seconds": 1.5437602996826172e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20764, "cosmopedia_v2": 0.20047, "finemath": 0.08235}}
{"step": 12700, "total_steps": 72479, "tokens": 6658457600, "loss": 4.622182190418243, "l_mix": 2.859364241361618, "l_a": 2.9615708887577057, "l_c": 2.9089541137218475, "z_loss": 16.603628396987915, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1740415096282959, "tokens_per_second": 1456850.264441033, "eta_seconds": 21513.132211995126, "data_wait_seconds": 2.2590160369873047e-05, "mix": {"fineweb_hq": 0.50961, "dclm": 0.20764, "cosmopedia_v2": 0.20039, "finemath": 0.08236}, "gates": {"gate_a2b/L1_absmean": 0.23547426, "gate_a2b/L1_absmax": 0.64922154, "gate_a2b/L1_rms": 0.26345697, "gate_a2b/L3_absmean": 0.30675653, "gate_a2b/L3_absmax": 0.49959624, "gate_a2b/L3_rms": 0.31198865, "gate_a2b/L5_absmean": 0.49658605, "gate_a2b/L5_absmax": 0.67380261, "gate_a2b/L5_rms": 0.50363725, "gate_c2b/L1_absmean": 0.0152456, "gate_c2b/L1_absmax": 0.50086564, "gate_c2b/L1_rms": 0.05838794, "gate_c2b/L3_absmean": 0.21044903, "gate_c2b/L3_absmax": 0.42674592, "gate_c2b/L3_rms": 0.21544285, "gate_c2b/L5_absmean": 0.53747332, "gate_c2b/L5_absmax": 0.7534734, "gate_c2b/L5_rms": 0.55010539, "gate_b2a/L7_absmean": 0.0763546, "gate_b2a/L7_absmax": 0.2492567, "gate_b2a/L7_rms": 0.07996653, "gate_b2a/L11_absmean": 0.12270114, "gate_b2a/L11_absmax": 0.36379999, "gate_b2a/L11_rms": 0.12506168, "gate_b2a/L14_absmean": 0.29801956, "gate_b2a/L14_absmax": 0.46201378, "gate_b2a/L14_rms": 0.29913977, "gate_b2c/L3_absmean": 0.07215001, "gate_b2c/L3_absmax": 0.54228747, "gate_b2c/L3_rms": 0.09323006, "gate_b2c/L5_absmean": 0.1094779, "gate_b2c/L5_absmax": 0.63375157, "gate_b2c/L5_rms": 0.11523646, "gate_b2c/L6_absmean": 0.16308782, "gate_b2c/L6_absmax": 0.44451305, "gate_b2c/L6_rms": 0.1658681}, "streams": {"cos_a2b/round0": 0.374506, "cos_c2b/round0": 0.262511, "cos_b2a/round0": 0.167448, "cos_b2c/round0": 0.127907, "rms_a/round0": 0.922465, "rms_b/round0": 1.783216, "rms_c/round0": 1.265952, "cos_a2b/round1": 0.408777, "cos_c2b/round1": 0.2769, "cos_b2a/round1": 0.220727, "cos_b2c/round1": 0.15128, "rms_a/round1": 1.670854, "rms_b/round1": 3.264134, "rms_c/round1": 2.070385, "cos_a2b/round2": 0.543423, "cos_c2b/round2": 0.462148, "cos_b2a/round2": 0.083475, "cos_b2c/round2": 0.146365, "rms_a/round2": 3.603948, "rms_b/round2": 7.006382, "rms_c/round2": 3.277726}, "alpha_mean": 0.491136120845141, "alpha_hist": [0.00587, 0.03761, 0.11211, 0.15422, 0.18054, 0.22249, 0.17563, 0.0921, 0.01877, 0.00066], "alpha_by_type": {"special": {"share": 0.00056, "mean": 0.58636, "std": 0.14468}, "whitespace": {"share": 0.02315, "mean": 0.56904, "std": 0.10558}, "digit": {"share": 0.02571, "mean": 0.47556, "std": 0.11802}, "word_start": {"share": 0.60657, "mean": 0.51225, "std": 0.15284}, "word_cont": {"share": 0.23555, "mean": 0.36614, "std": 0.14779}, "punct": {"share": 0.10846, "mean": 0.63106, "std": 0.14516}}, "alpha_p05": 0.22035, "alpha_p50": 0.50909, "alpha_p95": 0.75958}
{"step": 12710, "total_steps": 72479, "tokens": 6663700480, "loss": 4.78948849439621, "l_mix": 2.963695079088211, "l_a": 3.066423088312149, "l_c": 3.0139997005462646, "z_loss": 16.664690494537354, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.192729189991951, "tokens_per_second": 1462913.2498636725, "eta_seconds": 21420.388033890722, "data_wait_seconds": 1.537799835205078e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20763, "cosmopedia_v2": 0.20039, "finemath": 0.08238}}
{"step": 12720, "total_steps": 72479, "tokens": 6668943360, "loss": 4.750300765037537, "l_mix": 2.9400616884231567, "l_a": 3.0407887995243073, "l_c": 2.9877820909023285, "z_loss": 16.676903247833252, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1999678611755371, "tokens_per_second": 1471003.480369469, "eta_seconds": 21299.0159507513, "data_wait_seconds": 1.5616416931152344e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20763, "cosmopedia_v2": 0.20039, "finemath": 0.08239}}
{"step": 12730, "total_steps": 72479, "tokens": 6674186240, "loss": 5.075616717338562, "l_mix": 3.144774705171585, "l_a": 3.23367977142334, "l_c": 3.1969847977161407, "z_loss": 16.425410985946655, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15426814556121826, "tokens_per_second": 1427036.9410154568, "eta_seconds": 21951.55767285824, "data_wait_seconds": 5.1647424697875977e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.2077, "cosmopedia_v2": 0.20039, "finemath": 0.08233}}
{"step": 12740, "total_steps": 72479, "tokens": 6679429120, "loss": 5.101153373718262, "l_mix": 3.1595907509326935, "l_a": 3.2503321170806885, "l_c": 3.216076523065567, "z_loss": 16.398696660995483, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1776048243045807, "tokens_per_second": 1367775.0600157054, "eta_seconds": 22898.82433712482, "data_wait_seconds": 2.35140323638916e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20769, "cosmopedia_v2": 0.20039, "finemath": 0.08234}}
{"step": 12750, "total_steps": 72479, "tokens": 6684672000, "loss": 5.088465631008148, "l_mix": 3.1535023152828217, "l_a": 3.2417357563972473, "l_c": 3.2027153074741364, "z_loss": 16.279923677444458, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1558811068534851, "tokens_per_second": 1461550.1756048254, "eta_seconds": 21426.016345310214, "data_wait_seconds": 2.092123031616211e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20769, "cosmopedia_v2": 0.20039, "finemath": 0.08235}}
{"step": 12760, "total_steps": 72479, "tokens": 6689914880, "loss": 5.093192100524902, "l_mix": 3.1561547815799713, "l_a": 3.2440666556358337, "l_c": 3.2072757184505463, "z_loss": 16.344226121902466, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17062228918075562, "tokens_per_second": 1355283.0925026704, "eta_seconds": 23102.15131082535, "data_wait_seconds": 2.2858381271362305e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.20039, "finemath": 0.08237}}
{"step": 12770, "total_steps": 72479, "tokens": 6695157760, "loss": 3.545640617609024, "l_mix": 2.179244041442871, "l_a": 2.3070259988307953, "l_c": 2.242453098297119, "z_loss": 15.527385830879211, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20208150148391724, "tokens_per_second": 1361885.3840661962, "eta_seconds": 22986.30454387665, "data_wait_seconds": 2.3990869522094727e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20047, "finemath": 0.0823}}
{"step": 12780, "total_steps": 72479, "tokens": 6700400640, "loss": 3.5178518891334534, "l_mix": 2.1610392034053802, "l_a": 2.2928722202777863, "l_c": 2.224666476249695, "z_loss": 15.510166645050049, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18964195251464844, "tokens_per_second": 1435212.5884056152, "eta_seconds": 21808.246084833143, "data_wait_seconds": 1.7911195755004883e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20047, "finemath": 0.08232}}
{"step": 12790, "total_steps": 72479, "tokens": 6705643520, "loss": 3.522643268108368, "l_mix": 2.165350168943405, "l_a": 2.2907828092575073, "l_c": 2.228369355201721, "z_loss": 15.47394609451294, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1953083574771881, "tokens_per_second": 1364237.7301805776, "eta_seconds": 22938.9832429409, "data_wait_seconds": 2.199411392211914e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20766, "cosmopedia_v2": 0.20047, "finemath": 0.08233}}
{"step": 12800, "total_steps": 72479, "tokens": 6710886400, "loss": 3.5586487650871277, "l_mix": 2.1870805621147156, "l_a": 2.311287045478821, "l_c": 2.2555041313171387, "z_loss": 15.307854294776917, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2517508268356323, "tokens_per_second": 1227395.7045913602, "eta_seconds": 25492.172927570344, "data_wait_seconds": 3.120303153991699e-05, "mix": {"fineweb_hq": 0.50953, "dclm": 0.20766, "cosmopedia_v2": 0.20047, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.23617785, "gate_a2b/L1_absmax": 0.65264982, "gate_a2b/L1_rms": 0.26409915, "gate_a2b/L3_absmean": 0.30760494, "gate_a2b/L3_absmax": 0.49866441, "gate_a2b/L3_rms": 0.31281495, "gate_a2b/L5_absmean": 0.49689934, "gate_a2b/L5_absmax": 0.67520267, "gate_a2b/L5_rms": 0.50396222, "gate_c2b/L1_absmean": 0.01464131, "gate_c2b/L1_absmax": 0.49977329, "gate_c2b/L1_rms": 0.05808827, "gate_c2b/L3_absmean": 0.20951498, "gate_c2b/L3_absmax": 0.43114859, "gate_c2b/L3_rms": 0.21448861, "gate_c2b/L5_absmean": 0.53798258, "gate_c2b/L5_absmax": 0.74529237, "gate_c2b/L5_rms": 0.55056244, "gate_b2a/L7_absmean": 0.0765256, "gate_b2a/L7_absmax": 0.24666955, "gate_b2a/L7_rms": 0.0801041, "gate_b2a/L11_absmean": 0.12290667, "gate_b2a/L11_absmax": 0.36450076, "gate_b2a/L11_rms": 0.12524126, "gate_b2a/L14_absmean": 0.29844883, "gate_b2a/L14_absmax": 0.46339607, "gate_b2a/L14_rms": 0.29958776, "gate_b2c/L3_absmean": 0.07269246, "gate_b2c/L3_absmax": 0.54096925, "gate_b2c/L3_rms": 0.09359745, "gate_b2c/L5_absmean": 0.10961817, "gate_b2c/L5_absmax": 0.63604414, "gate_b2c/L5_rms": 0.11539351, "gate_b2c/L6_absmean": 0.16367681, "gate_b2c/L6_absmax": 0.44590589, "gate_b2c/L6_rms": 0.16649629}, "streams": {"cos_a2b/round0": 0.357964, "cos_c2b/round0": 0.227977, "cos_b2a/round0": 0.150617, "cos_b2c/round0": 0.081589, "rms_a/round0": 0.949158, "rms_b/round0": 1.755017, "rms_c/round0": 1.286255, "cos_a2b/round1": 0.395652, "cos_c2b/round1": 0.263267, "cos_b2a/round1": 0.206257, "cos_b2c/round1": 0.145865, "rms_a/round1": 1.721284, "rms_b/round1": 3.399646, "rms_c/round1": 2.17355, "cos_a2b/round2": 0.491101, "cos_c2b/round2": 0.45902, "cos_b2a/round2": 0.033128, "cos_b2c/round2": 0.115426, "rms_a/round2": 3.581769, "rms_b/round2": 7.328039, "rms_c/round2": 3.481046}, "alpha_mean": 0.4336915887207837, "alpha_hist": [0.05113, 0.14003, 0.16338, 0.1391, 0.11809, 0.10677, 0.10541, 0.10316, 0.06355, 0.00938], "alpha_by_type": {"special": {"share": 0.00107, "mean": 0.67966, "std": 0.11034}, "whitespace": {"share": 0.03465, "mean": 0.64175, "std": 0.19399}, "digit": {"share": 0.01207, "mean": 0.45311, "std": 0.17523}, "word_start": {"share": 0.57475, "mean": 0.44891, "std": 0.2179}, "word_cont": {"share": 0.25073, "mean": 0.26455, "std": 0.16167}, "punct": {"share": 0.12672, "mean": 0.63852, "std": 0.19541}}, "alpha_p05": 0.10339, "alpha_p50": 0.39888, "alpha_p95": 0.83602}
{"step": 12810, "total_steps": 72479, "tokens": 6716129280, "loss": 4.545660316944122, "l_mix": 2.812409520149231, "l_a": 2.9122623205184937, "l_c": 2.8596955239772797, "z_loss": 16.6337673664093, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17311500012874603, "tokens_per_second": 1365553.8889756193, "eta_seconds": 22909.19525370598, "data_wait_seconds": 1.5556812286376953e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20765, "cosmopedia_v2": 0.20039, "finemath": 0.08236}}
{"step": 12820, "total_steps": 72479, "tokens": 6721372160, "loss": 4.680826425552368, "l_mix": 2.8953288197517395, "l_a": 2.997848719358444, "l_c": 2.9481548070907593, "z_loss": 16.964543342590332, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19960862398147583, "tokens_per_second": 1410127.2714909397, "eta_seconds": 22181.32960362434, "data_wait_seconds": 1.9669532775878906e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20764, "cosmopedia_v2": 0.20039, "finemath": 0.08237}}
{"step": 12830, "total_steps": 72479, "tokens": 6726615040, "loss": 4.498547196388245, "l_mix": 2.7820949852466583, "l_a": 2.883226901292801, "l_c": 2.8325689136981964, "z_loss": 17.133301258087158, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1723320484161377, "tokens_per_second": 1472897.8644559814, "eta_seconds": 21232.466735601425, "data_wait_seconds": 2.065300941467285e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20764, "cosmopedia_v2": 0.20039, "finemath": 0.08239}}
{"step": 12840, "total_steps": 72479, "tokens": 6731857920, "loss": 3.5254993438720703, "l_mix": 2.16570782661438, "l_a": 2.298143684864044, "l_c": 2.229353755712509, "z_loss": 15.421606421470642, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20751848816871643, "tokens_per_second": 1470667.617553687, "eta_seconds": 21261.10050890446, "data_wait_seconds": 2.664327621459961e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20763, "cosmopedia_v2": 0.20047, "finemath": 0.08232}}
{"step": 12850, "total_steps": 72479, "tokens": 6737100800, "loss": 3.5686636567115784, "l_mix": 2.1933461129665375, "l_a": 2.324002265930176, "l_c": 2.255276083946228, "z_loss": 15.340307235717773, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19531868398189545, "tokens_per_second": 1428306.5909527119, "eta_seconds": 21887.99614174366, "data_wait_seconds": 1.823902130126953e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20763, "cosmopedia_v2": 0.20047, "finemath": 0.08233}}
{"step": 12860, "total_steps": 72479, "tokens": 6742343680, "loss": 3.5853377282619476, "l_mix": 2.2044923305511475, "l_a": 2.331148326396942, "l_c": 2.2665156722068787, "z_loss": 15.461187362670898, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2649063766002655, "tokens_per_second": 1459252.4522859687, "eta_seconds": 21420.23213531971, "data_wait_seconds": 2.428889274597168e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20762, "cosmopedia_v2": 0.20047, "finemath": 0.08235}}
{"step": 12870, "total_steps": 72479, "tokens": 6747586560, "loss": 5.021925687789917, "l_mix": 3.1117089986801147, "l_a": 3.1995609998703003, "l_c": 3.1624425053596497, "z_loss": 16.15556764602661, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19441917538642883, "tokens_per_second": 1484917.814518507, "eta_seconds": 21046.473472428323, "data_wait_seconds": 2.9474496841430664e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20769, "cosmopedia_v2": 0.20039, "finemath": 0.08236}}
{"step": 12880, "total_steps": 72479, "tokens": 6752829440, "loss": 5.070436120033264, "l_mix": 3.1420688927173615, "l_a": 3.2307128310203552, "l_c": 3.1917403042316437, "z_loss": 16.31234121322632, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17409703135490417, "tokens_per_second": 1361102.199838514, "eta_seconds": 22957.15965759754, "data_wait_seconds": 2.6553869247436523e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20769, "cosmopedia_v2": 0.20039, "finemath": 0.08238}}
{"step": 12890, "total_steps": 72479, "tokens": 6758072320, "loss": 3.539879322052002, "l_mix": 2.1753346621990204, "l_a": 2.3070700466632843, "l_c": 2.2361429631710052, "z_loss": 15.807621479034424, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2274819314479828, "tokens_per_second": 1335589.653590943, "eta_seconds": 23391.763741207124, "data_wait_seconds": 2.1070241928100586e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20768, "cosmopedia_v2": 0.20047, "finemath": 0.08231}}
{"step": 12900, "total_steps": 72479, "tokens": 6763315200, "loss": 3.586979418992996, "l_mix": 2.2042360305786133, "l_a": 2.3339995741844177, "l_c": 2.2699986696243286, "z_loss": 15.438040614128113, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20057694613933563, "tokens_per_second": 1333794.7415028196, "eta_seconds": 23419.311667704584, "data_wait_seconds": 2.1159648895263672e-05, "mix": {"fineweb_hq": 0.50953, "dclm": 0.20767, "cosmopedia_v2": 0.20047, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.23769538, "gate_a2b/L1_absmax": 0.64424568, "gate_a2b/L1_rms": 0.26566508, "gate_a2b/L3_absmean": 0.30776629, "gate_a2b/L3_absmax": 0.50104636, "gate_a2b/L3_rms": 0.31301039, "gate_a2b/L5_absmean": 0.49752894, "gate_a2b/L5_absmax": 0.67487031, "gate_a2b/L5_rms": 0.50452828, "gate_c2b/L1_absmean": 0.01579212, "gate_c2b/L1_absmax": 0.50078756, "gate_c2b/L1_rms": 0.05844671, "gate_c2b/L3_absmean": 0.20987597, "gate_c2b/L3_absmax": 0.43103054, "gate_c2b/L3_rms": 0.21495621, "gate_c2b/L5_absmean": 0.53752363, "gate_c2b/L5_absmax": 0.74844986, "gate_c2b/L5_rms": 0.55006307, "gate_b2a/L7_absmean": 0.07627252, "gate_b2a/L7_absmax": 0.24911752, "gate_b2a/L7_rms": 0.07996099, "gate_b2a/L11_absmean": 0.12255765, "gate_b2a/L11_absmax": 0.36460555, "gate_b2a/L11_rms": 0.12491436, "gate_b2a/L14_absmean": 0.29867774, "gate_b2a/L14_absmax": 0.46699759, "gate_b2a/L14_rms": 0.29982632, "gate_b2c/L3_absmean": 0.07265428, "gate_b2c/L3_absmax": 0.5451802, "gate_b2c/L3_rms": 0.09384087, "gate_b2c/L5_absmean": 0.10942914, "gate_b2c/L5_absmax": 0.63501829, "gate_b2c/L5_rms": 0.11517575, "gate_b2c/L6_absmean": 0.16303915, "gate_b2c/L6_absmax": 0.45184118, "gate_b2c/L6_rms": 0.16599219}, "streams": {"cos_a2b/round0": 0.360054, "cos_c2b/round0": 0.224994, "cos_b2a/round0": 0.148274, "cos_b2c/round0": 0.081812, "rms_a/round0": 0.953891, "rms_b/round0": 1.767303, "rms_c/round0": 1.299815, "cos_a2b/round1": 0.391283, "cos_c2b/round1": 0.263795, "cos_b2a/round1": 0.209035, "cos_b2c/round1": 0.143868, "rms_a/round1": 1.70433, "rms_b/round1": 3.423997, "rms_c/round1": 2.16649, "cos_a2b/round2": 0.482063, "cos_c2b/round2": 0.459962, "cos_b2a/round2": 0.030373, "cos_b2c/round2": 0.110776, "rms_a/round2": 3.531563, "rms_b/round2": 7.335492, "rms_c/round2": 3.473785}, "alpha_mean": 0.39536901121094914, "alpha_hist": [0.07732, 0.16734, 0.17152, 0.14226, 0.11264, 0.10043, 0.09412, 0.08408, 0.04471, 0.00558], "alpha_by_type": {"special": {"share": 0.00099, "mean": 0.57404, "std": 0.10369}, "whitespace": {"share": 0.03775, "mean": 0.56526, "std": 0.20651}, "digit": {"share": 0.01207, "mean": 0.38055, "std": 0.14527}, "word_start": {"share": 0.5636, "mean": 0.41726, "std": 0.2186}, "word_cont": {"share": 0.27071, "mean": 0.2416, "std": 0.15648}, "punct": {"share": 0.11488, "mean": 0.59449, "std": 0.2078}}, "alpha_p05": 0.08152, "alpha_p50": 0.34378, "alpha_p95": 0.80563}
{"step": 12910, "total_steps": 72479, "tokens": 6768558080, "loss": 3.609092801809311, "l_mix": 2.2185721695423126, "l_a": 2.3515422642230988, "l_c": 2.2783961594104767, "z_loss": 15.390820384025574, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20587188005447388, "tokens_per_second": 1319821.5142147895, "eta_seconds": 23663.28441810608, "data_wait_seconds": 2.7567148208618164e-05, "mix": {"fineweb_hq": 0.50953, "dclm": 0.20767, "cosmopedia_v2": 0.20046, "finemath": 0.08234}}
{"step": 12920, "total_steps": 72479, "tokens": 6773800960, "loss": 4.590128779411316, "l_mix": 2.8388646245002747, "l_a": 2.940524250268936, "l_c": 2.8915750980377197, "z_loss": 16.34151339530945, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18239784240722656, "tokens_per_second": 1455707.9270117485, "eta_seconds": 21450.778973293305, "data_wait_seconds": 1.7762184143066406e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20766, "cosmopedia_v2": 0.20039, "finemath": 0.08235}}
{"step": 12930, "total_steps": 72479, "tokens": 6779043840, "loss": 4.648947477340698, "l_mix": 2.8769598305225372, "l_a": 2.977332979440689, "l_c": 2.9236904978752136, "z_loss": 16.805080890655518, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16856306791305542, "tokens_per_second": 1400313.6284429664, "eta_seconds": 22295.595413661005, "data_wait_seconds": 2.6524066925048828e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20766, "cosmopedia_v2": 0.20039, "finemath": 0.08237}}
{"step": 12940, "total_steps": 72479, "tokens": 6784286720, "loss": 4.601979196071625, "l_mix": 2.8460769057273865, "l_a": 2.94912651181221, "l_c": 2.8981986045837402, "z_loss": 17.048357486724854, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19781523942947388, "tokens_per_second": 1413660.5839738613, "eta_seconds": 22081.384729743004, "data_wait_seconds": 2.619624137878418e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20039, "finemath": 0.08238}}
{"step": 12950, "total_steps": 72479, "tokens": 6789529600, "loss": 3.553108364343643, "l_mix": 2.183763027191162, "l_a": 2.3123231530189514, "l_c": 2.247094601392746, "z_loss": 15.199594020843506, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2198486179113388, "tokens_per_second": 1360656.6819521845, "eta_seconds": 22937.70409977436, "data_wait_seconds": 2.0712614059448242e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20046, "finemath": 0.08232}}
{"step": 12960, "total_steps": 72479, "tokens": 6794772480, "loss": 3.5378833413124084, "l_mix": 2.174040287733078, "l_a": 2.3012414276599884, "l_c": 2.239819437265396, "z_loss": 15.247795343399048, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2149360030889511, "tokens_per_second": 1475930.9059227158, "eta_seconds": 21142.654677653314, "data_wait_seconds": 1.621246337890625e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20764, "cosmopedia_v2": 0.20046, "finemath": 0.08233}}
{"step": 12970, "total_steps": 72479, "tokens": 6800015360, "loss": 3.5919982194900513, "l_mix": 2.207679808139801, "l_a": 2.3404945135116577, "l_c": 2.268895000219345, "z_loss": 15.014458417892456, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21383173763751984, "tokens_per_second": 1420638.6937652838, "eta_seconds": 21961.8504894495, "data_wait_seconds": 1.633167266845703e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20763, "cosmopedia_v2": 0.20046, "finemath": 0.08234}}
{"step": 12980, "total_steps": 72479, "tokens": 6805258240, "loss": 3.5871047973632812, "l_mix": 2.2050038278102875, "l_a": 2.3340940475463867, "l_c": 2.267617791891098, "z_loss": 15.874194979667664, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2599239945411682, "tokens_per_second": 1449844.3267064039, "eta_seconds": 21515.83527789116, "data_wait_seconds": 3.072619438171387e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20763, "cosmopedia_v2": 0.20046, "finemath": 0.08236}}
{"step": 12990, "total_steps": 72479, "tokens": 6810501120, "loss": 3.593458354473114, "l_mix": 2.2075090408325195, "l_a": 2.341983199119568, "l_c": 2.27274551987648, "z_loss": 15.30689561367035, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2360353320837021, "tokens_per_second": 1489054.956145459, "eta_seconds": 20945.74730319977, "data_wait_seconds": 1.6748905181884766e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20762, "cosmopedia_v2": 0.20046, "finemath": 0.08237}}
{"step": 13000, "total_steps": 72479, "tokens": 6815744000, "loss": 5.038126289844513, "l_mix": 3.1207329630851746, "l_a": 3.212925434112549, "l_c": 3.1730879843235016, "z_loss": 15.893492817878723, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22374080121517181, "tokens_per_second": 1425769.7311911094, "eta_seconds": 21871.782848095892, "data_wait_seconds": 3.0100345611572266e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20769, "cosmopedia_v2": 0.20046, "finemath": 0.08231}, "gates": {"gate_a2b/L1_absmean": 0.23747949, "gate_a2b/L1_absmax": 0.65352428, "gate_a2b/L1_rms": 0.26514578, "gate_a2b/L3_absmean": 0.30635417, "gate_a2b/L3_absmax": 0.5038318, "gate_a2b/L3_rms": 0.31168166, "gate_a2b/L5_absmean": 0.49720725, "gate_a2b/L5_absmax": 0.67938137, "gate_a2b/L5_rms": 0.50438476, "gate_c2b/L1_absmean": 0.01530531, "gate_c2b/L1_absmax": 0.507388, "gate_c2b/L1_rms": 0.05848771, "gate_c2b/L3_absmean": 0.20926605, "gate_c2b/L3_absmax": 0.44019076, "gate_c2b/L3_rms": 0.2142628, "gate_c2b/L5_absmean": 0.5358572, "gate_c2b/L5_absmax": 0.73976135, "gate_c2b/L5_rms": 0.54845625, "gate_b2a/L7_absmean": 0.07658404, "gate_b2a/L7_absmax": 0.25034499, "gate_b2a/L7_rms": 0.08028305, "gate_b2a/L11_absmean": 0.12222887, "gate_b2a/L11_absmax": 0.36493078, "gate_b2a/L11_rms": 0.12467091, "gate_b2a/L14_absmean": 0.29957223, "gate_b2a/L14_absmax": 0.46817768, "gate_b2a/L14_rms": 0.30073473, "gate_b2c/L3_absmean": 0.07234029, "gate_b2c/L3_absmax": 0.54454404, "gate_b2c/L3_rms": 0.09314711, "gate_b2c/L5_absmean": 0.10921706, "gate_b2c/L5_absmax": 0.64022368, "gate_b2c/L5_rms": 0.11509579, "gate_b2c/L6_absmean": 0.16350149, "gate_b2c/L6_absmax": 0.44965315, "gate_b2c/L6_rms": 0.16637424}, "streams": {"cos_a2b/round0": 0.373352, "cos_c2b/round0": 0.254182, "cos_b2a/round0": 0.164375, "cos_b2c/round0": 0.116468, "rms_a/round0": 0.923187, "rms_b/round0": 1.783005, "rms_c/round0": 1.26483, "cos_a2b/round1": 0.388897, "cos_c2b/round1": 0.272726, "cos_b2a/round1": 0.226997, "cos_b2c/round1": 0.149459, "rms_a/round1": 1.648719, "rms_b/round1": 3.301603, "rms_c/round1": 2.115434, "cos_a2b/round2": 0.527195, "cos_c2b/round2": 0.469621, "cos_b2a/round2": 0.085759, "cos_b2c/round2": 0.150843, "rms_a/round2": 3.536957, "rms_b/round2": 7.04465, "rms_c/round2": 3.342363}, "alpha_mean": 0.4659580099039289, "alpha_hist": [0.01207, 0.05138, 0.11815, 0.1727, 0.20372, 0.21397, 0.14168, 0.07253, 0.01361, 0.0002], "alpha_by_type": {"special": {"share": 0.0007, "mean": 0.55173, "std": 0.1175}, "whitespace": {"share": 0.04948, "mean": 0.55944, "std": 0.12118}, "digit": {"share": 0.03749, "mean": 0.50631, "std": 0.10876}, "word_start": {"share": 0.53389, "mean": 0.4937, "std": 0.15291}, "word_cont": {"share": 0.26271, "mean": 0.33813, "std": 0.14347}, "punct": {"share": 0.11572, "mean": 0.57458, "std": 0.15479}}, "alpha_p05": 0.19529, "alpha_p50": 0.4683, "alpha_p95": 0.73488}
{"step": 13010, "total_steps": 72479, "tokens": 6820986880, "loss": 3.58772075176239, "l_mix": 2.205220013856888, "l_a": 2.330383449792862, "l_c": 2.272722691297531, "z_loss": 15.68777585029602, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.28170159459114075, "tokens_per_second": 1376439.3970921193, "eta_seconds": 22651.83860464096, "data_wait_seconds": 2.586841583251953e-05, "mix": {"fineweb_hq": 0.50953, "dclm": 0.20769, "cosmopedia_v2": 0.20046, "finemath": 0.08232}}
{"step": 13020, "total_steps": 72479, "tokens": 6826229760, "loss": 4.606544375419617, "l_mix": 2.850702315568924, "l_a": 2.9485666751861572, "l_c": 2.8987529277801514, "z_loss": 16.462180376052856, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19587646424770355, "tokens_per_second": 1312087.0493139033, "eta_seconds": 23758.82012424469, "data_wait_seconds": 1.7940998077392578e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20768, "cosmopedia_v2": 0.20038, "finemath": 0.08233}}
{"step": 13030, "total_steps": 72479, "tokens": 6831472640, "loss": 4.569933891296387, "l_mix": 2.8265923261642456, "l_a": 2.9289324581623077, "l_c": 2.8765970170497894, "z_loss": 16.82564353942871, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.14451587200164795, "tokens_per_second": 1375979.9955974112, "eta_seconds": 22651.78084836006, "data_wait_seconds": 1.8328428268432617e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20767, "cosmopedia_v2": 0.20038, "finemath": 0.08235}}
{"step": 13040, "total_steps": 72479, "tokens": 6836715520, "loss": 4.6364442110061646, "l_mix": 2.8684360682964325, "l_a": 2.9685224890708923, "l_c": 2.9193191528320312, "z_loss": 16.556521892547607, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15651467442512512, "tokens_per_second": 1429708.3461914703, "eta_seconds": 21796.861237478253, "data_wait_seconds": 3.3795833587646484e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20767, "cosmopedia_v2": 0.20038, "finemath": 0.08236}}
{"step": 13050, "total_steps": 72479, "tokens": 6841958400, "loss": 4.606314778327942, "l_mix": 2.8499560356140137, "l_a": 2.9489453434944153, "l_c": 2.9000691175460815, "z_loss": 16.543534874916077, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21805799007415771, "tokens_per_second": 1435715.493917142, "eta_seconds": 21702.009683680535, "data_wait_seconds": 2.4974346160888672e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20038, "finemath": 0.08238}}
{"step": 13060, "total_steps": 72479, "tokens": 6847201280, "loss": 3.5588890314102173, "l_mix": 2.187049150466919, "l_a": 2.3165919184684753, "l_c": 2.2509376108646393, "z_loss": 15.80906069278717, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22738908231258392, "tokens_per_second": 1304332.2980947283, "eta_seconds": 23883.996982598303, "data_wait_seconds": 2.4557113647460938e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20046, "finemath": 0.08231}}
{"step": 13070, "total_steps": 72479, "tokens": 6852444160, "loss": 3.593226432800293, "l_mix": 2.209298938512802, "l_a": 2.33662211894989, "l_c": 2.2712791860103607, "z_loss": 15.570998072624207, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2583695948123932, "tokens_per_second": 1451097.2565062777, "eta_seconds": 21464.74032139778, "data_wait_seconds": 1.9490718841552734e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20046, "finemath": 0.08233}}
{"step": 13080, "total_steps": 72479, "tokens": 6857687040, "loss": 3.562128782272339, "l_mix": 2.189824730157852, "l_a": 2.3196378648281097, "l_c": 2.249633401632309, "z_loss": 15.226420402526855, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20837610960006714, "tokens_per_second": 1482394.3803839032, "eta_seconds": 21008.028176641463, "data_wait_seconds": 2.09808349609375e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20046, "finemath": 0.08234}}
{"step": 13090, "total_steps": 72479, "tokens": 6862929920, "loss": 3.581823855638504, "l_mix": 2.201472193002701, "l_a": 2.332267999649048, "l_c": 2.263752728700638, "z_loss": 15.454617381095886, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19073079526424408, "tokens_per_second": 1427767.422946012, "eta_seconds": 21808.1317248106, "data_wait_seconds": 4.3004751205444336e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20764, "cosmopedia_v2": 0.20046, "finemath": 0.08235}}
{"step": 13100, "total_steps": 72479, "tokens": 6868172800, "loss": 3.602421224117279, "l_mix": 2.213573604822159, "l_a": 2.3480264842510223, "l_c": 2.276304066181183, "z_loss": 15.483837842941284, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.23526731133460999, "tokens_per_second": 1394328.9617463332, "eta_seconds": 22327.368939542772, "data_wait_seconds": 4.032254219055176e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20763, "cosmopedia_v2": 0.20046, "finemath": 0.08237}, "gates": {"gate_a2b/L1_absmean": 0.23940597, "gate_a2b/L1_absmax": 0.65000486, "gate_a2b/L1_rms": 0.2666463, "gate_a2b/L3_absmean": 0.30711091, "gate_a2b/L3_absmax": 0.4998033, "gate_a2b/L3_rms": 0.31229025, "gate_a2b/L5_absmean": 0.49776363, "gate_a2b/L5_absmax": 0.68213934, "gate_a2b/L5_rms": 0.50482792, "gate_c2b/L1_absmean": 0.01465504, "gate_c2b/L1_absmax": 0.50168443, "gate_c2b/L1_rms": 0.0580334, "gate_c2b/L3_absmean": 0.20950699, "gate_c2b/L3_absmax": 0.42943197, "gate_c2b/L3_rms": 0.21442416, "gate_c2b/L5_absmean": 0.53700465, "gate_c2b/L5_absmax": 0.74111187, "gate_c2b/L5_rms": 0.54951859, "gate_b2a/L7_absmean": 0.07705969, "gate_b2a/L7_absmax": 0.25009394, "gate_b2a/L7_rms": 0.08054178, "gate_b2a/L11_absmean": 0.12249033, "gate_b2a/L11_absmax": 0.36640468, "gate_b2a/L11_rms": 0.12481567, "gate_b2a/L14_absmean": 0.30045384, "gate_b2a/L14_absmax": 0.46718615, "gate_b2a/L14_rms": 0.30158848, "gate_b2c/L3_absmean": 0.07329021, "gate_b2c/L3_absmax": 0.54650474, "gate_b2c/L3_rms": 0.09419551, "gate_b2c/L5_absmean": 0.11050729, "gate_b2c/L5_absmax": 0.64358592, "gate_b2c/L5_rms": 0.11637542, "gate_b2c/L6_absmean": 0.16434294, "gate_b2c/L6_absmax": 0.44841698, "gate_b2c/L6_rms": 0.16717872}, "streams": {"cos_a2b/round0": 0.357707, "cos_c2b/round0": 0.220327, "cos_b2a/round0": 0.148969, "cos_b2c/round0": 0.081735, "rms_a/round0": 0.940585, "rms_b/round0": 1.762721, "rms_c/round0": 1.296278, "cos_a2b/round1": 0.382864, "cos_c2b/round1": 0.265305, "cos_b2a/round1": 0.211171, "cos_b2c/round1": 0.14065, "rms_a/round1": 1.672812, "rms_b/round1": 3.376359, "rms_c/round1": 2.180393, "cos_a2b/round2": 0.489343, "cos_c2b/round2": 0.456149, "cos_b2a/round2": 0.031508, "cos_b2c/round2": 0.114365, "rms_a/round2": 3.51191, "rms_b/round2": 7.235381, "rms_c/round2": 3.459272}, "alpha_mean": 0.4400676241613475, "alpha_hist": [0.05225, 0.13211, 0.15956, 0.13873, 0.11806, 0.10947, 0.10646, 0.10332, 0.07062, 0.00943], "alpha_by_type": {"special": {"share": 0.00113, "mean": 0.56786, "std": 0.13745}, "whitespace": {"share": 0.03172, "mean": 0.66445, "std": 0.17562}, "digit": {"share": 0.01158, "mean": 0.46279, "std": 0.17559}, "word_start": {"share": 0.56876, "mean": 0.46073, "std": 0.21877}, "word_cont": {"share": 0.27116, "mean": 0.2758, "std": 0.16849}, "punct": {"share": 0.11565, "mean": 0.65853, "std": 0.19248}}, "alpha_p05": 0.10632, "alpha_p50": 0.40216, "alpha_p95": 0.83142}
{"step": 13110, "total_steps": 72479, "tokens": 6873415680, "loss": 3.0982911586761475, "l_mix": 1.9157679080963135, "l_a": 1.9884176850318909, "l_c": 1.9478265792131424, "z_loss": 16.49880027770996, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2776314616203308, "tokens_per_second": 1433256.608456685, "eta_seconds": 21717.29339208603, "data_wait_seconds": 2.0831823348999023e-05, "mix": {"fineweb_hq": 0.50953, "dclm": 0.20763, "cosmopedia_v2": 0.20046, "finemath": 0.08238}}
{"step": 13120, "total_steps": 72479, "tokens": 6878658560, "loss": 4.713772118091583, "l_mix": 2.9163288474082947, "l_a": 3.0169423818588257, "l_c": 2.9691592156887054, "z_loss": 16.12770938873291, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20001322031021118, "tokens_per_second": 1444660.626629209, "eta_seconds": 21542.22993161678, "data_wait_seconds": 3.2842159271240234e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20762, "cosmopedia_v2": 0.20046, "finemath": 0.08232}}
{"step": 13130, "total_steps": 72479, "tokens": 6883901440, "loss": 4.670042634010315, "l_mix": 2.8893359303474426, "l_a": 2.9885667264461517, "l_c": 2.94165575504303, "z_loss": 16.398578882217407, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19898675382137299, "tokens_per_second": 1244857.5898838248, "eta_seconds": 24995.60493092537, "data_wait_seconds": 1.952052116394043e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20762, "cosmopedia_v2": 0.20046, "finemath": 0.08233}}
{"step": 13140, "total_steps": 72479, "tokens": 6889144320, "loss": 4.71300595998764, "l_mix": 2.9167121946811676, "l_a": 3.0134258568286896, "l_c": 2.968541592359543, "z_loss": 17.03584575653076, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18414101004600525, "tokens_per_second": 1379872.890862119, "eta_seconds": 22546.080757164957, "data_wait_seconds": 2.3156404495239258e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20761, "cosmopedia_v2": 0.20046, "finemath": 0.08234}}
{"step": 13150, "total_steps": 72479, "tokens": 6894387200, "loss": 4.607141405344009, "l_mix": 2.8507531881332397, "l_a": 2.9471065998077393, "l_c": 2.901968777179718, "z_loss": 16.65655517578125, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1595742553472519, "tokens_per_second": 1386794.148353116, "eta_seconds": 22429.776466059684, "data_wait_seconds": 1.8268823623657227e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20768, "cosmopedia_v2": 0.20038, "finemath": 0.08236}}
{"step": 13160, "total_steps": 72479, "tokens": 6899630080, "loss": 4.673046052455902, "l_mix": 2.889733135700226, "l_a": 2.9981764554977417, "l_c": 2.940539985895157, "z_loss": 16.97929286956787, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.3291620910167694, "tokens_per_second": 1386728.8211113515, "eta_seconds": 22427.052354097366, "data_wait_seconds": 3.355741500854492e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20038, "finemath": 0.08237}}
{"step": 13170, "total_steps": 72479, "tokens": 6904872960, "loss": 3.580924540758133, "l_mix": 2.200346678495407, "l_a": 2.3306672275066376, "l_c": 2.266077011823654, "z_loss": 15.546405673027039, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.24332831799983978, "tokens_per_second": 1453678.2334459596, "eta_seconds": 21390.563796424867, "data_wait_seconds": 1.6003847122192383e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20046, "finemath": 0.08231}}
{"step": 13180, "total_steps": 72479, "tokens": 6910115840, "loss": 3.583343595266342, "l_mix": 2.20164555311203, "l_a": 2.3370290100574493, "l_c": 2.263471931219101, "z_loss": 15.476561546325684, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21738792955875397, "tokens_per_second": 1392432.4128445624, "eta_seconds": 22327.657576203346, "data_wait_seconds": 2.8133392333984375e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20046, "finemath": 0.08232}}
{"step": 13190, "total_steps": 72479, "tokens": 6915358720, "loss": 3.553399294614792, "l_mix": 2.1833690106868744, "l_a": 2.314982980489731, "l_c": 2.246640920639038, "z_loss": 15.430763721466064, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20601947605609894, "tokens_per_second": 1237614.4221538631, "eta_seconds": 25116.47462697029, "data_wait_seconds": 2.002716064453125e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20766, "cosmopedia_v2": 0.20045, "finemath": 0.08234}}
{"step": 13200, "total_steps": 72479, "tokens": 6920601600, "loss": 3.5154018104076385, "l_mix": 2.159953087568283, "l_a": 2.2908071875572205, "l_c": 2.2222688794136047, "z_loss": 15.258522152900696, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20175406336784363, "tokens_per_second": 1477927.8981459695, "eta_seconds": 21028.947617125512, "data_wait_seconds": 1.8656253814697266e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20765, "cosmopedia_v2": 0.20045, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.24166794, "gate_a2b/L1_absmax": 0.64562356, "gate_a2b/L1_rms": 0.26835835, "gate_a2b/L3_absmean": 0.30764514, "gate_a2b/L3_absmax": 0.49559322, "gate_a2b/L3_rms": 0.31288794, "gate_a2b/L5_absmean": 0.49931788, "gate_a2b/L5_absmax": 0.68024755, "gate_a2b/L5_rms": 0.50631917, "gate_c2b/L1_absmean": 0.01523874, "gate_c2b/L1_absmax": 0.49906737, "gate_c2b/L1_rms": 0.05790251, "gate_c2b/L3_absmean": 0.20932984, "gate_c2b/L3_absmax": 0.41800869, "gate_c2b/L3_rms": 0.21433978, "gate_c2b/L5_absmean": 0.53569692, "gate_c2b/L5_absmax": 0.73709023, "gate_c2b/L5_rms": 0.54816258, "gate_b2a/L7_absmean": 0.07734911, "gate_b2a/L7_absmax": 0.24774912, "gate_b2a/L7_rms": 0.08094732, "gate_b2a/L11_absmean": 0.12368831, "gate_b2a/L11_absmax": 0.36310935, "gate_b2a/L11_rms": 0.12610604, "gate_b2a/L14_absmean": 0.30038622, "gate_b2a/L14_absmax": 0.4715035, "gate_b2a/L14_rms": 0.30151811, "gate_b2c/L3_absmean": 0.07347304, "gate_b2c/L3_absmax": 0.54590851, "gate_b2c/L3_rms": 0.09440777, "gate_b2c/L5_absmean": 0.11090042, "gate_b2c/L5_absmax": 0.62663251, "gate_b2c/L5_rms": 0.11659995, "gate_b2c/L6_absmean": 0.16528831, "gate_b2c/L6_absmax": 0.46094885, "gate_b2c/L6_rms": 0.16821685}, "streams": {"cos_a2b/round0": 0.362705, "cos_c2b/round0": 0.227834, "cos_b2a/round0": 0.14855, "cos_b2c/round0": 0.082622, "rms_a/round0": 0.957522, "rms_b/round0": 1.769667, "rms_c/round0": 1.304638, "cos_a2b/round1": 0.396452, "cos_c2b/round1": 0.265884, "cos_b2a/round1": 0.216911, "cos_b2c/round1": 0.145581, "rms_a/round1": 1.695397, "rms_b/round1": 3.405235, "rms_c/round1": 2.163607, "cos_a2b/round2": 0.490768, "cos_c2b/round2": 0.456868, "cos_b2a/round2": 0.04208, "cos_b2c/round2": 0.116912, "rms_a/round2": 3.563433, "rms_b/round2": 7.379475, "rms_c/round2": 3.416746}, "alpha_mean": 0.4192739868939199, "alpha_hist": [0.05377, 0.15283, 0.17735, 0.14436, 0.11086, 0.09877, 0.09752, 0.09599, 0.06053, 0.00801], "alpha_by_type": {"special": {"share": 0.00107, "mean": 0.53943, "std": 0.11704}, "whitespace": {"share": 0.03691, "mean": 0.61447, "std": 0.185}, "digit": {"share": 0.00929, "mean": 0.38574, "std": 0.18606}, "word_start": {"share": 0.55792, "mean": 0.44062, "std": 0.2215}, "word_cont": {"share": 0.27675, "mean": 0.26161, "std": 0.15607}, "punct": {"share": 0.11806, "mean": 0.62852, "std": 0.20183}}, "alpha_p05": 0.10088, "alpha_p50": 0.34422, "alpha_p95": 0.80441}
{"step": 13210, "total_steps": 72479, "tokens": 6925844480, "loss": 3.5882504880428314, "l_mix": 2.205388128757477, "l_a": 2.3390360474586487, "l_c": 2.2653884291648865, "z_loss": 15.349668264389038, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2161291539669037, "tokens_per_second": 1373978.8867802136, "eta_seconds": 22616.086586904526, "data_wait_seconds": 3.6269426345825195e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20765, "cosmopedia_v2": 0.20045, "finemath": 0.08236}}
{"step": 13220, "total_steps": 72479, "tokens": 6931087360, "loss": 3.096874862909317, "l_mix": 1.9126458317041397, "l_a": 1.9923587143421173, "l_c": 1.9490423649549484, "z_loss": 18.086880683898926, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2642991840839386, "tokens_per_second": 1235988.0015173433, "eta_seconds": 25136.799510884284, "data_wait_seconds": 2.3603439331054688e-05, "mix": {"fineweb_hq": 0.50953, "dclm": 0.20764, "cosmopedia_v2": 0.20045, "finemath": 0.08238}}
{"step": 13230, "total_steps": 72479, "tokens": 6936330240, "loss": 4.689882755279541, "l_mix": 2.901765763759613, "l_a": 2.9988781213760376, "l_c": 2.956113576889038, "z_loss": 16.1952965259552, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19502033293247223, "tokens_per_second": 1454834.7109914413, "eta_seconds": 21351.93742444515, "data_wait_seconds": 2.3990869522094727e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20763, "cosmopedia_v2": 0.20045, "finemath": 0.08231}}
{"step": 13240, "total_steps": 72479, "tokens": 6941573120, "loss": 4.722675204277039, "l_mix": 2.9214793145656586, "l_a": 3.0282952189445496, "l_c": 2.9700936675071716, "z_loss": 16.7911159992218, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16620685160160065, "tokens_per_second": 1315139.8540168337, "eta_seconds": 23615.965052795407, "data_wait_seconds": 1.7017126083374023e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20763, "cosmopedia_v2": 0.20045, "finemath": 0.08233}}
{"step": 13250, "total_steps": 72479, "tokens": 6946816000, "loss": 4.699887990951538, "l_mix": 2.908934146165848, "l_a": 3.0071910619735718, "l_c": 2.9572233855724335, "z_loss": 16.294747352600098, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20122568309307098, "tokens_per_second": 1314592.4227024543, "eta_seconds": 23621.81115281582, "data_wait_seconds": 3.9577484130859375e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20762, "cosmopedia_v2": 0.20045, "finemath": 0.08234}}
{"step": 13260, "total_steps": 72479, "tokens": 6952058880, "loss": 4.64256078004837, "l_mix": 2.8709711730480194, "l_a": 2.979721963405609, "l_c": 2.9200779497623444, "z_loss": 16.49479389190674, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1883721500635147, "tokens_per_second": 1408526.1339376972, "eta_seconds": 22042.76535871029, "data_wait_seconds": 2.5600194931030273e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20762, "cosmopedia_v2": 0.20045, "finemath": 0.08235}}
{"step": 13270, "total_steps": 72479, "tokens": 6957301760, "loss": 4.749241471290588, "l_mix": 2.938287317752838, "l_a": 3.0438441336154938, "l_c": 2.987018972635269, "z_loss": 16.950757265090942, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20561379194259644, "tokens_per_second": 1464676.1062247276, "eta_seconds": 21194.15211327076, "data_wait_seconds": 1.7017126083374023e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20761, "cosmopedia_v2": 0.20045, "finemath": 0.08237}}
{"step": 13280, "total_steps": 72479, "tokens": 6962544640, "loss": 5.131080806255341, "l_mix": 3.1801609992980957, "l_a": 3.2686390578746796, "l_c": 3.2290918231010437, "z_loss": 16.004811644554138, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18429185450077057, "tokens_per_second": 1361839.587119871, "eta_seconds": 22790.73512442112, "data_wait_seconds": 2.2709369659423828e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.20038, "finemath": 0.08238}}
{"step": 13290, "total_steps": 72479, "tokens": 6967787520, "loss": 3.535535544157028, "l_mix": 2.1733374893665314, "l_a": 2.3053017258644104, "l_c": 2.2300938069820404, "z_loss": 15.79348909854889, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.23778575658798218, "tokens_per_second": 1482701.9300691267, "eta_seconds": 20929.41393187046, "data_wait_seconds": 2.2798776626586914e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20045, "finemath": 0.08232}}
{"step": 13300, "total_steps": 72479, "tokens": 6973030400, "loss": 3.529720813035965, "l_mix": 2.1697320342063904, "l_a": 2.2954447865486145, "l_c": 2.232597529888153, "z_loss": 15.760042905807495, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.205965057015419, "tokens_per_second": 1417890.5040009043, "eta_seconds": 21882.394630932806, "data_wait_seconds": 3.0487775802612305e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20045, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.24175902, "gate_a2b/L1_absmax": 0.65491551, "gate_a2b/L1_rms": 0.26873127, "gate_a2b/L3_absmean": 0.30855748, "gate_a2b/L3_absmax": 0.49653998, "gate_a2b/L3_rms": 0.313743, "gate_a2b/L5_absmean": 0.4985505, "gate_a2b/L5_absmax": 0.67448241, "gate_a2b/L5_rms": 0.50555629, "gate_c2b/L1_absmean": 0.01472868, "gate_c2b/L1_absmax": 0.50002754, "gate_c2b/L1_rms": 0.05786283, "gate_c2b/L3_absmean": 0.20971003, "gate_c2b/L3_absmax": 0.42245415, "gate_c2b/L3_rms": 0.21461441, "gate_c2b/L5_absmean": 0.53694534, "gate_c2b/L5_absmax": 0.73771352, "gate_c2b/L5_rms": 0.54949021, "gate_b2a/L7_absmean": 0.07707417, "gate_b2a/L7_absmax": 0.24933945, "gate_b2a/L7_rms": 0.08057734, "gate_b2a/L11_absmean": 0.12316005, "gate_b2a/L11_absmax": 0.3686235, "gate_b2a/L11_rms": 0.12558052, "gate_b2a/L14_absmean": 0.30169854, "gate_b2a/L14_absmax": 0.4709321, "gate_b2a/L14_rms": 0.30280763, "gate_b2c/L3_absmean": 0.07432826, "gate_b2c/L3_absmax": 0.54742175, "gate_b2c/L3_rms": 0.09503405, "gate_b2c/L5_absmean": 0.11067703, "gate_b2c/L5_absmax": 0.6399619, "gate_b2c/L5_rms": 0.11648141, "gate_b2c/L6_absmean": 0.16491622, "gate_b2c/L6_absmax": 0.44879359, "gate_b2c/L6_rms": 0.1677971}, "streams": {"cos_a2b/round0": 0.361268, "cos_c2b/round0": 0.231156, "cos_b2a/round0": 0.146462, "cos_b2c/round0": 0.088122, "rms_a/round0": 0.950604, "rms_b/round0": 1.756237, "rms_c/round0": 1.291608, "cos_a2b/round1": 0.395494, "cos_c2b/round1": 0.267463, "cos_b2a/round1": 0.212867, "cos_b2c/round1": 0.142998, "rms_a/round1": 1.697212, "rms_b/round1": 3.389392, "rms_c/round1": 2.181008, "cos_a2b/round2": 0.492521, "cos_c2b/round2": 0.463637, "cos_b2a/round2": 0.037343, "cos_b2c/round2": 0.114201, "rms_a/round2": 3.552996, "rms_b/round2": 7.318948, "rms_c/round2": 3.477033}, "alpha_mean": 0.4311062817804512, "alpha_hist": [0.05537, 0.13795, 0.16205, 0.13974, 0.1192, 0.10811, 0.1066, 0.1021, 0.06316, 0.00572], "alpha_by_type": {"special": {"share": 0.00124, "mean": 0.57363, "std": 0.09104}, "whitespace": {"share": 0.03912, "mean": 0.54783, "std": 0.21251}, "digit": {"share": 0.01089, "mean": 0.38882, "std": 0.16708}, "word_start": {"share": 0.5739, "mean": 0.46206, "std": 0.21673}, "word_cont": {"share": 0.25128, "mean": 0.25622, "std": 0.16298}, "punct": {"share": 0.12357, "mean": 0.60835, "std": 0.21026}}, "alpha_p05": 0.10781, "alpha_p50": 0.40404, "alpha_p95": 0.83032}
{"step": 13310, "total_steps": 72479, "tokens": 6978273280, "loss": 3.5875562131404877, "l_mix": 2.2051116228103638, "l_a": 2.331513226032257, "l_c": 2.271483927965164, "z_loss": 15.453863739967346, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1951402723789215, "tokens_per_second": 1366430.0795068725, "eta_seconds": 22702.6593875885, "data_wait_seconds": 2.5898218154907227e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20766, "cosmopedia_v2": 0.20045, "finemath": 0.08234}}
{"step": 13320, "total_steps": 72479, "tokens": 6983516160, "loss": 3.5760181546211243, "l_mix": 2.1981437504291534, "l_a": 2.3314614593982697, "l_c": 2.2563224136829376, "z_loss": 15.391531825065613, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.24277423322200775, "tokens_per_second": 1482154.586179941, "eta_seconds": 20926.53093085289, "data_wait_seconds": 2.1696090698242188e-05, "mix": {"fineweb_hq": 0.50953, "dclm": 0.20766, "cosmopedia_v2": 0.20045, "finemath": 0.08236}}
{"step": 13330, "total_steps": 72479, "tokens": 6988759040, "loss": 3.4249504804611206, "l_mix": 2.116944283246994, "l_a": 2.2053223252296448, "l_c": 2.1487914621829987, "z_loss": 17.719588041305542, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2911664545536041, "tokens_per_second": 1474217.4599615755, "eta_seconds": 21035.64213166237, "data_wait_seconds": 3.173947334289551e-05, "mix": {"fineweb_hq": 0.50953, "dclm": 0.20765, "cosmopedia_v2": 0.20045, "finemath": 0.08237}}
{"step": 13340, "total_steps": 72479, "tokens": 6994001920, "loss": 4.741988241672516, "l_mix": 2.9344289004802704, "l_a": 3.0379790365695953, "l_c": 2.981689304113388, "z_loss": 16.58727765083313, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2542603015899658, "tokens_per_second": 1414724.752127919, "eta_seconds": 21916.537464523317, "data_wait_seconds": 1.677870750427246e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20765, "cosmopedia_v2": 0.20045, "finemath": 0.08231}}
{"step": 13350, "total_steps": 72479, "tokens": 6999244800, "loss": 4.68455183506012, "l_mix": 2.8978353440761566, "l_a": 3.0053955912590027, "l_c": 2.9446516931056976, "z_loss": 17.022128105163574, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21133403480052948, "tokens_per_second": 1476088.924960836, "eta_seconds": 21001.86826672554, "data_wait_seconds": 1.7076730728149414e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20764, "cosmopedia_v2": 0.20045, "finemath": 0.08232}}
{"step": 13360, "total_steps": 72479, "tokens": 7004487680, "loss": 4.572401165962219, "l_mix": 2.828179806470871, "l_a": 2.9324571192264557, "l_c": 2.876218557357788, "z_loss": 16.18702208995819, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19289539754390717, "tokens_per_second": 1349628.9882462, "eta_seconds": 22965.853980565073, "data_wait_seconds": 2.1219253540039062e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20763, "cosmopedia_v2": 0.20045, "finemath": 0.08234}}
{"step": 13370, "total_steps": 72479, "tokens": 7009730560, "loss": 4.716168224811554, "l_mix": 2.9177885353565216, "l_a": 3.0266057550907135, "l_c": 2.962566465139389, "z_loss": 16.279274582862854, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21749833226203918, "tokens_per_second": 1354786.4521506068, "eta_seconds": 22874.55660839081, "data_wait_seconds": 2.0235776901245117e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20763, "cosmopedia_v2": 0.20045, "finemath": 0.08235}}
{"step": 13380, "total_steps": 72479, "tokens": 7014973440, "loss": 4.648046553134918, "l_mix": 2.8744848370552063, "l_a": 2.977525383234024, "l_c": 2.9288187623023987, "z_loss": 16.584892988204956, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2248997688293457, "tokens_per_second": 1412285.9327610007, "eta_seconds": 21939.534900999068, "data_wait_seconds": 2.0623207092285156e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20762, "cosmopedia_v2": 0.20045, "finemath": 0.08236}}
{"step": 13390, "total_steps": 72479, "tokens": 7020216320, "loss": 3.6205481588840485, "l_mix": 2.2266618609428406, "l_a": 2.3528424501419067, "l_c": 2.2883524894714355, "z_loss": 15.278153777122498, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2316720187664032, "tokens_per_second": 1424708.1796056638, "eta_seconds": 21744.560798811915, "data_wait_seconds": 4.5865774154663086e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20762, "cosmopedia_v2": 0.20045, "finemath": 0.08237}}
{"step": 13400, "total_steps": 72479, "tokens": 7025459200, "loss": 4.953626334667206, "l_mix": 3.068745791912079, "l_a": 3.1630618274211884, "l_c": 3.114397883415222, "z_loss": 16.425978899002075, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17015238106250763, "tokens_per_second": 1394573.9885034794, "eta_seconds": 22210.66146891117, "data_wait_seconds": 5.7309865951538086e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20769, "cosmopedia_v2": 0.20045, "finemath": 0.08231}, "gates": {"gate_a2b/L1_absmean": 0.24306202, "gate_a2b/L1_absmax": 0.64703161, "gate_a2b/L1_rms": 0.26964101, "gate_a2b/L3_absmean": 0.30834255, "gate_a2b/L3_absmax": 0.49730021, "gate_a2b/L3_rms": 0.31357136, "gate_a2b/L5_absmean": 0.49787632, "gate_a2b/L5_absmax": 0.68104082, "gate_a2b/L5_rms": 0.50480926, "gate_c2b/L1_absmean": 0.01525953, "gate_c2b/L1_absmax": 0.5023815, "gate_c2b/L1_rms": 0.05809098, "gate_c2b/L3_absmean": 0.20989601, "gate_c2b/L3_absmax": 0.43213785, "gate_c2b/L3_rms": 0.21484812, "gate_c2b/L5_absmean": 0.53785336, "gate_c2b/L5_absmax": 0.73272765, "gate_c2b/L5_rms": 0.55038488, "gate_b2a/L7_absmean": 0.07718594, "gate_b2a/L7_absmax": 0.248474, "gate_b2a/L7_rms": 0.08072387, "gate_b2a/L11_absmean": 0.12313439, "gate_b2a/L11_absmax": 0.36733186, "gate_b2a/L11_rms": 0.12554945, "gate_b2a/L14_absmean": 0.30137768, "gate_b2a/L14_absmax": 0.46905923, "gate_b2a/L14_rms": 0.30246598, "gate_b2c/L3_absmean": 0.073898, "gate_b2c/L3_absmax": 0.54946852, "gate_b2c/L3_rms": 0.09467005, "gate_b2c/L5_absmean": 0.11003825, "gate_b2c/L5_absmax": 0.64367861, "gate_b2c/L5_rms": 0.11594681, "gate_b2c/L6_absmean": 0.1645177, "gate_b2c/L6_absmax": 0.44846439, "gate_b2c/L6_rms": 0.16743158}, "streams": {"cos_a2b/round0": 0.378324, "cos_c2b/round0": 0.261707, "cos_b2a/round0": 0.16487, "cos_b2c/round0": 0.124064, "rms_a/round0": 0.95543, "rms_b/round0": 1.781664, "rms_c/round0": 1.298797, "cos_a2b/round1": 0.406593, "cos_c2b/round1": 0.286271, "cos_b2a/round1": 0.241759, "cos_b2c/round1": 0.151898, "rms_a/round1": 1.664503, "rms_b/round1": 3.25847, "rms_c/round1": 2.101014, "cos_a2b/round2": 0.538779, "cos_c2b/round2": 0.482839, "cos_b2a/round2": 0.100819, "cos_b2c/round2": 0.162483, "rms_a/round2": 3.569832, "rms_b/round2": 7.022337, "rms_c/round2": 3.321482}, "alpha_mean": 0.48521221108424584, "alpha_hist": [0.00885, 0.04442, 0.10365, 0.15868, 0.19383, 0.21457, 0.17056, 0.09096, 0.01427, 0.00021], "alpha_by_type": {"special": {"share": 0.00041, "mean": 0.55621, "std": 0.09145}, "whitespace": {"share": 0.03981, "mean": 0.55069, "std": 0.10205}, "digit": {"share": 0.02444, "mean": 0.53254, "std": 0.13318}, "word_start": {"share": 0.57407, "mean": 0.51296, "std": 0.1543}, "word_cont": {"share": 0.19432, "mean": 0.33438, "std": 0.15016}, "punct": {"share": 0.16695, "mean": 0.54264, "std": 0.15299}}, "alpha_p05": 0.199, "alpha_p50": 0.4803, "alpha_p95": 0.73259}
{"step": 13410, "total_steps": 72479, "tokens": 7030702080, "loss": 4.841489791870117, "l_mix": 2.9995988309383392, "l_a": 3.085137575864792, "l_c": 3.0490602552890778, "z_loss": 16.31613850593567, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.34967154264450073, "tokens_per_second": 1356431.1492958446, "eta_seconds": 22831.359990572928, "data_wait_seconds": 1.817941665649414e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20768, "cosmopedia_v2": 0.20045, "finemath": 0.08233}}
{"step": 13420, "total_steps": 72479, "tokens": 7035944960, "loss": 3.569475829601288, "l_mix": 2.1939040422439575, "l_a": 2.321974277496338, "l_c": 2.258245497941971, "z_loss": 15.057080388069153, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21670496463775635, "tokens_per_second": 1483882.9381597145, "eta_seconds": 20866.82459628582, "data_wait_seconds": 2.3186206817626953e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20768, "cosmopedia_v2": 0.20045, "finemath": 0.08234}}
{"step": 13430, "total_steps": 72479, "tokens": 7041187840, "loss": 3.5901583433151245, "l_mix": 2.2069227397441864, "l_a": 2.33243328332901, "l_c": 2.2732276916503906, "z_loss": 15.37219226360321, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.200850710272789, "tokens_per_second": 1411321.2539151877, "eta_seconds": 21935.95683910847, "data_wait_seconds": 2.8759241104125977e-05, "mix": {"fineweb_hq": 0.50953, "dclm": 0.20767, "cosmopedia_v2": 0.20045, "finemath": 0.08235}}
{"step": 13440, "total_steps": 72479, "tokens": 7046430720, "loss": 3.5830487608909607, "l_mix": 2.201231300830841, "l_a": 2.338914692401886, "l_c": 2.2620272636413574, "z_loss": 15.34770154953003, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.28898316621780396, "tokens_per_second": 1446171.428592473, "eta_seconds": 21403.71370918751, "data_wait_seconds": 2.008676528930664e-05, "mix": {"fineweb_hq": 0.50952, "dclm": 0.20766, "cosmopedia_v2": 0.20045, "finemath": 0.08237}}
{"step": 13450, "total_steps": 72479, "tokens": 7051673600, "loss": 4.618589460849762, "l_mix": 2.857075810432434, "l_a": 2.962061256170273, "l_c": 2.9039701521396637, "z_loss": 17.04263949394226, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2019842118024826, "tokens_per_second": 1415362.692381413, "eta_seconds": 21865.91219239235, "data_wait_seconds": 2.2470951080322266e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20766, "cosmopedia_v2": 0.20045, "finemath": 0.0823}}
{"step": 13460, "total_steps": 72479, "tokens": 7056916480, "loss": 4.600004076957703, "l_mix": 2.845783144235611, "l_a": 2.948072761297226, "l_c": 2.8939009308815002, "z_loss": 16.288010597229004, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20886263251304626, "tokens_per_second": 1401638.9732029093, "eta_seconds": 22076.265046548844, "data_wait_seconds": 1.895427703857422e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20045, "finemath": 0.08232}}
{"step": 13470, "total_steps": 72479, "tokens": 7062159360, "loss": 4.650792479515076, "l_mix": 2.877845048904419, "l_a": 2.9829768240451813, "l_c": 2.921357572078705, "z_loss": 16.46992778778076, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19234023988246918, "tokens_per_second": 1367782.7167659216, "eta_seconds": 22618.87813961506, "data_wait_seconds": 2.181529998779297e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20045, "finemath": 0.08233}}
{"step": 13480, "total_steps": 72479, "tokens": 7067402240, "loss": 4.6639227867126465, "l_mix": 2.884613037109375, "l_a": 2.9915086925029755, "l_c": 2.934084475040436, "z_loss": 16.31832218170166, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1889365017414093, "tokens_per_second": 1410747.1335012012, "eta_seconds": 21926.302012205124, "data_wait_seconds": 1.7464160919189453e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20764, "cosmopedia_v2": 0.20045, "finemath": 0.08234}}
{"step": 13490, "total_steps": 72479, "tokens": 7072645120, "loss": 4.650435745716095, "l_mix": 2.8779301047325134, "l_a": 2.979802906513214, "l_c": 2.9229176342487335, "z_loss": 16.894633769989014, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.23295502364635468, "tokens_per_second": 1418006.8036214183, "eta_seconds": 21810.3501005888, "data_wait_seconds": 3.591179847717285e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08236}}
{"step": 13500, "total_steps": 72479, "tokens": 7077888000, "loss": 3.5769980251789093, "l_mix": 2.198712795972824, "l_a": 2.3269955217838287, "l_c": 2.2620555460453033, "z_loss": 15.699583411216736, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22682704031467438, "tokens_per_second": 1425011.8332094103, "eta_seconds": 21699.456265115736, "data_wait_seconds": 5.182623863220215e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20763, "cosmopedia_v2": 0.20044, "finemath": 0.08237}, "gates": {"gate_a2b/L1_absmean": 0.24272668, "gate_a2b/L1_absmax": 0.64955968, "gate_a2b/L1_rms": 0.26925969, "gate_a2b/L3_absmean": 0.30856901, "gate_a2b/L3_absmax": 0.49529713, "gate_a2b/L3_rms": 0.31383482, "gate_a2b/L5_absmean": 0.49848896, "gate_a2b/L5_absmax": 0.67438519, "gate_a2b/L5_rms": 0.50547153, "gate_c2b/L1_absmean": 0.01438296, "gate_c2b/L1_absmax": 0.49983513, "gate_c2b/L1_rms": 0.058067, "gate_c2b/L3_absmean": 0.20975433, "gate_c2b/L3_absmax": 0.43092585, "gate_c2b/L3_rms": 0.21473657, "gate_c2b/L5_absmean": 0.53729868, "gate_c2b/L5_absmax": 0.74269551, "gate_c2b/L5_rms": 0.54985011, "gate_b2a/L7_absmean": 0.07727608, "gate_b2a/L7_absmax": 0.24892105, "gate_b2a/L7_rms": 0.08077752, "gate_b2a/L11_absmean": 0.12313463, "gate_b2a/L11_absmax": 0.3695243, "gate_b2a/L11_rms": 0.125604, "gate_b2a/L14_absmean": 0.30136615, "gate_b2a/L14_absmax": 0.47139156, "gate_b2a/L14_rms": 0.30248725, "gate_b2c/L3_absmean": 0.07428171, "gate_b2c/L3_absmax": 0.55193204, "gate_b2c/L3_rms": 0.0949711, "gate_b2c/L5_absmean": 0.11045063, "gate_b2c/L5_absmax": 0.64996266, "gate_b2c/L5_rms": 0.11640206, "gate_b2c/L6_absmean": 0.16435248, "gate_b2c/L6_absmax": 0.450811, "gate_b2c/L6_rms": 0.16725132}, "streams": {"cos_a2b/round0": 0.365033, "cos_c2b/round0": 0.235402, "cos_b2a/round0": 0.149006, "cos_b2c/round0": 0.086083, "rms_a/round0": 0.946716, "rms_b/round0": 1.75011, "rms_c/round0": 1.292445, "cos_a2b/round1": 0.391663, "cos_c2b/round1": 0.264882, "cos_b2a/round1": 0.212007, "cos_b2c/round1": 0.14187, "rms_a/round1": 1.679576, "rms_b/round1": 3.357722, "rms_c/round1": 2.180466, "cos_a2b/round2": 0.4921, "cos_c2b/round2": 0.462042, "cos_b2a/round2": 0.039159, "cos_b2c/round2": 0.114575, "rms_a/round2": 3.538574, "rms_b/round2": 7.322007, "rms_c/round2": 3.480752}, "alpha_mean": 0.4462168670737654, "alpha_hist": [0.04611, 0.13531, 0.15887, 0.14041, 0.12152, 0.09979, 0.09927, 0.09978, 0.08443, 0.0145], "alpha_by_type": {"special": {"share": 0.0011, "mean": 0.63394, "std": 0.10031}, "whitespace": {"share": 0.04533, "mean": 0.58854, "std": 0.18953}, "digit": {"share": 0.00961, "mean": 0.41754, "std": 0.17726}, "word_start": {"share": 0.56372, "mean": 0.46867, "std": 0.22839}, "word_cont": {"share": 0.26099, "mean": 0.2743, "std": 0.15969}, "punct": {"share": 0.11925, "mean": 0.66282, "std": 0.2023}}, "alpha_p05": 0.10596, "alpha_p50": 0.41277, "alpha_p95": 0.84897}
{"step": 13510, "total_steps": 72479, "tokens": 7083130880, "loss": 5.010620296001434, "l_mix": 3.104051351547241, "l_a": 3.1963157057762146, "l_c": 3.1533927619457245, "z_loss": 16.563067197799683, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2041589468717575, "tokens_per_second": 1469352.7957278434, "eta_seconds": 21041.059139704706, "data_wait_seconds": 2.0444393157958984e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.2077, "cosmopedia_v2": 0.20044, "finemath": 0.08231}}
{"step": 13520, "total_steps": 72479, "tokens": 7088373760, "loss": 5.058004558086395, "l_mix": 3.134237825870514, "l_a": 3.226064920425415, "l_c": 3.181018739938736, "z_loss": 16.414976835250854, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18618832528591156, "tokens_per_second": 1405009.0730756721, "eta_seconds": 22000.92282986641, "data_wait_seconds": 3.600120544433594e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20769, "cosmopedia_v2": 0.20044, "finemath": 0.08232}}
{"step": 13530, "total_steps": 72479, "tokens": 7093616640, "loss": 5.050961136817932, "l_mix": 3.129909098148346, "l_a": 3.221145421266556, "l_c": 3.176747113466263, "z_loss": 16.840962886810303, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.3002152740955353, "tokens_per_second": 1374115.5704281458, "eta_seconds": 22491.74230837822, "data_wait_seconds": 1.7344951629638672e-05, "mix": {"fineweb_hq": 0.50953, "dclm": 0.20769, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 13540, "total_steps": 72479, "tokens": 7098859520, "loss": 5.008812248706818, "l_mix": 3.10227832198143, "l_a": 3.191923290491104, "l_c": 3.1578462421894073, "z_loss": 16.029760003089905, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.27807533740997314, "tokens_per_second": 1484432.661812081, "eta_seconds": 20816.714174342153, "data_wait_seconds": 2.1249055862426758e-05, "mix": {"fineweb_hq": 0.50953, "dclm": 0.20768, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 13550, "total_steps": 72479, "tokens": 7104102400, "loss": 5.000815510749817, "l_mix": 3.0985363721847534, "l_a": 3.192219853401184, "l_c": 3.1432231068611145, "z_loss": 16.460925579071045, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.24870185554027557, "tokens_per_second": 1414902.1630607632, "eta_seconds": 21835.974499583244, "data_wait_seconds": 1.7464160919189453e-05, "mix": {"fineweb_hq": 0.50952, "dclm": 0.20768, "cosmopedia_v2": 0.20044, "finemath": 0.08236}}
{"step": 13560, "total_steps": 72479, "tokens": 7109345280, "loss": 3.546464592218399, "l_mix": 2.1796659231185913, "l_a": 2.310289442539215, "l_c": 2.24044868350029, "z_loss": 15.772317886352539, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2253357172012329, "tokens_per_second": 1420748.652032615, "eta_seconds": 21742.42757704258, "data_wait_seconds": 1.615285873413086e-05, "mix": {"fineweb_hq": 0.50951, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08237}}
{"step": 13570, "total_steps": 72479, "tokens": 7114588160, "loss": 4.689586341381073, "l_mix": 2.902477502822876, "l_a": 3.000344306230545, "l_c": 2.9512564837932587, "z_loss": 16.2856388092041, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21651148796081543, "tokens_per_second": 1212400.0114413183, "eta_seconds": 25474.498103380203, "data_wait_seconds": 2.434849739074707e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08231}}
{"step": 13580, "total_steps": 72479, "tokens": 7119831040, "loss": 4.595828831195831, "l_mix": 2.841790407896042, "l_a": 2.9486692249774933, "l_c": 2.8925959169864655, "z_loss": 16.587839603424072, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21415366232395172, "tokens_per_second": 1466376.6233844522, "eta_seconds": 21058.73649344444, "data_wait_seconds": 3.287196159362793e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 13590, "total_steps": 72479, "tokens": 7125073920, "loss": 4.739282071590424, "l_mix": 2.932631343603134, "l_a": 3.0344944298267365, "l_c": 2.982162207365036, "z_loss": 16.5351722240448, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2580473721027374, "tokens_per_second": 1283526.8256616031, "eta_seconds": 24054.655823874476, "data_wait_seconds": 2.440810203552246e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 13600, "total_steps": 72479, "tokens": 7130316800, "loss": 4.613226771354675, "l_mix": 2.8533177077770233, "l_a": 2.953602820634842, "l_c": 2.907368630170822, "z_loss": 16.176563501358032, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2606917917728424, "tokens_per_second": 1361174.9926708404, "eta_seconds": 22678.607319569586, "data_wait_seconds": 3.1054019927978516e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.241961, "gate_a2b/L1_absmax": 0.63593215, "gate_a2b/L1_rms": 0.26862562, "gate_a2b/L3_absmean": 0.30802846, "gate_a2b/L3_absmax": 0.49169359, "gate_a2b/L3_rms": 0.31327569, "gate_a2b/L5_absmean": 0.49782822, "gate_a2b/L5_absmax": 0.68401515, "gate_a2b/L5_rms": 0.50470978, "gate_c2b/L1_absmean": 0.01668762, "gate_c2b/L1_absmax": 0.49579582, "gate_c2b/L1_rms": 0.05826978, "gate_c2b/L3_absmean": 0.20991872, "gate_c2b/L3_absmax": 0.42855448, "gate_c2b/L3_rms": 0.21477781, "gate_c2b/L5_absmean": 0.53551161, "gate_c2b/L5_absmax": 0.73722887, "gate_c2b/L5_rms": 0.5480575, "gate_b2a/L7_absmean": 0.07760876, "gate_b2a/L7_absmax": 0.25290594, "gate_b2a/L7_rms": 0.08129589, "gate_b2a/L11_absmean": 0.12384008, "gate_b2a/L11_absmax": 0.37056088, "gate_b2a/L11_rms": 0.12646607, "gate_b2a/L14_absmean": 0.30107555, "gate_b2a/L14_absmax": 0.48414239, "gate_b2a/L14_rms": 0.30233687, "gate_b2c/L3_absmean": 0.07341637, "gate_b2c/L3_absmax": 0.56340545, "gate_b2c/L3_rms": 0.09405807, "gate_b2c/L5_absmean": 0.11007365, "gate_b2c/L5_absmax": 0.65773708, "gate_b2c/L5_rms": 0.11633234, "gate_b2c/L6_absmean": 0.16330811, "gate_b2c/L6_absmax": 0.44922888, "gate_b2c/L6_rms": 0.16635194}, "streams": {"cos_a2b/round0": 0.381741, "cos_c2b/round0": 0.271181, "cos_b2a/round0": 0.164044, "cos_b2c/round0": 0.127315, "rms_a/round0": 0.916618, "rms_b/round0": 1.76242, "rms_c/round0": 1.286862, "cos_a2b/round1": 0.407368, "cos_c2b/round1": 0.275185, "cos_b2a/round1": 0.224499, "cos_b2c/round1": 0.144976, "rms_a/round1": 1.656082, "rms_b/round1": 3.326822, "rms_c/round1": 2.131129, "cos_a2b/round2": 0.522493, "cos_c2b/round2": 0.481827, "cos_b2a/round2": 0.080986, "cos_b2c/round2": 0.140883, "rms_a/round2": 3.583515, "rms_b/round2": 7.287978, "rms_c/round2": 3.446541}, "alpha_mean": 0.42219782787790905, "alpha_hist": [0.01942, 0.07991, 0.16823, 0.19562, 0.19855, 0.17088, 0.10855, 0.0522, 0.00659, 5e-05], "alpha_by_type": {"special": {"share": 0.00124, "mean": 0.51723, "std": 0.08771}, "whitespace": {"share": 0.02534, "mean": 0.49871, "std": 0.11654}, "digit": {"share": 0.02895, "mean": 0.47754, "std": 0.12633}, "word_start": {"share": 0.60612, "mean": 0.43596, "std": 0.15959}, "word_cont": {"share": 0.23424, "mean": 0.31215, "std": 0.14594}, "punct": {"share": 0.10411, "mean": 0.55452, "std": 0.16542}}, "alpha_p05": 0.1188, "alpha_p50": 0.40169, "alpha_p95": 0.72115}
{"step": 13610, "total_steps": 72479, "tokens": 7135559680, "loss": 3.589958757162094, "l_mix": 2.2060996294021606, "l_a": 2.3342714607715607, "l_c": 2.273453652858734, "z_loss": 15.415313243865967, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.25887376070022583, "tokens_per_second": 1433416.2730150018, "eta_seconds": 21531.99377810955, "data_wait_seconds": 2.6851892471313477e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08237}}
{"step": 13620, "total_steps": 72479, "tokens": 7140802560, "loss": 3.607970416545868, "l_mix": 2.231420025229454, "l_a": 2.3283925652503967, "l_c": 2.2541688680648804, "z_loss": 17.819380044937134, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.6079956293106079, "tokens_per_second": 1411494.9129927536, "eta_seconds": 21862.684100341798, "data_wait_seconds": 2.092123031616211e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08238}}
{"step": 13630, "total_steps": 72479, "tokens": 7146045440, "loss": 4.713463664054871, "l_mix": 2.915938079357147, "l_a": 3.021471470594406, "l_c": 2.964702308177948, "z_loss": 16.734104871749878, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.154952734708786, "tokens_per_second": 1378281.1114381456, "eta_seconds": 22385.72687091827, "data_wait_seconds": 1.946091651916504e-05, "mix": {"fineweb_hq": 0.50961, "dclm": 0.20763, "cosmopedia_v2": 0.20044, "finemath": 0.08232}}
{"step": 13640, "total_steps": 72479, "tokens": 7151288320, "loss": 4.680106580257416, "l_mix": 2.895528256893158, "l_a": 2.9989532232284546, "l_c": 2.94413486123085, "z_loss": 16.51961874961853, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17381320893764496, "tokens_per_second": 1472373.6055631163, "eta_seconds": 20951.59918341637, "data_wait_seconds": 1.4841556549072266e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20762, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 13650, "total_steps": 72479, "tokens": 7156531200, "loss": 4.601149260997772, "l_mix": 2.845542460680008, "l_a": 2.9525255858898163, "l_c": 2.89375838637352, "z_loss": 17.21600914001465, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20031961798667908, "tokens_per_second": 1424872.777216791, "eta_seconds": 21646.38081741333, "data_wait_seconds": 2.276897430419922e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20762, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 13660, "total_steps": 72479, "tokens": 7161774080, "loss": 4.602330446243286, "l_mix": 2.846421778202057, "l_a": 2.9479146003723145, "l_c": 2.899559795856476, "z_loss": 16.662875652313232, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20953647792339325, "tokens_per_second": 1414942.6761578568, "eta_seconds": 21794.590262651443, "data_wait_seconds": 1.7315149307250977e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20761, "cosmopedia_v2": 0.20044, "finemath": 0.08236}}
{"step": 13670, "total_steps": 72479, "tokens": 7167016960, "loss": 4.679421126842499, "l_mix": 2.894406884908676, "l_a": 3.0009721219539642, "l_c": 2.9435492753982544, "z_loss": 16.577980279922485, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1930527538061142, "tokens_per_second": 1469165.197997146, "eta_seconds": 20986.648086977006, "data_wait_seconds": 1.6450881958007812e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20761, "cosmopedia_v2": 0.20044, "finemath": 0.08237}}
{"step": 13680, "total_steps": 72479, "tokens": 7172259840, "loss": 4.753643691539764, "l_mix": 2.9417057633399963, "l_a": 3.0454782247543335, "l_c": 2.9888152182102203, "z_loss": 16.498209714889526, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17587652802467346, "tokens_per_second": 1442207.9199476168, "eta_seconds": 21375.288323974608, "data_wait_seconds": 1.8984079360961914e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20768, "cosmopedia_v2": 0.20044, "finemath": 0.08231}}
{"step": 13690, "total_steps": 72479, "tokens": 7177502720, "loss": 4.624212861061096, "l_mix": 2.859932392835617, "l_a": 2.9685168862342834, "l_c": 2.9068860709667206, "z_loss": 16.595059871673584, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18145230412483215, "tokens_per_second": 1408014.4149646722, "eta_seconds": 21890.661703753474, "data_wait_seconds": 2.288818359375e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08232}}
{"step": 13700, "total_steps": 72479, "tokens": 7182745600, "loss": 4.5983277559280396, "l_mix": 2.844164699316025, "l_a": 2.9521554112434387, "l_c": 2.889469861984253, "z_loss": 16.753055095672607, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19234788417816162, "tokens_per_second": 1445812.8712882483, "eta_seconds": 21314.739247369766, "data_wait_seconds": 1.862645149230957e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.24311021, "gate_a2b/L1_absmax": 0.6253863, "gate_a2b/L1_rms": 0.26928964, "gate_a2b/L3_absmean": 0.30899093, "gate_a2b/L3_absmax": 0.48962843, "gate_a2b/L3_rms": 0.31420025, "gate_a2b/L5_absmean": 0.49710917, "gate_a2b/L5_absmax": 0.68452507, "gate_a2b/L5_rms": 0.50391525, "gate_c2b/L1_absmean": 0.01458822, "gate_c2b/L1_absmax": 0.50087553, "gate_c2b/L1_rms": 0.05824103, "gate_c2b/L3_absmean": 0.21117543, "gate_c2b/L3_absmax": 0.43521437, "gate_c2b/L3_rms": 0.21599607, "gate_c2b/L5_absmean": 0.53604919, "gate_c2b/L5_absmax": 0.74078906, "gate_c2b/L5_rms": 0.54873264, "gate_b2a/L7_absmean": 0.07813918, "gate_b2a/L7_absmax": 0.25864407, "gate_b2a/L7_rms": 0.08169343, "gate_b2a/L11_absmean": 0.12413715, "gate_b2a/L11_absmax": 0.36962804, "gate_b2a/L11_rms": 0.12662211, "gate_b2a/L14_absmean": 0.30342743, "gate_b2a/L14_absmax": 0.48909846, "gate_b2a/L14_rms": 0.30467895, "gate_b2c/L3_absmean": 0.07468013, "gate_b2c/L3_absmax": 0.55904043, "gate_b2c/L3_rms": 0.09516129, "gate_b2c/L5_absmean": 0.1107467, "gate_b2c/L5_absmax": 0.65973657, "gate_b2c/L5_rms": 0.11681521, "gate_b2c/L6_absmean": 0.16449997, "gate_b2c/L6_absmax": 0.4424808, "gate_b2c/L6_rms": 0.16743933}, "streams": {"cos_a2b/round0": 0.380017, "cos_c2b/round0": 0.266152, "cos_b2a/round0": 0.160558, "cos_b2c/round0": 0.122245, "rms_a/round0": 0.92737, "rms_b/round0": 1.774333, "rms_c/round0": 1.29717, "cos_a2b/round1": 0.404797, "cos_c2b/round1": 0.273561, "cos_b2a/round1": 0.228584, "cos_b2c/round1": 0.147907, "rms_a/round1": 1.668924, "rms_b/round1": 3.330761, "rms_c/round1": 2.116041, "cos_a2b/round2": 0.522431, "cos_c2b/round2": 0.474882, "cos_b2a/round2": 0.074739, "cos_b2c/round2": 0.14317, "rms_a/round2": 3.562363, "rms_b/round2": 7.209133, "rms_c/round2": 3.382608}, "alpha_mean": 0.45987461946330654, "alpha_hist": [0.01346, 0.06157, 0.14304, 0.18703, 0.1787, 0.16382, 0.13341, 0.09735, 0.0213, 0.00034], "alpha_by_type": {"special": {"share": 0.00153, "mean": 0.57494, "std": 0.09721}, "whitespace": {"share": 0.02689, "mean": 0.55432, "std": 0.12217}, "digit": {"share": 0.0247, "mean": 0.50668, "std": 0.11632}, "word_start": {"share": 0.60013, "mean": 0.4848, "std": 0.17125}, "word_cont": {"share": 0.25217, "mean": 0.32998, "std": 0.14775}, "punct": {"share": 0.09459, "mean": 0.60708, "std": 0.16689}}, "alpha_p05": 0.17583, "alpha_p50": 0.46199, "alpha_p95": 0.78956}
{"step": 13710, "total_steps": 72479, "tokens": 7187988480, "loss": 3.5705446004867554, "l_mix": 2.1941820085048676, "l_a": 2.32666552066803, "l_c": 2.2561406791210175, "z_loss": 15.207538366317749, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18207183480262756, "tokens_per_second": 1420474.7058388838, "eta_seconds": 21691.256694221498, "data_wait_seconds": 2.378225326538086e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 13720, "total_steps": 72479, "tokens": 7193231360, "loss": 3.538581997156143, "l_mix": 2.1753481030464172, "l_a": 2.303836226463318, "l_c": 2.2350562810897827, "z_loss": 15.66091275215149, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20864678919315338, "tokens_per_second": 1422126.6832780172, "eta_seconds": 21662.372947669028, "data_wait_seconds": 2.065300941467285e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08236}}
{"step": 13730, "total_steps": 72479, "tokens": 7198474240, "loss": 3.3213604986667633, "l_mix": 2.051270604133606, "l_a": 2.1392752826213837, "l_c": 2.0882826149463654, "z_loss": 18.224449157714844, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2422475516796112, "tokens_per_second": 1471751.9020314848, "eta_seconds": 20928.388588786125, "data_wait_seconds": 2.6881694793701172e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08237}}
{"step": 13740, "total_steps": 72479, "tokens": 7203717120, "loss": 4.64834988117218, "l_mix": 2.875632494688034, "l_a": 2.9802605509757996, "l_c": 2.923094004392624, "z_loss": 17.110063314437866, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.14607247710227966, "tokens_per_second": 1402866.4076071684, "eta_seconds": 21952.306124806404, "data_wait_seconds": 2.2113323211669922e-05, "mix": {"fineweb_hq": 0.50961, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08231}}
{"step": 13750, "total_steps": 72479, "tokens": 7208960000, "loss": 4.610752820968628, "l_mix": 2.8528718650341034, "l_a": 2.95457261800766, "l_c": 2.899273246526718, "z_loss": 17.27027201652527, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2565765082836151, "tokens_per_second": 1427698.9200841158, "eta_seconds": 21566.80902314186, "data_wait_seconds": 1.609325408935547e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 13760, "total_steps": 72479, "tokens": 7214202880, "loss": 4.643681764602661, "l_mix": 2.872412145137787, "l_a": 2.9750238955020905, "l_c": 2.9235601127147675, "z_loss": 16.94437050819397, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21862225234508514, "tokens_per_second": 1477959.7834875665, "eta_seconds": 20829.840849494933, "data_wait_seconds": 1.958012580871582e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20763, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 13770, "total_steps": 72479, "tokens": 7219445760, "loss": 4.6665841937065125, "l_mix": 2.8872987627983093, "l_a": 2.990290343761444, "l_c": 2.9350383281707764, "z_loss": 16.86705994606018, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2146645039319992, "tokens_per_second": 1447291.2217628176, "eta_seconds": 21267.609261465073, "data_wait_seconds": 1.5497207641601562e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20763, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 13780, "total_steps": 72479, "tokens": 7224688640, "loss": 4.518125295639038, "l_mix": 2.793647736310959, "l_a": 2.8966871798038483, "l_c": 2.8460640013217926, "z_loss": 16.52042269706726, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20151188969612122, "tokens_per_second": 1421791.7150519015, "eta_seconds": 21645.35141553879, "data_wait_seconds": 1.722574234008789e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20762, "cosmopedia_v2": 0.20044, "finemath": 0.08237}}
{"step": 13790, "total_steps": 72479, "tokens": 7229931520, "loss": 5.046234607696533, "l_mix": 3.12559512257576, "l_a": 3.2231395542621613, "l_c": 3.1734269559383392, "z_loss": 16.69450044631958, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17741070687770844, "tokens_per_second": 1457741.3637208538, "eta_seconds": 21107.954536914825, "data_wait_seconds": 4.4792890548706055e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20769, "cosmopedia_v2": 0.20044, "finemath": 0.08231}}
{"step": 13800, "total_steps": 72479, "tokens": 7235174400, "loss": 5.036350667476654, "l_mix": 3.1196740865707397, "l_a": 3.2121580839157104, "l_c": 3.1713198125362396, "z_loss": 16.332056999206543, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17273463308811188, "tokens_per_second": 1352129.1785296455, "eta_seconds": 22752.77838871479, "data_wait_seconds": 8.77082347869873e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20768, "cosmopedia_v2": 0.20043, "finemath": 0.08232}, "gates": {"gate_a2b/L1_absmean": 0.24432468, "gate_a2b/L1_absmax": 0.6341415, "gate_a2b/L1_rms": 0.2704688, "gate_a2b/L3_absmean": 0.30907452, "gate_a2b/L3_absmax": 0.485928, "gate_a2b/L3_rms": 0.31416634, "gate_a2b/L5_absmean": 0.49781138, "gate_a2b/L5_absmax": 0.68296033, "gate_a2b/L5_rms": 0.50466877, "gate_c2b/L1_absmean": 0.01465317, "gate_c2b/L1_absmax": 0.49328431, "gate_c2b/L1_rms": 0.05806791, "gate_c2b/L3_absmean": 0.21107279, "gate_c2b/L3_absmax": 0.43172386, "gate_c2b/L3_rms": 0.21589521, "gate_c2b/L5_absmean": 0.53593528, "gate_c2b/L5_absmax": 0.74357599, "gate_c2b/L5_rms": 0.54853773, "gate_b2a/L7_absmean": 0.0783015, "gate_b2a/L7_absmax": 0.26311344, "gate_b2a/L7_rms": 0.0820168, "gate_b2a/L11_absmean": 0.12441078, "gate_b2a/L11_absmax": 0.37616295, "gate_b2a/L11_rms": 0.12686644, "gate_b2a/L14_absmean": 0.30373782, "gate_b2a/L14_absmax": 0.47922403, "gate_b2a/L14_rms": 0.3049233, "gate_b2c/L3_absmean": 0.07522381, "gate_b2c/L3_absmax": 0.56173646, "gate_b2c/L3_rms": 0.09595819, "gate_b2c/L5_absmean": 0.11109506, "gate_b2c/L5_absmax": 0.65199018, "gate_b2c/L5_rms": 0.1171026, "gate_b2c/L6_absmean": 0.16517559, "gate_b2c/L6_absmax": 0.43986499, "gate_b2c/L6_rms": 0.16806012}, "streams": {"cos_a2b/round0": 0.383051, "cos_c2b/round0": 0.278149, "cos_b2a/round0": 0.161418, "cos_b2c/round0": 0.130408, "rms_a/round0": 0.919546, "rms_b/round0": 1.75533, "rms_c/round0": 1.271288, "cos_a2b/round1": 0.405843, "cos_c2b/round1": 0.276686, "cos_b2a/round1": 0.230294, "cos_b2c/round1": 0.148378, "rms_a/round1": 1.635648, "rms_b/round1": 3.234201, "rms_c/round1": 2.081778, "cos_a2b/round2": 0.535865, "cos_c2b/round2": 0.478265, "cos_b2a/round2": 0.087982, "cos_b2c/round2": 0.144347, "rms_a/round2": 3.503936, "rms_b/round2": 7.007368, "rms_c/round2": 3.299749}, "alpha_mean": 0.46343664093376447, "alpha_hist": [0.0054, 0.0406, 0.11789, 0.20079, 0.22493, 0.20007, 0.1291, 0.06909, 0.01193, 0.00018], "alpha_by_type": {"special": {"share": 0.00072, "mean": 0.54999, "std": 0.1224}, "whitespace": {"share": 0.05782, "mean": 0.47663, "std": 0.09955}, "digit": {"share": 0.01927, "mean": 0.41244, "std": 0.12689}, "word_start": {"share": 0.54628, "mean": 0.48707, "std": 0.15321}, "word_cont": {"share": 0.26323, "mean": 0.36255, "std": 0.13152}, "punct": {"share": 0.11269, "mean": 0.58593, "std": 0.14948}}, "alpha_p05": 0.18625, "alpha_p50": 0.44662, "alpha_p95": 0.73259}
{"step": 13810, "total_steps": 72479, "tokens": 7240417280, "loss": 4.9912795424461365, "l_mix": 3.0916773676872253, "l_a": 3.185898184776306, "l_c": 3.1405398547649384, "z_loss": 16.70721197128296, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17266786098480225, "tokens_per_second": 1363239.0064690246, "eta_seconds": 22563.506858325003, "data_wait_seconds": 1.7911195755004883e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 13820, "total_steps": 72479, "tokens": 7245660160, "loss": 5.0769214034080505, "l_mix": 3.146948218345642, "l_a": 3.231905698776245, "l_c": 3.1960578858852386, "z_loss": 15.841130495071411, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2152683585882187, "tokens_per_second": 1389567.2567780656, "eta_seconds": 22132.221122789382, "data_wait_seconds": 4.303455352783203e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 13830, "total_steps": 72479, "tokens": 7250903040, "loss": 5.127232193946838, "l_mix": 3.176549792289734, "l_a": 3.26964271068573, "l_c": 3.2270917296409607, "z_loss": 16.618601083755493, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2032136172056198, "tokens_per_second": 1334230.4490447976, "eta_seconds": 23046.218840241432, "data_wait_seconds": 2.086162567138672e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08236}}
{"step": 13840, "total_steps": 72479, "tokens": 7256145920, "loss": 3.2961719036102295, "l_mix": 2.035628244280815, "l_a": 2.1247595250606537, "l_c": 2.070806920528412, "z_loss": 18.736119031906128, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.27302417159080505, "tokens_per_second": 1425528.775833141, "eta_seconds": 21566.54046778679, "data_wait_seconds": 3.8504600524902344e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08237}}
{"step": 13850, "total_steps": 72479, "tokens": 7261388800, "loss": 4.523723065853119, "l_mix": 2.7978567481040955, "l_a": 2.9019336104393005, "l_c": 2.8453838527202606, "z_loss": 16.709753274917603, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1548456996679306, "tokens_per_second": 1472412.1528975714, "eta_seconds": 20876.275091528893, "data_wait_seconds": 1.8388032913208008e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08231}}
{"step": 13860, "total_steps": 72479, "tokens": 7266631680, "loss": 4.679640233516693, "l_mix": 2.8968312740325928, "l_a": 2.997540533542633, "l_c": 2.9394534826278687, "z_loss": 17.106328010559082, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2588313817977905, "tokens_per_second": 1406266.700392836, "eta_seconds": 21854.487675356864, "data_wait_seconds": 1.5914440155029297e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08232}}
{"step": 13870, "total_steps": 72479, "tokens": 7271874560, "loss": 4.53741180896759, "l_mix": 2.80621737241745, "l_a": 2.906255781650543, "l_c": 2.8588806688785553, "z_loss": 16.53351402282715, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22091339528560638, "tokens_per_second": 1452011.53009891, "eta_seconds": 21162.363214778903, "data_wait_seconds": 1.5407800674438477e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20764, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 13880, "total_steps": 72479, "tokens": 7277117440, "loss": 4.80581521987915, "l_mix": 2.9744429290294647, "l_a": 3.07859069108963, "l_c": 3.0204035937786102, "z_loss": 16.73940658569336, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22905831038951874, "tokens_per_second": 1463886.2323801287, "eta_seconds": 20987.117600011825, "data_wait_seconds": 1.5974044799804688e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 13890, "total_steps": 72479, "tokens": 7282360320, "loss": 4.633078694343567, "l_mix": 2.866239696741104, "l_a": 2.970871865749359, "l_c": 2.9132281839847565, "z_loss": 16.08939301967621, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20134086906909943, "tokens_per_second": 1377112.6027528686, "eta_seconds": 22305.73561711311, "data_wait_seconds": 2.0951032638549805e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20763, "cosmopedia_v2": 0.20043, "finemath": 0.08236}}
{"step": 13900, "total_steps": 72479, "tokens": 7287603200, "loss": 3.2133124470710754, "l_mix": 1.9856294989585876, "l_a": 2.0680406391620636, "l_c": 2.0183746367692947, "z_loss": 17.583472728729248, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.26434317231178284, "tokens_per_second": 1436666.983234424, "eta_seconds": 21377.443144726753, "data_wait_seconds": 1.874566078186035e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20763, "cosmopedia_v2": 0.20043, "finemath": 0.08237}, "gates": {"gate_a2b/L1_absmean": 0.24500576, "gate_a2b/L1_absmax": 0.62983251, "gate_a2b/L1_rms": 0.27098155, "gate_a2b/L3_absmean": 0.30860779, "gate_a2b/L3_absmax": 0.48508126, "gate_a2b/L3_rms": 0.3139272, "gate_a2b/L5_absmean": 0.49703142, "gate_a2b/L5_absmax": 0.69057548, "gate_a2b/L5_rms": 0.50391734, "gate_c2b/L1_absmean": 0.01602545, "gate_c2b/L1_absmax": 0.4929913, "gate_c2b/L1_rms": 0.05815391, "gate_c2b/L3_absmean": 0.21004649, "gate_c2b/L3_absmax": 0.43322113, "gate_c2b/L3_rms": 0.21481325, "gate_c2b/L5_absmean": 0.53673238, "gate_c2b/L5_absmax": 0.73928928, "gate_c2b/L5_rms": 0.54941118, "gate_b2a/L7_absmean": 0.07834814, "gate_b2a/L7_absmax": 0.26308638, "gate_b2a/L7_rms": 0.08205369, "gate_b2a/L11_absmean": 0.12414546, "gate_b2a/L11_absmax": 0.37603262, "gate_b2a/L11_rms": 0.1266844, "gate_b2a/L14_absmean": 0.30348089, "gate_b2a/L14_absmax": 0.48458126, "gate_b2a/L14_rms": 0.30469111, "gate_b2c/L3_absmean": 0.07468577, "gate_b2c/L3_absmax": 0.56109184, "gate_b2c/L3_rms": 0.0954854, "gate_b2c/L5_absmean": 0.11058304, "gate_b2c/L5_absmax": 0.65208834, "gate_b2c/L5_rms": 0.11670238, "gate_b2c/L6_absmean": 0.16442546, "gate_b2c/L6_absmax": 0.45616212, "gate_b2c/L6_rms": 0.16742872}, "streams": {"cos_a2b/round0": 0.423102, "cos_c2b/round0": 0.312639, "cos_b2a/round0": 0.162944, "cos_b2c/round0": 0.153963, "rms_a/round0": 0.969994, "rms_b/round0": 1.755609, "rms_c/round0": 1.245962, "cos_a2b/round1": 0.375486, "cos_c2b/round1": 0.230707, "cos_b2a/round1": 0.217824, "cos_b2c/round1": 0.125435, "rms_a/round1": 1.655122, "rms_b/round1": 3.289247, "rms_c/round1": 2.128798, "cos_a2b/round2": 0.468772, "cos_c2b/round2": 0.551643, "cos_b2a/round2": 0.107448, "cos_b2c/round2": 0.127746, "rms_a/round2": 3.451163, "rms_b/round2": 7.754819, "rms_c/round2": 3.556435}, "alpha_mean": 0.42130501755023175, "alpha_hist": [0.02393, 0.0468, 0.14653, 0.23232, 0.22583, 0.20856, 0.08635, 0.02626, 0.00323, 0.0002], "alpha_by_type": {"special": {"share": 0.00049, "mean": 0.61508, "std": 0.09637}, "whitespace": {"share": 0.09157, "mean": 0.47143, "std": 0.13642}, "digit": {"share": 0.13681, "mean": 0.45575, "std": 0.12067}, "word_start": {"share": 0.25291, "mean": 0.39883, "std": 0.17411}, "word_cont": {"share": 0.18495, "mean": 0.34039, "std": 0.125}, "punct": {"share": 0.33327, "mean": 0.45507, "std": 0.14184}}, "alpha_p05": 0.16831, "alpha_p50": 0.41632, "alpha_p95": 0.67061}
{"step": 13910, "total_steps": 72479, "tokens": 7292846080, "loss": 5.063683748245239, "l_mix": 3.137288123369217, "l_a": 3.2279874682426453, "l_c": 3.1878757774829865, "z_loss": 16.36667013168335, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17990770936012268, "tokens_per_second": 1418117.2693528237, "eta_seconds": 21653.374185347555, "data_wait_seconds": 1.6182661056518555e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20769, "cosmopedia_v2": 0.20043, "finemath": 0.08231}}
{"step": 13920, "total_steps": 72479, "tokens": 7298088960, "loss": 4.999001324176788, "l_mix": 3.096769332885742, "l_a": 3.1889861822128296, "l_c": 3.1463665664196014, "z_loss": 16.259369254112244, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.166076198220253, "tokens_per_second": 1345204.4004247596, "eta_seconds": 22823.13452312946, "data_wait_seconds": 1.71661376953125e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20769, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 13930, "total_steps": 72479, "tokens": 7303331840, "loss": 5.02583771944046, "l_mix": 3.1134064495563507, "l_a": 3.2033690214157104, "l_c": 3.1659114956855774, "z_loss": 16.46882653236389, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21155285835266113, "tokens_per_second": 1396869.6194183533, "eta_seconds": 21975.23497202396, "data_wait_seconds": 2.086162567138672e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20768, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 13940, "total_steps": 72479, "tokens": 7308574720, "loss": 5.007166385650635, "l_mix": 3.1020544171333313, "l_a": 3.1933400630950928, "l_c": 3.1515315771102905, "z_loss": 16.50386619567871, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1685493439435959, "tokens_per_second": 1205330.8140698494, "eta_seconds": 25462.96408731937, "data_wait_seconds": 2.8878450393676758e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20768, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 13950, "total_steps": 72479, "tokens": 7313817600, "loss": 3.237518757581711, "l_mix": 1.9997271746397018, "l_a": 2.0875396728515625, "l_c": 2.032302513718605, "z_loss": 18.38855528831482, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.24290193617343903, "tokens_per_second": 1391757.7141592794, "eta_seconds": 22048.41549632549, "data_wait_seconds": 2.0205974578857422e-05, "mix": {"fineweb_hq": 0.50953, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08237}}
{"step": 13960, "total_steps": 72479, "tokens": 7319060480, "loss": 4.62601774930954, "l_mix": 2.861867606639862, "l_a": 2.9645878076553345, "l_c": 2.9102788865566254, "z_loss": 16.901039123535156, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17160044610500336, "tokens_per_second": 1446126.7299671972, "eta_seconds": 21215.851167273522, "data_wait_seconds": 1.7315149307250977e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08231}}
{"step": 13970, "total_steps": 72479, "tokens": 7324303360, "loss": 4.669455647468567, "l_mix": 2.8889052867889404, "l_a": 2.9928549230098724, "l_c": 2.9367865324020386, "z_loss": 16.578959703445435, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15100431442260742, "tokens_per_second": 1363159.8240891534, "eta_seconds": 22503.279549407962, "data_wait_seconds": 1.9490718841552734e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08232}}
{"step": 13980, "total_steps": 72479, "tokens": 7329546240, "loss": 4.62402069568634, "l_mix": 2.8600900769233704, "l_a": 2.963004618883133, "l_c": 2.911347985267639, "z_loss": 16.24949324131012, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2250981628894806, "tokens_per_second": 1424210.6485348886, "eta_seconds": 21534.963064312935, "data_wait_seconds": 2.849102020263672e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 13990, "total_steps": 72479, "tokens": 7334789120, "loss": 4.60667610168457, "l_mix": 2.8490301370620728, "l_a": 2.958425283432007, "l_c": 2.894719660282135, "z_loss": 17.024373531341553, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2639846205711365, "tokens_per_second": 1445746.332880526, "eta_seconds": 21210.554116296767, "data_wait_seconds": 1.773238182067871e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 14000, "total_steps": 72479, "tokens": 7340032000, "loss": 4.645443916320801, "l_mix": 2.8742734491825104, "l_a": 2.9697760343551636, "l_c": 2.928772211074829, "z_loss": 16.059520363807678, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.265201210975647, "tokens_per_second": 1450954.4035320831, "eta_seconds": 21130.807334375382, "data_wait_seconds": 2.73287296295166e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20764, "cosmopedia_v2": 0.20043, "finemath": 0.08236}, "gates": {"gate_a2b/L1_absmean": 0.24615598, "gate_a2b/L1_absmax": 0.6273945, "gate_a2b/L1_rms": 0.27229139, "gate_a2b/L3_absmean": 0.30989799, "gate_a2b/L3_absmax": 0.48760357, "gate_a2b/L3_rms": 0.31505063, "gate_a2b/L5_absmean": 0.49787971, "gate_a2b/L5_absmax": 0.68365419, "gate_a2b/L5_rms": 0.50476772, "gate_c2b/L1_absmean": 0.01536051, "gate_c2b/L1_absmax": 0.48921293, "gate_c2b/L1_rms": 0.05779522, "gate_c2b/L3_absmean": 0.21001163, "gate_c2b/L3_absmax": 0.43091553, "gate_c2b/L3_rms": 0.21474433, "gate_c2b/L5_absmean": 0.53669435, "gate_c2b/L5_absmax": 0.73366386, "gate_c2b/L5_rms": 0.54938233, "gate_b2a/L7_absmean": 0.07874259, "gate_b2a/L7_absmax": 0.26056191, "gate_b2a/L7_rms": 0.0823762, "gate_b2a/L11_absmean": 0.12422191, "gate_b2a/L11_absmax": 0.37419528, "gate_b2a/L11_rms": 0.1266962, "gate_b2a/L14_absmean": 0.30345315, "gate_b2a/L14_absmax": 0.48024538, "gate_b2a/L14_rms": 0.30466425, "gate_b2c/L3_absmean": 0.07570873, "gate_b2c/L3_absmax": 0.5589574, "gate_b2c/L3_rms": 0.09625878, "gate_b2c/L5_absmean": 0.11111691, "gate_b2c/L5_absmax": 0.65551251, "gate_b2c/L5_rms": 0.11718, "gate_b2c/L6_absmean": 0.16511261, "gate_b2c/L6_absmax": 0.45407584, "gate_b2c/L6_rms": 0.16812964}, "streams": {"cos_a2b/round0": 0.377911, "cos_c2b/round0": 0.27749, "cos_b2a/round0": 0.156988, "cos_b2c/round0": 0.127938, "rms_a/round0": 0.917011, "rms_b/round0": 1.755764, "rms_c/round0": 1.285277, "cos_a2b/round1": 0.402698, "cos_c2b/round1": 0.264055, "cos_b2a/round1": 0.220748, "cos_b2c/round1": 0.143757, "rms_a/round1": 1.665128, "rms_b/round1": 3.310069, "rms_c/round1": 2.105843, "cos_a2b/round2": 0.509004, "cos_c2b/round2": 0.48574, "cos_b2a/round2": 0.07247, "cos_b2c/round2": 0.131097, "rms_a/round2": 3.525316, "rms_b/round2": 7.229817, "rms_c/round2": 3.404863}, "alpha_mean": 0.4146682066491536, "alpha_hist": [0.03006, 0.10088, 0.16936, 0.18329, 0.1783, 0.16267, 0.1086, 0.05739, 0.00934, 0.00012], "alpha_by_type": {"special": {"share": 0.00114, "mean": 0.56921, "std": 0.1049}, "whitespace": {"share": 0.03206, "mean": 0.52936, "std": 0.131}, "digit": {"share": 0.03366, "mean": 0.46043, "std": 0.12455}, "word_start": {"share": 0.58368, "mean": 0.43428, "std": 0.16828}, "word_cont": {"share": 0.24513, "mean": 0.28752, "std": 0.15142}, "punct": {"share": 0.10432, "mean": 0.55198, "std": 0.17185}}, "alpha_p05": 0.10781, "alpha_p50": 0.38908, "alpha_p95": 0.72643}
{"step": 14010, "total_steps": 72479, "tokens": 7345274880, "loss": 3.2008389234542847, "l_mix": 1.9775839000940323, "l_a": 2.0548941642045975, "l_c": 2.0168553441762924, "z_loss": 17.30147624015808, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2633759677410126, "tokens_per_second": 1445249.2946857123, "eta_seconds": 21210.593345189092, "data_wait_seconds": 3.135204315185547e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20764, "cosmopedia_v2": 0.20043, "finemath": 0.08237}}
{"step": 14020, "total_steps": 72479, "tokens": 7350517760, "loss": 3.237045556306839, "l_mix": 2.0013328343629837, "l_a": 2.0750249922275543, "l_c": 2.0379310697317123, "z_loss": 18.25898551940918, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.29444408416748047, "tokens_per_second": 1432720.5117292572, "eta_seconds": 21392.41529738903, "data_wait_seconds": 1.7851591110229492e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20763, "cosmopedia_v2": 0.20043, "finemath": 0.08238}}
{"step": 14030, "total_steps": 72479, "tokens": 7355760640, "loss": 4.991151809692383, "l_mix": 3.0928750336170197, "l_a": 3.18166247010231, "l_c": 3.1404260993003845, "z_loss": 16.502734541893005, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17070406675338745, "tokens_per_second": 1424408.8073212414, "eta_seconds": 21513.563489985467, "data_wait_seconds": 2.053380012512207e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.2077, "cosmopedia_v2": 0.20043, "finemath": 0.08232}}
{"step": 14040, "total_steps": 72479, "tokens": 7361003520, "loss": 4.948468565940857, "l_mix": 3.0648875534534454, "l_a": 3.1604118645191193, "l_c": 3.112814426422119, "z_loss": 16.130543112754822, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2142048478126526, "tokens_per_second": 1368659.0448404434, "eta_seconds": 22386.047531342505, "data_wait_seconds": 1.9222497940063477e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20769, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 14050, "total_steps": 72479, "tokens": 7366246400, "loss": 4.929011881351471, "l_mix": 3.0535088777542114, "l_a": 3.143306404352188, "l_c": 3.1028000116348267, "z_loss": 16.709295511245728, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2587336003780365, "tokens_per_second": 1350237.664829487, "eta_seconds": 22687.578898096082, "data_wait_seconds": 2.872943878173828e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20769, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 14060, "total_steps": 72479, "tokens": 7371489280, "loss": 5.024608016014099, "l_mix": 3.1128638684749603, "l_a": 3.205041319131851, "l_c": 3.1620211005210876, "z_loss": 16.25383472442627, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1941412091255188, "tokens_per_second": 1444006.344073326, "eta_seconds": 21210.696751928328, "data_wait_seconds": 1.621246337890625e-05, "mix": {"fineweb_hq": 0.50953, "dclm": 0.20768, "cosmopedia_v2": 0.20043, "finemath": 0.08236}}
{"step": 14070, "total_steps": 72479, "tokens": 7376732160, "loss": 4.62556654214859, "l_mix": 2.861891746520996, "l_a": 2.963542550802231, "l_c": 2.909823626279831, "z_loss": 16.650449752807617, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1773274540901184, "tokens_per_second": 1445383.6161695898, "eta_seconds": 21186.858249545097, "data_wait_seconds": 2.7447938919067383e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20768, "cosmopedia_v2": 0.20043, "finemath": 0.0823}}
{"step": 14080, "total_steps": 72479, "tokens": 7381975040, "loss": 4.678582310676575, "l_mix": 2.8944800794124603, "l_a": 2.9995229840278625, "l_c": 2.941921830177307, "z_loss": 16.686774253845215, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17150583863258362, "tokens_per_second": 1376128.2727926734, "eta_seconds": 22249.303002738954, "data_wait_seconds": 2.658367156982422e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08232}}
{"step": 14090, "total_steps": 72479, "tokens": 7387217920, "loss": 4.641824305057526, "l_mix": 2.8720485866069794, "l_a": 2.97380593419075, "l_c": 2.919995456933975, "z_loss": 16.35255515575409, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15413784980773926, "tokens_per_second": 1460179.458441833, "eta_seconds": 20964.992936325074, "data_wait_seconds": 1.6689300537109375e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 14100, "total_steps": 72479, "tokens": 7392460800, "loss": 4.585829496383667, "l_mix": 2.8373952209949493, "l_a": 2.939743459224701, "l_c": 2.882902204990387, "z_loss": 16.405503273010254, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16609106957912445, "tokens_per_second": 1313284.696060522, "eta_seconds": 23305.996973705292, "data_wait_seconds": 1.823902130126953e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.2471938, "gate_a2b/L1_absmax": 0.63477534, "gate_a2b/L1_rms": 0.27324888, "gate_a2b/L3_absmean": 0.3102603, "gate_a2b/L3_absmax": 0.48527768, "gate_a2b/L3_rms": 0.31544945, "gate_a2b/L5_absmean": 0.49826899, "gate_a2b/L5_absmax": 0.68327385, "gate_a2b/L5_rms": 0.50507325, "gate_c2b/L1_absmean": 0.01543382, "gate_c2b/L1_absmax": 0.48790938, "gate_c2b/L1_rms": 0.05788343, "gate_c2b/L3_absmean": 0.20973773, "gate_c2b/L3_absmax": 0.43507475, "gate_c2b/L3_rms": 0.21457098, "gate_c2b/L5_absmean": 0.53666371, "gate_c2b/L5_absmax": 0.74617052, "gate_c2b/L5_rms": 0.54938018, "gate_b2a/L7_absmean": 0.0783897, "gate_b2a/L7_absmax": 0.25889531, "gate_b2a/L7_rms": 0.08202475, "gate_b2a/L11_absmean": 0.12409739, "gate_b2a/L11_absmax": 0.37403473, "gate_b2a/L11_rms": 0.12648593, "gate_b2a/L14_absmean": 0.30392593, "gate_b2a/L14_absmax": 0.48427585, "gate_b2a/L14_rms": 0.30512884, "gate_b2c/L3_absmean": 0.07621285, "gate_b2c/L3_absmax": 0.55966592, "gate_b2c/L3_rms": 0.09672979, "gate_b2c/L5_absmean": 0.11170352, "gate_b2c/L5_absmax": 0.65490055, "gate_b2c/L5_rms": 0.11774737, "gate_b2c/L6_absmean": 0.16566229, "gate_b2c/L6_absmax": 0.45674172, "gate_b2c/L6_rms": 0.16867246}, "streams": {"cos_a2b/round0": 0.378059, "cos_c2b/round0": 0.281456, "cos_b2a/round0": 0.162516, "cos_b2c/round0": 0.135256, "rms_a/round0": 0.900336, "rms_b/round0": 1.734513, "rms_c/round0": 1.278285, "cos_a2b/round1": 0.409711, "cos_c2b/round1": 0.27936, "cos_b2a/round1": 0.229682, "cos_b2c/round1": 0.145816, "rms_a/round1": 1.627954, "rms_b/round1": 3.238546, "rms_c/round1": 2.08812, "cos_a2b/round2": 0.53413, "cos_c2b/round2": 0.479705, "cos_b2a/round2": 0.081852, "cos_b2c/round2": 0.151008, "rms_a/round2": 3.529505, "rms_b/round2": 7.03197, "rms_c/round2": 3.330631}, "alpha_mean": 0.45647146275491934, "alpha_hist": [0.01263, 0.05673, 0.13947, 0.18559, 0.19151, 0.18674, 0.13449, 0.07777, 0.01494, 0.00012], "alpha_by_type": {"special": {"share": 0.00081, "mean": 0.54317, "std": 0.09835}, "whitespace": {"share": 0.02243, "mean": 0.53016, "std": 0.11068}, "digit": {"share": 0.01886, "mean": 0.46914, "std": 0.10712}, "word_start": {"share": 0.61992, "mean": 0.48317, "std": 0.15788}, "word_cont": {"share": 0.235, "mean": 0.31769, "std": 0.13753}, "punct": {"share": 0.10298, "mean": 0.59343, "std": 0.1646}}, "alpha_p05": 0.19559, "alpha_p50": 0.46685, "alpha_p95": 0.73829}
{"step": 14110, "total_steps": 72479, "tokens": 7397703680, "loss": 4.805414617061615, "l_mix": 2.9749421775341034, "l_a": 3.0730974078178406, "l_c": 3.0228568017482758, "z_loss": 16.860551118850708, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22757194936275482, "tokens_per_second": 1458651.4532180468, "eta_seconds": 20979.76607398987, "data_wait_seconds": 4.1991472244262695e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 14120, "total_steps": 72479, "tokens": 7402946560, "loss": 3.210511267185211, "l_mix": 1.984583169221878, "l_a": 2.063901349902153, "l_c": 2.016522705554962, "z_loss": 18.00876474380493, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2287473976612091, "tokens_per_second": 1342331.22533523, "eta_seconds": 22793.86995885372, "data_wait_seconds": 2.333521842956543e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08237}}
{"step": 14130, "total_steps": 72479, "tokens": 7408189440, "loss": 3.2423061430454254, "l_mix": 2.0037536323070526, "l_a": 2.0831989645957947, "l_c": 2.039186105132103, "z_loss": 18.36950421333313, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2257322371006012, "tokens_per_second": 1452630.5781857753, "eta_seconds": 21059.504716062547, "data_wait_seconds": 1.6450881958007812e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20764, "cosmopedia_v2": 0.20042, "finemath": 0.08238}}
{"step": 14140, "total_steps": 72479, "tokens": 7413432320, "loss": 4.582826018333435, "l_mix": 2.8341706097126007, "l_a": 2.9384846687316895, "l_c": 2.8846420347690582, "z_loss": 17.17367434501648, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16106654703617096, "tokens_per_second": 1306883.4502594364, "eta_seconds": 23404.105106639865, "data_wait_seconds": 1.9758939743041992e-05, "mix": {"fineweb_hq": 0.50962, "dclm": 0.20764, "cosmopedia_v2": 0.20042, "finemath": 0.08232}}
{"step": 14150, "total_steps": 72479, "tokens": 7418675200, "loss": 4.54065865278244, "l_mix": 2.8088513910770416, "l_a": 2.911424160003662, "l_c": 2.8557773530483246, "z_loss": 16.46656632423401, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17715144157409668, "tokens_per_second": 1290259.0028509786, "eta_seconds": 23701.593776464462, "data_wait_seconds": 1.7374753952026367e-05, "mix": {"fineweb_hq": 0.50961, "dclm": 0.20763, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 14160, "total_steps": 72479, "tokens": 7423918080, "loss": 4.641042649745941, "l_mix": 2.8704839050769806, "l_a": 2.975242018699646, "l_c": 2.9208729565143585, "z_loss": 17.240482807159424, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19418390095233917, "tokens_per_second": 1402202.3050788837, "eta_seconds": 21805.66367723942, "data_wait_seconds": 1.728534698486328e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20763, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 14170, "total_steps": 72479, "tokens": 7429160960, "loss": 4.494942247867584, "l_mix": 2.7804519832134247, "l_a": 2.8831619322299957, "l_c": 2.8261739313602448, "z_loss": 16.89547348022461, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17597395181655884, "tokens_per_second": 1330086.6752201002, "eta_seconds": 22983.997630786893, "data_wait_seconds": 1.7464160919189453e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20762, "cosmopedia_v2": 0.20042, "finemath": 0.08236}}
{"step": 14180, "total_steps": 72479, "tokens": 7434403840, "loss": 4.448735415935516, "l_mix": 2.752222955226898, "l_a": 2.852456659078598, "l_c": 2.7970124185085297, "z_loss": 16.71653187274933, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19936911761760712, "tokens_per_second": 1431641.691251603, "eta_seconds": 21349.94132874012, "data_wait_seconds": 1.8715858459472656e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20762, "cosmopedia_v2": 0.20042, "finemath": 0.08237}}
{"step": 14190, "total_steps": 72479, "tokens": 7439646720, "loss": 4.581084311008453, "l_mix": 2.8339011669158936, "l_a": 2.935152590274811, "l_c": 2.883183926343918, "z_loss": 16.82187509536743, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17385171353816986, "tokens_per_second": 1432013.302508797, "eta_seconds": 21340.739767193794, "data_wait_seconds": 1.6361474990844727e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20768, "cosmopedia_v2": 0.20042, "finemath": 0.08231}}
{"step": 14200, "total_steps": 72479, "tokens": 7444889600, "loss": 4.625400900840759, "l_mix": 2.8618715703487396, "l_a": 2.9631995856761932, "l_c": 2.9096101224422455, "z_loss": 16.862187147140503, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16411727666854858, "tokens_per_second": 1407766.6255877155, "eta_seconds": 21704.57787294388, "data_wait_seconds": 1.6570091247558594e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20768, "cosmopedia_v2": 0.20042, "finemath": 0.08232}, "gates": {"gate_a2b/L1_absmean": 0.2487537, "gate_a2b/L1_absmax": 0.64402449, "gate_a2b/L1_rms": 0.27448985, "gate_a2b/L3_absmean": 0.31227598, "gate_a2b/L3_absmax": 0.4855516, "gate_a2b/L3_rms": 0.31727546, "gate_a2b/L5_absmean": 0.49976912, "gate_a2b/L5_absmax": 0.68318653, "gate_a2b/L5_rms": 0.5065369, "gate_c2b/L1_absmean": 0.01513642, "gate_c2b/L1_absmax": 0.48449504, "gate_c2b/L1_rms": 0.05737226, "gate_c2b/L3_absmean": 0.20936118, "gate_c2b/L3_absmax": 0.43264705, "gate_c2b/L3_rms": 0.21416843, "gate_c2b/L5_absmean": 0.53657359, "gate_c2b/L5_absmax": 0.73571986, "gate_c2b/L5_rms": 0.54916793, "gate_b2a/L7_absmean": 0.07836688, "gate_b2a/L7_absmax": 0.25621167, "gate_b2a/L7_rms": 0.08191827, "gate_b2a/L11_absmean": 0.12415317, "gate_b2a/L11_absmax": 0.37336385, "gate_b2a/L11_rms": 0.12658373, "gate_b2a/L14_absmean": 0.30389047, "gate_b2a/L14_absmax": 0.48668447, "gate_b2a/L14_rms": 0.30509102, "gate_b2c/L3_absmean": 0.07661751, "gate_b2c/L3_absmax": 0.55770606, "gate_b2c/L3_rms": 0.09724542, "gate_b2c/L5_absmean": 0.11245029, "gate_b2c/L5_absmax": 0.65819144, "gate_b2c/L5_rms": 0.11849321, "gate_b2c/L6_absmean": 0.16641515, "gate_b2c/L6_absmax": 0.46001622, "gate_b2c/L6_rms": 0.16942596}, "streams": {"cos_a2b/round0": 0.379833, "cos_c2b/round0": 0.273348, "cos_b2a/round0": 0.159285, "cos_b2c/round0": 0.127076, "rms_a/round0": 0.904877, "rms_b/round0": 1.739506, "rms_c/round0": 1.282057, "cos_a2b/round1": 0.403243, "cos_c2b/round1": 0.273033, "cos_b2a/round1": 0.228072, "cos_b2c/round1": 0.148471, "rms_a/round1": 1.634615, "rms_b/round1": 3.262313, "rms_c/round1": 2.092387, "cos_a2b/round2": 0.520426, "cos_c2b/round2": 0.485811, "cos_b2a/round2": 0.077099, "cos_b2c/round2": 0.137643, "rms_a/round2": 3.506383, "rms_b/round2": 7.154012, "rms_c/round2": 3.384342}, "alpha_mean": 0.4401929135280156, "alpha_hist": [0.0152, 0.07332, 0.16, 0.18878, 0.18864, 0.16307, 0.12102, 0.07375, 0.01598, 0.00024], "alpha_by_type": {"special": {"share": 0.00117, "mean": 0.54803, "std": 0.11193}, "whitespace": {"share": 0.02734, "mean": 0.53895, "std": 0.11953}, "digit": {"share": 0.02596, "mean": 0.48119, "std": 0.12368}, "word_start": {"share": 0.58464, "mean": 0.45443, "std": 0.17212}, "word_cont": {"share": 0.24258, "mean": 0.32446, "std": 0.14228}, "punct": {"share": 0.1183, "mean": 0.57425, "std": 0.1644}}, "alpha_p05": 0.16941, "alpha_p50": 0.44782, "alpha_p95": 0.74391}
{"step": 14210, "total_steps": 72479, "tokens": 7450132480, "loss": 4.611779510974884, "l_mix": 2.8526680171489716, "l_a": 2.9583474695682526, "l_c": 2.8998507261276245, "z_loss": 16.520638465881348, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16494786739349365, "tokens_per_second": 1408842.7850278562, "eta_seconds": 21684.277193069458, "data_wait_seconds": 1.7434358596801758e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 14220, "total_steps": 72479, "tokens": 7455375360, "loss": 4.597771465778351, "l_mix": 2.8447156250476837, "l_a": 2.94494765996933, "l_c": 2.892963021993637, "z_loss": 16.825101375579834, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16220985352993011, "tokens_per_second": 1384702.0164435876, "eta_seconds": 22058.53261516094, "data_wait_seconds": 2.193450927734375e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 14230, "total_steps": 72479, "tokens": 7460618240, "loss": 4.624124646186829, "l_mix": 2.8610746264457703, "l_a": 2.961952418088913, "l_c": 2.909257799386978, "z_loss": 16.869573831558228, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15944945812225342, "tokens_per_second": 1433817.2261322325, "eta_seconds": 21299.26405918598, "data_wait_seconds": 2.2292137145996094e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08236}}
{"step": 14240, "total_steps": 72479, "tokens": 7465861120, "loss": 3.2453351616859436, "l_mix": 2.005725920200348, "l_a": 2.0864760279655457, "l_c": 2.0394866466522217, "z_loss": 18.2044677734375, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2552158534526825, "tokens_per_second": 1386144.0361465225, "eta_seconds": 22028.02020263672, "data_wait_seconds": 1.9609928131103516e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08237}}
{"step": 14250, "total_steps": 72479, "tokens": 7471104000, "loss": 4.569799602031708, "l_mix": 2.826481282711029, "l_a": 2.929583638906479, "l_c": 2.87584525346756, "z_loss": 16.897010564804077, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15709440410137177, "tokens_per_second": 1462626.2087420225, "eta_seconds": 20872.568650507925, "data_wait_seconds": 2.6881694793701172e-05, "mix": {"fineweb_hq": 0.50961, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08232}}
{"step": 14260, "total_steps": 72479, "tokens": 7476346880, "loss": 4.546374261379242, "l_mix": 2.811821699142456, "l_a": 2.9174606800079346, "l_c": 2.858861893415451, "z_loss": 16.55640745162964, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16413098573684692, "tokens_per_second": 1472376.6616719768, "eta_seconds": 20730.784361481667, "data_wait_seconds": 1.4871358871459961e-05, "mix": {"fineweb_hq": 0.50961, "dclm": 0.20764, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 14270, "total_steps": 72479, "tokens": 7481589760, "loss": 4.6963210105896, "l_mix": 2.905429780483246, "l_a": 3.010721653699875, "l_c": 2.9532854855060577, "z_loss": 16.889836072921753, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17588958144187927, "tokens_per_second": 1351458.9090787517, "eta_seconds": 22581.730000805856, "data_wait_seconds": 1.940131187438965e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20764, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 14280, "total_steps": 72479, "tokens": 7486832640, "loss": 4.595107197761536, "l_mix": 2.842781722545624, "l_a": 2.9429629743099213, "l_c": 2.892512857913971, "z_loss": 16.82569932937622, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17312860488891602, "tokens_per_second": 1448844.8569430725, "eta_seconds": 21060.251665854452, "data_wait_seconds": 1.8090009689331055e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20763, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 14290, "total_steps": 72479, "tokens": 7492075520, "loss": 4.618350803852081, "l_mix": 2.85839307308197, "l_a": 2.9582373201847076, "l_c": 2.9027668833732605, "z_loss": 16.56388282775879, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1957116425037384, "tokens_per_second": 1473313.0195287955, "eta_seconds": 20706.933304476737, "data_wait_seconds": 1.5974044799804688e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20763, "cosmopedia_v2": 0.20042, "finemath": 0.08237}}
{"step": 14300, "total_steps": 72479, "tokens": 7497318400, "loss": 4.6231467723846436, "l_mix": 2.859086185693741, "l_a": 2.9638056457042694, "l_c": 2.910795420408249, "z_loss": 16.802125453948975, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22077786922454834, "tokens_per_second": 1324708.1760509512, "eta_seconds": 23025.864944028854, "data_wait_seconds": 1.6450881958007812e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20762, "cosmopedia_v2": 0.20042, "finemath": 0.08238}, "gates": {"gate_a2b/L1_absmean": 0.25060079, "gate_a2b/L1_absmax": 0.6425609, "gate_a2b/L1_rms": 0.27608195, "gate_a2b/L3_absmean": 0.31317452, "gate_a2b/L3_absmax": 0.48747966, "gate_a2b/L3_rms": 0.31823179, "gate_a2b/L5_absmean": 0.50087875, "gate_a2b/L5_absmax": 0.68216324, "gate_a2b/L5_rms": 0.50764114, "gate_c2b/L1_absmean": 0.01461965, "gate_c2b/L1_absmax": 0.48296142, "gate_c2b/L1_rms": 0.057025, "gate_c2b/L3_absmean": 0.20974636, "gate_c2b/L3_absmax": 0.43412489, "gate_c2b/L3_rms": 0.21454008, "gate_c2b/L5_absmean": 0.53664237, "gate_c2b/L5_absmax": 0.73398948, "gate_c2b/L5_rms": 0.54916567, "gate_b2a/L7_absmean": 0.07825071, "gate_b2a/L7_absmax": 0.25655383, "gate_b2a/L7_rms": 0.08185463, "gate_b2a/L11_absmean": 0.12451941, "gate_b2a/L11_absmax": 0.37611434, "gate_b2a/L11_rms": 0.12693895, "gate_b2a/L14_absmean": 0.30449453, "gate_b2a/L14_absmax": 0.48638102, "gate_b2a/L14_rms": 0.30566975, "gate_b2c/L3_absmean": 0.07732256, "gate_b2c/L3_absmax": 0.55995905, "gate_b2c/L3_rms": 0.09768355, "gate_b2c/L5_absmean": 0.11284113, "gate_b2c/L5_absmax": 0.65994543, "gate_b2c/L5_rms": 0.11893836, "gate_b2c/L6_absmean": 0.16716264, "gate_b2c/L6_absmax": 0.45800844, "gate_b2c/L6_rms": 0.17020498}, "streams": {"cos_a2b/round0": 0.384252, "cos_c2b/round0": 0.280839, "cos_b2a/round0": 0.16126, "cos_b2c/round0": 0.134216, "rms_a/round0": 0.889463, "rms_b/round0": 1.711486, "rms_c/round0": 1.263151, "cos_a2b/round1": 0.409128, "cos_c2b/round1": 0.266911, "cos_b2a/round1": 0.227029, "cos_b2c/round1": 0.145222, "rms_a/round1": 1.616224, "rms_b/round1": 3.245226, "rms_c/round1": 2.084604, "cos_a2b/round2": 0.524545, "cos_c2b/round2": 0.476916, "cos_b2a/round2": 0.068584, "cos_b2c/round2": 0.127648, "rms_a/round2": 3.474999, "rms_b/round2": 7.071094, "rms_c/round2": 3.336232}, "alpha_mean": 0.441161669847304, "alpha_hist": [0.01968, 0.07451, 0.15352, 0.18184, 0.18619, 0.16887, 0.12686, 0.07402, 0.01425, 0.00026], "alpha_by_type": {"special": {"share": 0.00099, "mean": 0.58877, "std": 0.09623}, "whitespace": {"share": 0.02747, "mean": 0.55741, "std": 0.12184}, "digit": {"share": 0.03702, "mean": 0.51609, "std": 0.14219}, "word_start": {"share": 0.58072, "mean": 0.4581, "std": 0.16776}, "word_cont": {"share": 0.24811, "mean": 0.31403, "std": 0.14205}, "punct": {"share": 0.1057, "mean": 0.58867, "std": 0.16988}}, "alpha_p05": 0.15611, "alpha_p50": 0.4187, "alpha_p95": 0.73181}
{"step": 14310, "total_steps": 72479, "tokens": 7502561280, "loss": 4.9626622796058655, "l_mix": 3.0743268430233, "l_a": 3.1670672595500946, "l_c": 3.1219473481178284, "z_loss": 16.30962884426117, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15173085033893585, "tokens_per_second": 1442324.0806622526, "eta_seconds": 21144.560422229766, "data_wait_seconds": 2.1398067474365234e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20769, "cosmopedia_v2": 0.20042, "finemath": 0.08232}}
{"step": 14320, "total_steps": 72479, "tokens": 7507804160, "loss": 4.978833258152008, "l_mix": 3.084016263484955, "l_a": 3.1751990020275116, "l_c": 3.135395348072052, "z_loss": 16.385713934898376, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17457175254821777, "tokens_per_second": 1459912.2917021294, "eta_seconds": 20886.231293010715, "data_wait_seconds": 1.84476375579834e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20768, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 14330, "total_steps": 72479, "tokens": 7513047040, "loss": 4.887875139713287, "l_mix": 3.0276039838790894, "l_a": 3.1180010437965393, "l_c": 3.077456533908844, "z_loss": 16.337265014648438, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1784171611070633, "tokens_per_second": 1205021.7011493396, "eta_seconds": 25299.812346053124, "data_wait_seconds": 2.130866050720215e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 14340, "total_steps": 72479, "tokens": 7518289920, "loss": 5.069771766662598, "l_mix": 3.1412993669509888, "l_a": 3.2303291857242584, "l_c": 3.1925454437732697, "z_loss": 16.10073173046112, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17992301285266876, "tokens_per_second": 1434342.2622436516, "eta_seconds": 21251.259782528876, "data_wait_seconds": 2.7447938919067383e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08236}}
{"step": 14350, "total_steps": 72479, "tokens": 7523532800, "loss": 3.3128509521484375, "l_mix": 2.0456262975931168, "l_a": 2.1239795833826065, "l_c": 2.094139948487282, "z_loss": 17.887240409851074, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.3847169876098633, "tokens_per_second": 1473849.9990127522, "eta_seconds": 20678.045372605324, "data_wait_seconds": 3.0040740966796875e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08237}}
{"step": 14360, "total_steps": 72479, "tokens": 7528775680, "loss": 4.559004545211792, "l_mix": 2.8196685314178467, "l_a": 2.9255981743335724, "l_c": 2.8666180968284607, "z_loss": 16.710206508636475, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1411142647266388, "tokens_per_second": 1435842.142580294, "eta_seconds": 21221.757857894896, "data_wait_seconds": 1.940131187438965e-05, "mix": {"fineweb_hq": 0.50961, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08231}}
{"step": 14370, "total_steps": 72479, "tokens": 7534018560, "loss": 4.562249541282654, "l_mix": 2.8222701251506805, "l_a": 2.926836669445038, "l_c": 2.8675279319286346, "z_loss": 16.69934916496277, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19049257040023804, "tokens_per_second": 1226111.981229924, "eta_seconds": 24847.527679681778, "data_wait_seconds": 2.300739288330078e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08232}}
{"step": 14380, "total_steps": 72479, "tokens": 7539261440, "loss": 4.649465441703796, "l_mix": 2.87702739238739, "l_a": 2.9741635620594025, "l_c": 2.9284406900405884, "z_loss": 16.56786298751831, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1624096930027008, "tokens_per_second": 1459174.214037957, "eta_seconds": 20875.237664532662, "data_wait_seconds": 1.6748905181884766e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 14390, "total_steps": 72479, "tokens": 7544504320, "loss": 4.710675239562988, "l_mix": 2.9142265617847443, "l_a": 3.01716810464859, "l_c": 2.965438097715378, "z_loss": 16.66684913635254, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19330932199954987, "tokens_per_second": 1454334.483044782, "eta_seconds": 20941.101230192187, "data_wait_seconds": 2.047419548034668e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20764, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 14400, "total_steps": 72479, "tokens": 7549747200, "loss": 4.611692726612091, "l_mix": 2.8534737527370453, "l_a": 2.9536398351192474, "l_c": 2.901513397693634, "z_loss": 16.72813653945923, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16765139997005463, "tokens_per_second": 1410013.7072863018, "eta_seconds": 21595.62179760933, "data_wait_seconds": 2.2917985916137695e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20042, "finemath": 0.08236}, "gates": {"gate_a2b/L1_absmean": 0.25202128, "gate_a2b/L1_absmax": 0.64222246, "gate_a2b/L1_rms": 0.27758503, "gate_a2b/L3_absmean": 0.31318241, "gate_a2b/L3_absmax": 0.48914859, "gate_a2b/L3_rms": 0.31822884, "gate_a2b/L5_absmean": 0.50088036, "gate_a2b/L5_absmax": 0.68511671, "gate_a2b/L5_rms": 0.50765479, "gate_c2b/L1_absmean": 0.01493127, "gate_c2b/L1_absmax": 0.48436788, "gate_c2b/L1_rms": 0.05718254, "gate_c2b/L3_absmean": 0.2093643, "gate_c2b/L3_absmax": 0.43354878, "gate_c2b/L3_rms": 0.21419045, "gate_c2b/L5_absmean": 0.53541452, "gate_c2b/L5_absmax": 0.7311005, "gate_c2b/L5_rms": 0.54775733, "gate_b2a/L7_absmean": 0.07838447, "gate_b2a/L7_absmax": 0.25703529, "gate_b2a/L7_rms": 0.08192781, "gate_b2a/L11_absmean": 0.12431974, "gate_b2a/L11_absmax": 0.37846434, "gate_b2a/L11_rms": 0.12673838, "gate_b2a/L14_absmean": 0.30401587, "gate_b2a/L14_absmax": 0.48324174, "gate_b2a/L14_rms": 0.30522946, "gate_b2c/L3_absmean": 0.07719225, "gate_b2c/L3_absmax": 0.56230038, "gate_b2c/L3_rms": 0.09761671, "gate_b2c/L5_absmean": 0.11205086, "gate_b2c/L5_absmax": 0.65790063, "gate_b2c/L5_rms": 0.11821484, "gate_b2c/L6_absmean": 0.16619086, "gate_b2c/L6_absmax": 0.4600364, "gate_b2c/L6_rms": 0.16931295}, "streams": {"cos_a2b/round0": 0.381382, "cos_c2b/round0": 0.283201, "cos_b2a/round0": 0.164769, "cos_b2c/round0": 0.129901, "rms_a/round0": 0.900604, "rms_b/round0": 1.737262, "rms_c/round0": 1.287124, "cos_a2b/round1": 0.408537, "cos_c2b/round1": 0.273588, "cos_b2a/round1": 0.227631, "cos_b2c/round1": 0.146797, "rms_a/round1": 1.636892, "rms_b/round1": 3.267106, "rms_c/round1": 2.126188, "cos_a2b/round2": 0.517527, "cos_c2b/round2": 0.48228, "cos_b2a/round2": 0.074254, "cos_b2c/round2": 0.138123, "rms_a/round2": 3.506801, "rms_b/round2": 7.222755, "rms_c/round2": 3.419046}, "alpha_mean": 0.4392287970896831, "alpha_hist": [0.01981, 0.0849, 0.16319, 0.18106, 0.17, 0.15791, 0.11902, 0.08231, 0.02144, 0.00037], "alpha_by_type": {"special": {"share": 0.00108, "mean": 0.58726, "std": 0.1061}, "whitespace": {"share": 0.03004, "mean": 0.58429, "std": 0.13121}, "digit": {"share": 0.03633, "mean": 0.61037, "std": 0.12002}, "word_start": {"share": 0.56674, "mean": 0.44121, "std": 0.17424}, "word_cont": {"share": 0.24509, "mean": 0.31915, "std": 0.15122}, "punct": {"share": 0.12071, "mean": 0.58477, "std": 0.17566}}, "alpha_p05": 0.14176, "alpha_p50": 0.43662, "alpha_p95": 0.7454}
{"step": 14410, "total_steps": 72479, "tokens": 7554990080, "loss": 4.6744449734687805, "l_mix": 2.893271952867508, "l_a": 2.990619271993637, "l_c": 2.9411048889160156, "z_loss": 16.557050228118896, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2208385467529297, "tokens_per_second": 1444981.011449727, "eta_seconds": 21069.397888803484, "data_wait_seconds": 1.5020370483398438e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20763, "cosmopedia_v2": 0.20042, "finemath": 0.08237}}
{"step": 14420, "total_steps": 72479, "tokens": 7560232960, "loss": 5.045517206192017, "l_mix": 3.1251773834228516, "l_a": 3.217399835586548, "l_c": 3.178156852722168, "z_loss": 16.72708249092102, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19735313951969147, "tokens_per_second": 1424960.3996430573, "eta_seconds": 21361.742403244974, "data_wait_seconds": 2.4944543838500977e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.2077, "cosmopedia_v2": 0.20042, "finemath": 0.08232}}
{"step": 14430, "total_steps": 72479, "tokens": 7565475840, "loss": 4.999188482761383, "l_mix": 3.0972880423069, "l_a": 3.1883016228675842, "l_c": 3.1458070278167725, "z_loss": 16.67726469039917, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16257533431053162, "tokens_per_second": 1328132.4255131842, "eta_seconds": 22915.18038966656, "data_wait_seconds": 1.71661376953125e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20769, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 14440, "total_steps": 72479, "tokens": 7570718720, "loss": 5.083575665950775, "l_mix": 3.149237781763077, "l_a": 3.24196919798851, "l_c": 3.2002183198928833, "z_loss": 16.81568741798401, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19374817609786987, "tokens_per_second": 1311800.7341186397, "eta_seconds": 23196.473702573774, "data_wait_seconds": 3.2842159271240234e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20769, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 14450, "total_steps": 72479, "tokens": 7575961600, "loss": 4.936544358730316, "l_mix": 3.0576459169387817, "l_a": 3.15202796459198, "l_c": 3.1053545773029327, "z_loss": 16.834765434265137, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1900724470615387, "tokens_per_second": 1397795.3558673698, "eta_seconds": 21765.6384565115, "data_wait_seconds": 2.1278858184814453e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20768, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 14460, "total_steps": 72479, "tokens": 7581204480, "loss": 4.998881697654724, "l_mix": 3.0969561636447906, "l_a": 3.1867430210113525, "l_c": 3.1475073397159576, "z_loss": 16.502368688583374, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16465574502944946, "tokens_per_second": 1328914.0944093133, "eta_seconds": 22889.86594390869, "data_wait_seconds": 2.5063753128051758e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20768, "cosmopedia_v2": 0.20041, "finemath": 0.08237}}
{"step": 14470, "total_steps": 72479, "tokens": 7586447360, "loss": 4.606148958206177, "l_mix": 2.848764032125473, "l_a": 2.9543155431747437, "l_c": 2.898059129714966, "z_loss": 16.72479486465454, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16545838117599487, "tokens_per_second": 1429599.7849134936, "eta_seconds": 21274.08167862892, "data_wait_seconds": 2.7954578399658203e-05, "mix": {"fineweb_hq": 0.50961, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08231}}
{"step": 14480, "total_steps": 72479, "tokens": 7591690240, "loss": 4.610644280910492, "l_mix": 2.8532307744026184, "l_a": 2.9535856544971466, "l_c": 2.8989031314849854, "z_loss": 16.668075919151306, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1490488052368164, "tokens_per_second": 1462102.6246572256, "eta_seconds": 20797.568651604655, "data_wait_seconds": 2.714991569519043e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08232}}
{"step": 14490, "total_steps": 72479, "tokens": 7596933120, "loss": 4.688421189785004, "l_mix": 2.901060312986374, "l_a": 3.004560500383377, "l_c": 2.9477399289608, "z_loss": 16.705806255340576, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16259834170341492, "tokens_per_second": 1432587.4134891499, "eta_seconds": 21222.395607924464, "data_wait_seconds": 2.1159648895263672e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20766, "cosmopedia_v2": 0.20041, "finemath": 0.08233}}
{"step": 14500, "total_steps": 72479, "tokens": 7602176000, "loss": 4.571162045001984, "l_mix": 2.828098237514496, "l_a": 2.9286945164203644, "l_c": 2.876004457473755, "z_loss": 16.54061007499695, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16538472473621368, "tokens_per_second": 1460333.8315407522, "eta_seconds": 20815.578804969788, "data_wait_seconds": 1.8328428268432617e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20766, "cosmopedia_v2": 0.20041, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.25340208, "gate_a2b/L1_absmax": 0.6447075, "gate_a2b/L1_rms": 0.27861449, "gate_a2b/L3_absmean": 0.31403476, "gate_a2b/L3_absmax": 0.48926014, "gate_a2b/L3_rms": 0.31907332, "gate_a2b/L5_absmean": 0.50171113, "gate_a2b/L5_absmax": 0.68633896, "gate_a2b/L5_rms": 0.50845402, "gate_c2b/L1_absmean": 0.01523504, "gate_c2b/L1_absmax": 0.48153269, "gate_c2b/L1_rms": 0.0571183, "gate_c2b/L3_absmean": 0.20963398, "gate_c2b/L3_absmax": 0.43514276, "gate_c2b/L3_rms": 0.21434934, "gate_c2b/L5_absmean": 0.53587532, "gate_c2b/L5_absmax": 0.73321611, "gate_c2b/L5_rms": 0.54834497, "gate_b2a/L7_absmean": 0.07777002, "gate_b2a/L7_absmax": 0.25516456, "gate_b2a/L7_rms": 0.0812562, "gate_b2a/L11_absmean": 0.12479004, "gate_b2a/L11_absmax": 0.37948862, "gate_b2a/L11_rms": 0.12716454, "gate_b2a/L14_absmean": 0.30408588, "gate_b2a/L14_absmax": 0.48674867, "gate_b2a/L14_rms": 0.30528912, "gate_b2c/L3_absmean": 0.0781875, "gate_b2c/L3_absmax": 0.56349695, "gate_b2c/L3_rms": 0.09872331, "gate_b2c/L5_absmean": 0.11290014, "gate_b2c/L5_absmax": 0.66082925, "gate_b2c/L5_rms": 0.11898583, "gate_b2c/L6_absmean": 0.16714559, "gate_b2c/L6_absmax": 0.4594864, "gate_b2c/L6_rms": 0.170256}, "streams": {"cos_a2b/round0": 0.375897, "cos_c2b/round0": 0.27017, "cos_b2a/round0": 0.158362, "cos_b2c/round0": 0.129585, "rms_a/round0": 0.893706, "rms_b/round0": 1.730069, "rms_c/round0": 1.272923, "cos_a2b/round1": 0.404929, "cos_c2b/round1": 0.274259, "cos_b2a/round1": 0.219217, "cos_b2c/round1": 0.148443, "rms_a/round1": 1.625928, "rms_b/round1": 3.270216, "rms_c/round1": 2.092248, "cos_a2b/round2": 0.528558, "cos_c2b/round2": 0.469601, "cos_b2a/round2": 0.080964, "cos_b2c/round2": 0.140151, "rms_a/round2": 3.512944, "rms_b/round2": 7.070088, "rms_c/round2": 3.374058}, "alpha_mean": 0.44496214675843504, "alpha_hist": [0.01677, 0.06647, 0.14348, 0.18272, 0.20453, 0.18007, 0.12444, 0.07024, 0.01125, 5e-05], "alpha_by_type": {"special": {"share": 0.00122, "mean": 0.54115, "std": 0.08941}, "whitespace": {"share": 0.02426, "mean": 0.49132, "std": 0.10861}, "digit": {"share": 0.02969, "mean": 0.42469, "std": 0.12106}, "word_start": {"share": 0.58917, "mean": 0.4743, "std": 0.15793}, "word_cont": {"share": 0.25328, "mean": 0.31552, "std": 0.14308}, "punct": {"share": 0.10237, "mean": 0.59009, "std": 0.16148}}, "alpha_p05": 0.14854, "alpha_p50": 0.44963, "alpha_p95": 0.71958}
{"step": 14510, "total_steps": 72479, "tokens": 7607418880, "loss": 4.649294078350067, "l_mix": 2.8768533766269684, "l_a": 2.9751369655132294, "l_c": 2.9274410009384155, "z_loss": 16.671327829360962, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1665029525756836, "tokens_per_second": 1369766.2312479801, "eta_seconds": 22188.056895160677, "data_wait_seconds": 2.2232532501220703e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20041, "finemath": 0.08236}}
{"step": 14520, "total_steps": 72479, "tokens": 7612661760, "loss": 4.720636069774628, "l_mix": 2.9219004213809967, "l_a": 3.0203666985034943, "l_c": 2.9698992669582367, "z_loss": 16.55757975578308, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16157133877277374, "tokens_per_second": 1400556.8384104415, "eta_seconds": 21696.519097709654, "data_wait_seconds": 1.7523765563964844e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20764, "cosmopedia_v2": 0.20041, "finemath": 0.08237}}
{"step": 14530, "total_steps": 72479, "tokens": 7617904640, "loss": 3.30499404668808, "l_mix": 2.0417418628931046, "l_a": 2.124178171157837, "l_c": 2.0804316252470016, "z_loss": 18.69273066520691, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.23232142627239227, "tokens_per_second": 1308131.6838349947, "eta_seconds": 23225.463986110688, "data_wait_seconds": 2.5272369384765625e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20764, "cosmopedia_v2": 0.20041, "finemath": 0.08238}}
{"step": 14540, "total_steps": 72479, "tokens": 7623147520, "loss": 4.94722855091095, "l_mix": 3.064295679330826, "l_a": 3.1611914336681366, "l_c": 3.109824448823929, "z_loss": 16.280097603797913, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18238914012908936, "tokens_per_second": 1482637.0511544154, "eta_seconds": 20488.30656723976, "data_wait_seconds": 1.6450881958007812e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.2077, "cosmopedia_v2": 0.20041, "finemath": 0.08232}}
{"step": 14550, "total_steps": 72479, "tokens": 7628390400, "loss": 4.905020356178284, "l_mix": 3.038609504699707, "l_a": 3.1299022138118744, "l_c": 3.086097836494446, "z_loss": 16.106985092163086, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17791807651519775, "tokens_per_second": 1373239.532439053, "eta_seconds": 22116.665617728235, "data_wait_seconds": 2.104043960571289e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.2077, "cosmopedia_v2": 0.20041, "finemath": 0.08234}}
{"step": 14560, "total_steps": 72479, "tokens": 7633633280, "loss": 4.912252366542816, "l_mix": 3.04152050614357, "l_a": 3.1382767260074615, "l_c": 3.0920459926128387, "z_loss": 16.350210666656494, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2079019993543625, "tokens_per_second": 1385329.2198196873, "eta_seconds": 21919.870192265513, "data_wait_seconds": 2.1845102310180664e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20769, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 14570, "total_steps": 72479, "tokens": 7638876160, "loss": 5.026654899120331, "l_mix": 3.1131553947925568, "l_a": 3.2107642590999603, "l_c": 3.162008613348007, "z_loss": 16.677693605422974, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.166847363114357, "tokens_per_second": 1474433.041937756, "eta_seconds": 20591.639585137367, "data_wait_seconds": 3.272294998168945e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20769, "cosmopedia_v2": 0.20041, "finemath": 0.08236}}
{"step": 14580, "total_steps": 72479, "tokens": 7644119040, "loss": 4.71318244934082, "l_mix": 2.917125880718231, "l_a": 3.017395108938217, "l_c": 2.9639433920383453, "z_loss": 16.550251007080078, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16308298707008362, "tokens_per_second": 1438165.9004281752, "eta_seconds": 21107.266486406326, "data_wait_seconds": 2.390146255493164e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20768, "cosmopedia_v2": 0.20041, "finemath": 0.0823}}
{"step": 14590, "total_steps": 72479, "tokens": 7649361920, "loss": 4.720850646495819, "l_mix": 2.9207742512226105, "l_a": 3.026679575443268, "l_c": 2.9680658876895905, "z_loss": 16.52643346786499, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17506350576877594, "tokens_per_second": 1429844.5358324419, "eta_seconds": 21226.439148736, "data_wait_seconds": 1.582503318786621e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20768, "cosmopedia_v2": 0.20041, "finemath": 0.08232}}
{"step": 14600, "total_steps": 72479, "tokens": 7654604800, "loss": 4.728981077671051, "l_mix": 2.926445573568344, "l_a": 3.0249996185302734, "l_c": 2.978037655353546, "z_loss": 16.243010997772217, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.14597882330417633, "tokens_per_second": 1433146.2934608096, "eta_seconds": 21173.87826383114, "data_wait_seconds": 1.609325408935547e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.25397086, "gate_a2b/L1_absmax": 0.64509577, "gate_a2b/L1_rms": 0.279006, "gate_a2b/L3_absmean": 0.31442186, "gate_a2b/L3_absmax": 0.49171802, "gate_a2b/L3_rms": 0.31945798, "gate_a2b/L5_absmean": 0.50210297, "gate_a2b/L5_absmax": 0.68494707, "gate_a2b/L5_rms": 0.50882864, "gate_c2b/L1_absmean": 0.0147189, "gate_c2b/L1_absmax": 0.47733691, "gate_c2b/L1_rms": 0.05666901, "gate_c2b/L3_absmean": 0.2097287, "gate_c2b/L3_absmax": 0.43796021, "gate_c2b/L3_rms": 0.21439335, "gate_c2b/L5_absmean": 0.53591239, "gate_c2b/L5_absmax": 0.73062915, "gate_c2b/L5_rms": 0.54836476, "gate_b2a/L7_absmean": 0.07860473, "gate_b2a/L7_absmax": 0.25603405, "gate_b2a/L7_rms": 0.08206853, "gate_b2a/L11_absmean": 0.12567537, "gate_b2a/L11_absmax": 0.38432136, "gate_b2a/L11_rms": 0.12810507, "gate_b2a/L14_absmean": 0.30456614, "gate_b2a/L14_absmax": 0.48587623, "gate_b2a/L14_rms": 0.30573002, "gate_b2c/L3_absmean": 0.07843593, "gate_b2c/L3_absmax": 0.56210434, "gate_b2c/L3_rms": 0.09864783, "gate_b2c/L5_absmean": 0.11365467, "gate_b2c/L5_absmax": 0.66239566, "gate_b2c/L5_rms": 0.11974063, "gate_b2c/L6_absmean": 0.16750839, "gate_b2c/L6_absmax": 0.46060956, "gate_b2c/L6_rms": 0.17058305}, "streams": {"cos_a2b/round0": 0.37705, "cos_c2b/round0": 0.272387, "cos_b2a/round0": 0.155828, "cos_b2c/round0": 0.128904, "rms_a/round0": 0.898003, "rms_b/round0": 1.721015, "rms_c/round0": 1.269555, "cos_a2b/round1": 0.401623, "cos_c2b/round1": 0.268352, "cos_b2a/round1": 0.223456, "cos_b2c/round1": 0.144571, "rms_a/round1": 1.609361, "rms_b/round1": 3.255753, "rms_c/round1": 2.072099, "cos_a2b/round2": 0.529076, "cos_c2b/round2": 0.467229, "cos_b2a/round2": 0.079367, "cos_b2c/round2": 0.143393, "rms_a/round2": 3.509943, "rms_b/round2": 7.097825, "rms_c/round2": 3.32636}, "alpha_mean": 0.4883697107455731, "alpha_hist": [0.01262, 0.04797, 0.1154, 0.16283, 0.1774, 0.17969, 0.15759, 0.10866, 0.03679, 0.00105], "alpha_by_type": {"special": {"share": 0.00117, "mean": 0.62925, "std": 0.11365}, "whitespace": {"share": 0.02661, "mean": 0.58268, "std": 0.13128}, "digit": {"share": 0.03818, "mean": 0.56101, "std": 0.13783}, "word_start": {"share": 0.56061, "mean": 0.51243, "std": 0.17109}, "word_cont": {"share": 0.26318, "mean": 0.35587, "std": 0.15151}, "punct": {"share": 0.11024, "mean": 0.63292, "std": 0.1596}}, "alpha_p05": 0.18565, "alpha_p50": 0.50152, "alpha_p95": 0.77219}
{"step": 14610, "total_steps": 72479, "tokens": 7659847680, "loss": 4.799417316913605, "l_mix": 2.9700670540332794, "l_a": 3.0710590481758118, "l_c": 3.021299123764038, "z_loss": 16.428252339363098, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1810341477394104, "tokens_per_second": 1464362.238733803, "eta_seconds": 20718.932426333427, "data_wait_seconds": 1.7821788787841797e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08234}}
{"step": 14620, "total_steps": 72479, "tokens": 7665090560, "loss": 4.7027517557144165, "l_mix": 2.9098134338855743, "l_a": 3.014107495546341, "l_c": 2.9568032026290894, "z_loss": 16.651761293411255, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17202575504779816, "tokens_per_second": 1380166.307058588, "eta_seconds": 21979.075446820258, "data_wait_seconds": 1.7315149307250977e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 14630, "total_steps": 72479, "tokens": 7670333440, "loss": 4.649479031562805, "l_mix": 2.8765366971492767, "l_a": 2.979861229658127, "l_c": 2.9243218898773193, "z_loss": 16.873311758041382, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21342402696609497, "tokens_per_second": 1392206.7354819349, "eta_seconds": 21785.2246645689, "data_wait_seconds": 4.079937934875488e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20041, "finemath": 0.08237}}
{"step": 14640, "total_steps": 72479, "tokens": 7675576320, "loss": 3.1736740469932556, "l_mix": 1.9603324234485626, "l_a": 2.040119469165802, "l_c": 1.9982420206069946, "z_loss": 18.330438137054443, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2603566348552704, "tokens_per_second": 1360176.2875917917, "eta_seconds": 22294.384859251975, "data_wait_seconds": 3.358721733093262e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20041, "finemath": 0.08238}}
{"step": 14650, "total_steps": 72479, "tokens": 7680819200, "loss": 3.523869067430496, "l_mix": 2.1661105155944824, "l_a": 2.2941685914993286, "l_c": 2.226438343524933, "z_loss": 15.76417851448059, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21574203670024872, "tokens_per_second": 1473114.935614371, "eta_seconds": 20581.59212088585, "data_wait_seconds": 1.671910285949707e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20048, "finemath": 0.08232}}
{"step": 14660, "total_steps": 72479, "tokens": 7686062080, "loss": 3.51889106631279, "l_mix": 2.1636531949043274, "l_a": 2.284744083881378, "l_c": 2.227476418018341, "z_loss": 15.717304110527039, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2305273860692978, "tokens_per_second": 1377396.5635928938, "eta_seconds": 22008.046682596207, "data_wait_seconds": 1.683831214904785e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20764, "cosmopedia_v2": 0.20048, "finemath": 0.08233}}
{"step": 14670, "total_steps": 72479, "tokens": 7691304960, "loss": 4.756608307361603, "l_mix": 2.945201188325882, "l_a": 3.041045904159546, "l_c": 2.9914574921131134, "z_loss": 16.560970783233643, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16826488077640533, "tokens_per_second": 1441107.497783477, "eta_seconds": 21031.439388537405, "data_wait_seconds": 2.625584602355957e-05, "mix": {"fineweb_hq": 0.50961, "dclm": 0.20763, "cosmopedia_v2": 0.20041, "finemath": 0.08234}}
{"step": 14680, "total_steps": 72479, "tokens": 7696547840, "loss": 4.732585787773132, "l_mix": 2.928289771080017, "l_a": 3.0308815240859985, "l_c": 2.9778464436531067, "z_loss": 16.77642059326172, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1970975399017334, "tokens_per_second": 1479479.3576476702, "eta_seconds": 20482.42306008339, "data_wait_seconds": 1.5050172805786133e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20763, "cosmopedia_v2": 0.20041, "finemath": 0.08236}}
{"step": 14690, "total_steps": 72479, "tokens": 7701790720, "loss": 4.727986216545105, "l_mix": 2.925579994916916, "l_a": 3.0248014330863953, "l_c": 2.977809339761734, "z_loss": 16.22941029071808, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.23755252361297607, "tokens_per_second": 1452852.7546014942, "eta_seconds": 20854.19815328121, "data_wait_seconds": 1.5527009963989258e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20762, "cosmopedia_v2": 0.20041, "finemath": 0.08237}}
{"step": 14700, "total_steps": 72479, "tokens": 7707033600, "loss": 4.726740121841431, "l_mix": 2.924837589263916, "l_a": 3.0256766378879547, "l_c": 2.9751522839069366, "z_loss": 16.537014961242676, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.13933275640010834, "tokens_per_second": 1401713.8435288293, "eta_seconds": 21611.28428020477, "data_wait_seconds": 1.7821788787841797e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20769, "cosmopedia_v2": 0.20041, "finemath": 0.08231}, "gates": {"gate_a2b/L1_absmean": 0.25473881, "gate_a2b/L1_absmax": 0.63949585, "gate_a2b/L1_rms": 0.27951908, "gate_a2b/L3_absmean": 0.31471044, "gate_a2b/L3_absmax": 0.49125752, "gate_a2b/L3_rms": 0.31977171, "gate_a2b/L5_absmean": 0.50264448, "gate_a2b/L5_absmax": 0.68400735, "gate_a2b/L5_rms": 0.50927269, "gate_c2b/L1_absmean": 0.01449927, "gate_c2b/L1_absmax": 0.47697434, "gate_c2b/L1_rms": 0.05641449, "gate_c2b/L3_absmean": 0.20933543, "gate_c2b/L3_absmax": 0.42921403, "gate_c2b/L3_rms": 0.21400245, "gate_c2b/L5_absmean": 0.5353604, "gate_c2b/L5_absmax": 0.73238581, "gate_c2b/L5_rms": 0.54781181, "gate_b2a/L7_absmean": 0.07823911, "gate_b2a/L7_absmax": 0.25427249, "gate_b2a/L7_rms": 0.08173409, "gate_b2a/L11_absmean": 0.1246779, "gate_b2a/L11_absmax": 0.37962323, "gate_b2a/L11_rms": 0.12710467, "gate_b2a/L14_absmean": 0.3042196, "gate_b2a/L14_absmax": 0.4874838, "gate_b2a/L14_rms": 0.30541092, "gate_b2c/L3_absmean": 0.07824971, "gate_b2c/L3_absmax": 0.56336892, "gate_b2c/L3_rms": 0.09867339, "gate_b2c/L5_absmean": 0.11321668, "gate_b2c/L5_absmax": 0.66391277, "gate_b2c/L5_rms": 0.11939688, "gate_b2c/L6_absmean": 0.16752706, "gate_b2c/L6_absmax": 0.46237478, "gate_b2c/L6_rms": 0.17067547}, "streams": {"cos_a2b/round0": 0.375131, "cos_c2b/round0": 0.270705, "cos_b2a/round0": 0.15667, "cos_b2c/round0": 0.125088, "rms_a/round0": 0.899796, "rms_b/round0": 1.71963, "rms_c/round0": 1.278792, "cos_a2b/round1": 0.40536, "cos_c2b/round1": 0.271336, "cos_b2a/round1": 0.21784, "cos_b2c/round1": 0.138117, "rms_a/round1": 1.619934, "rms_b/round1": 3.250741, "rms_c/round1": 2.102096, "cos_a2b/round2": 0.530709, "cos_c2b/round2": 0.4707, "cos_b2a/round2": 0.080673, "cos_b2c/round2": 0.142387, "rms_a/round2": 3.488921, "rms_b/round2": 6.999275, "rms_c/round2": 3.352117}, "alpha_mean": 0.46340478727807977, "alpha_hist": [0.01086, 0.06044, 0.13634, 0.17545, 0.17531, 0.19466, 0.14627, 0.08345, 0.01698, 0.00024], "alpha_by_type": {"special": {"share": 0.00111, "mean": 0.58896, "std": 0.10193}, "whitespace": {"share": 0.02921, "mean": 0.54956, "std": 0.11422}, "digit": {"share": 0.0191, "mean": 0.5124, "std": 0.12455}, "word_start": {"share": 0.56075, "mean": 0.4832, "std": 0.16587}, "word_cont": {"share": 0.26088, "mean": 0.34613, "std": 0.14738}, "punct": {"share": 0.12895, "mean": 0.58675, "std": 0.16615}}, "alpha_p05": 0.17899, "alpha_p50": 0.47329, "alpha_p95": 0.73829}
{"step": 14710, "total_steps": 72479, "tokens": 7712276480, "loss": 4.687692642211914, "l_mix": 2.90036404132843, "l_a": 3.0033794045448303, "l_c": 2.9488860368728638, "z_loss": 16.489800691604614, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1460295468568802, "tokens_per_second": 1434816.6589600188, "eta_seconds": 21109.033884477616, "data_wait_seconds": 1.6421079635620117e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20768, "cosmopedia_v2": 0.20041, "finemath": 0.08232}}
{"step": 14720, "total_steps": 72479, "tokens": 7717519360, "loss": 4.620432674884796, "l_mix": 2.8583501875400543, "l_a": 2.9596597254276276, "l_c": 2.9083844125270844, "z_loss": 16.69207239151001, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1466348022222519, "tokens_per_second": 1465757.1395761974, "eta_seconds": 20659.869069957735, "data_wait_seconds": 1.659989356994629e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20768, "cosmopedia_v2": 0.20041, "finemath": 0.08234}}
{"step": 14730, "total_steps": 72479, "tokens": 7722762240, "loss": 4.726454019546509, "l_mix": 2.924727112054825, "l_a": 3.0280380845069885, "l_c": 2.9721040427684784, "z_loss": 16.842575788497925, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16009555757045746, "tokens_per_second": 1449701.8158737307, "eta_seconds": 20885.058831048013, "data_wait_seconds": 1.9103288650512695e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 14740, "total_steps": 72479, "tokens": 7728005120, "loss": 4.717152953147888, "l_mix": 2.919092833995819, "l_a": 3.018997222185135, "l_c": 2.968836337327957, "z_loss": 17.100926637649536, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17618359625339508, "tokens_per_second": 1435337.2746177157, "eta_seconds": 21090.419211792945, "data_wait_seconds": 7.56978988647461e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08236}}
{"step": 14750, "total_steps": 72479, "tokens": 7733248000, "loss": 3.2003742456436157, "l_mix": 1.976699411869049, "l_a": 2.062827840447426, "l_c": 2.0098624527454376, "z_loss": 18.677526473999023, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22773849964141846, "tokens_per_second": 1394525.436123547, "eta_seconds": 21703.886618328095, "data_wait_seconds": 3.466010093688965e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20041, "finemath": 0.08237}}
{"step": 14760, "total_steps": 72479, "tokens": 7738490880, "loss": 3.51892814040184, "l_mix": 2.1629623770713806, "l_a": 2.290786623954773, "l_c": 2.223900645971298, "z_loss": 15.595002055168152, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18613837659358978, "tokens_per_second": 1474741.2527455783, "eta_seconds": 20519.788821029662, "data_wait_seconds": 1.850724220275879e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20766, "cosmopedia_v2": 0.20047, "finemath": 0.08232}}
{"step": 14770, "total_steps": 72479, "tokens": 7743733760, "loss": 3.5027692019939423, "l_mix": 2.1537398993968964, "l_a": 2.2780069708824158, "l_c": 2.213410198688507, "z_loss": 16.04086923599243, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2181859016418457, "tokens_per_second": 1463361.6485729248, "eta_seconds": 20675.77500169277, "data_wait_seconds": 2.1338462829589844e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20765, "cosmopedia_v2": 0.20047, "finemath": 0.08233}}
{"step": 14780, "total_steps": 72479, "tokens": 7748976640, "loss": 4.6835450530052185, "l_mix": 2.898752450942993, "l_a": 2.9996767342090607, "l_c": 2.9441959261894226, "z_loss": 16.306756258010864, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1585286557674408, "tokens_per_second": 1463393.4928004995, "eta_seconds": 20671.742399311068, "data_wait_seconds": 2.6047229766845703e-05, "mix": {"fineweb_hq": 0.50961, "dclm": 0.20765, "cosmopedia_v2": 0.20041, "finemath": 0.08234}}
{"step": 14790, "total_steps": 72479, "tokens": 7754219520, "loss": 4.664309322834015, "l_mix": 2.8859059512615204, "l_a": 2.987538665533066, "l_c": 2.934952735900879, "z_loss": 16.559846878051758, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1817624866962433, "tokens_per_second": 1459699.9658291768, "eta_seconds": 20720.457039141656, "data_wait_seconds": 2.0176172256469727e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20764, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 14800, "total_steps": 72479, "tokens": 7759462400, "loss": 4.697168707847595, "l_mix": 2.9062588214874268, "l_a": 3.0084167420864105, "l_c": 2.9555908143520355, "z_loss": 17.075545072555542, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18640902638435364, "tokens_per_second": 1443399.5497428689, "eta_seconds": 20950.82235364914, "data_wait_seconds": 2.6702880859375e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20764, "cosmopedia_v2": 0.20041, "finemath": 0.08236}, "gates": {"gate_a2b/L1_absmean": 0.25714141, "gate_a2b/L1_absmax": 0.65535539, "gate_a2b/L1_rms": 0.28214675, "gate_a2b/L3_absmean": 0.31599244, "gate_a2b/L3_absmax": 0.4936544, "gate_a2b/L3_rms": 0.32097709, "gate_a2b/L5_absmean": 0.50399089, "gate_a2b/L5_absmax": 0.68469942, "gate_a2b/L5_rms": 0.51060414, "gate_c2b/L1_absmean": 0.01444617, "gate_c2b/L1_absmax": 0.4748683, "gate_c2b/L1_rms": 0.05634421, "gate_c2b/L3_absmean": 0.20907994, "gate_c2b/L3_absmax": 0.432659, "gate_c2b/L3_rms": 0.21375713, "gate_c2b/L5_absmean": 0.53549719, "gate_c2b/L5_absmax": 0.73273486, "gate_c2b/L5_rms": 0.54775786, "gate_b2a/L7_absmean": 0.07872288, "gate_b2a/L7_absmax": 0.25298536, "gate_b2a/L7_rms": 0.08210464, "gate_b2a/L11_absmean": 0.12529421, "gate_b2a/L11_absmax": 0.37971315, "gate_b2a/L11_rms": 0.12767451, "gate_b2a/L14_absmean": 0.30441949, "gate_b2a/L14_absmax": 0.48130444, "gate_b2a/L14_rms": 0.30557397, "gate_b2c/L3_absmean": 0.08009219, "gate_b2c/L3_absmax": 0.5645802, "gate_b2c/L3_rms": 0.10016345, "gate_b2c/L5_absmean": 0.11456846, "gate_b2c/L5_absmax": 0.66776854, "gate_b2c/L5_rms": 0.12067989, "gate_b2c/L6_absmean": 0.16878507, "gate_b2c/L6_absmax": 0.46275318, "gate_b2c/L6_rms": 0.17190845}, "streams": {"cos_a2b/round0": 0.378247, "cos_c2b/round0": 0.28064, "cos_b2a/round0": 0.154811, "cos_b2c/round0": 0.133059, "rms_a/round0": 0.882115, "rms_b/round0": 1.711755, "rms_c/round0": 1.270709, "cos_a2b/round1": 0.40821, "cos_c2b/round1": 0.270143, "cos_b2a/round1": 0.226488, "cos_b2c/round1": 0.138632, "rms_a/round1": 1.588109, "rms_b/round1": 3.223293, "rms_c/round1": 2.081904, "cos_a2b/round2": 0.520114, "cos_c2b/round2": 0.486512, "cos_b2a/round2": 0.079956, "cos_b2c/round2": 0.132192, "rms_a/round2": 3.449514, "rms_b/round2": 7.116868, "rms_c/round2": 3.362363}, "alpha_mean": 0.42577709156249455, "alpha_hist": [0.01591, 0.07762, 0.16988, 0.19466, 0.19421, 0.17291, 0.11819, 0.05103, 0.00552, 6e-05], "alpha_by_type": {"special": {"share": 0.0009, "mean": 0.55297, "std": 0.11895}, "whitespace": {"share": 0.03021, "mean": 0.49603, "std": 0.12479}, "digit": {"share": 0.04146, "mean": 0.43942, "std": 0.11748}, "word_start": {"share": 0.56168, "mean": 0.45463, "std": 0.15822}, "word_cont": {"share": 0.25383, "mean": 0.29907, "std": 0.13329}, "punct": {"share": 0.11192, "mean": 0.5433, "std": 0.15891}}, "alpha_p05": 0.14536, "alpha_p50": 0.41917, "alpha_p95": 0.71302}
{"step": 14810, "total_steps": 72479, "tokens": 7764705280, "loss": 4.750841975212097, "l_mix": 2.939303994178772, "l_a": 3.045123428106308, "l_c": 2.9878174662590027, "z_loss": 16.55593490600586, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19030968844890594, "tokens_per_second": 1413187.629522032, "eta_seconds": 21395.01085374355, "data_wait_seconds": 1.8268823623657227e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20763, "cosmopedia_v2": 0.20041, "finemath": 0.08238}}
{"step": 14820, "total_steps": 72479, "tokens": 7769948160, "loss": 5.012638628482819, "l_mix": 3.1060769855976105, "l_a": 3.197776108980179, "l_c": 3.1520629227161407, "z_loss": 16.099488258361816, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15472716093063354, "tokens_per_second": 1443334.4648098235, "eta_seconds": 20944.502143502235, "data_wait_seconds": 2.2649765014648438e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20769, "cosmopedia_v2": 0.2004, "finemath": 0.08232}}
{"step": 14830, "total_steps": 72479, "tokens": 7775191040, "loss": 4.964222252368927, "l_mix": 3.0750370621681213, "l_a": 3.1704646050930023, "l_c": 3.121373236179352, "z_loss": 16.3365296125412, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.13801302015781403, "tokens_per_second": 1403954.7150876739, "eta_seconds": 21528.24345912933, "data_wait_seconds": 1.6242265701293945e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20769, "cosmopedia_v2": 0.2004, "finemath": 0.08233}}
{"step": 14840, "total_steps": 72479, "tokens": 7780433920, "loss": 5.052058577537537, "l_mix": 3.130650043487549, "l_a": 3.217758595943451, "l_c": 3.1813796758651733, "z_loss": 16.669289350509644, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1784915179014206, "tokens_per_second": 1464645.8645318807, "eta_seconds": 20632.58891708851, "data_wait_seconds": 2.378225326538086e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20768, "cosmopedia_v2": 0.2004, "finemath": 0.08235}}
{"step": 14850, "total_steps": 72479, "tokens": 7785676800, "loss": 4.942800045013428, "l_mix": 3.0617687106132507, "l_a": 3.1517254114151, "l_c": 3.112893134355545, "z_loss": 16.457127571105957, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17025895416736603, "tokens_per_second": 1430971.7682810728, "eta_seconds": 21114.457896184922, "data_wait_seconds": 2.065300941467285e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.2004, "finemath": 0.08236}}
{"step": 14860, "total_steps": 72479, "tokens": 7790919680, "loss": 3.302733600139618, "l_mix": 2.0410889238119125, "l_a": 2.1196452528238297, "l_c": 2.079939544200897, "z_loss": 17.691736221313477, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.3274766206741333, "tokens_per_second": 1353570.353707768, "eta_seconds": 22317.975707173347, "data_wait_seconds": 2.613663673400879e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.2004, "finemath": 0.08237}}
{"step": 14870, "total_steps": 72479, "tokens": 7796162560, "loss": 3.2570942640304565, "l_mix": 2.0138301849365234, "l_a": 2.0883679240942, "l_c": 2.049717992544174, "z_loss": 18.382771730422974, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.24564199149608612, "tokens_per_second": 1242873.4090311914, "eta_seconds": 24301.51548221111, "data_wait_seconds": 1.7642974853515625e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.2004, "finemath": 0.08238}}
{"step": 14880, "total_steps": 72479, "tokens": 7801405440, "loss": 4.626258075237274, "l_mix": 2.8622539043426514, "l_a": 2.96502685546875, "l_c": 2.9095633924007416, "z_loss": 16.27023184299469, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1546933799982071, "tokens_per_second": 1471075.5131079203, "eta_seconds": 20528.153886675835, "data_wait_seconds": 1.58846378326416e-05, "mix": {"fineweb_hq": 0.50961, "dclm": 0.20766, "cosmopedia_v2": 0.2004, "finemath": 0.08233}}
{"step": 14890, "total_steps": 72479, "tokens": 7806648320, "loss": 4.679494917392731, "l_mix": 2.8962543606758118, "l_a": 2.994816869497299, "l_c": 2.9437988102436066, "z_loss": 16.557318210601807, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.14104749262332916, "tokens_per_second": 1382134.8456280439, "eta_seconds": 21845.351578760146, "data_wait_seconds": 2.4497509002685547e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20766, "cosmopedia_v2": 0.2004, "finemath": 0.08234}}
{"step": 14900, "total_steps": 72479, "tokens": 7811891200, "loss": 4.671500623226166, "l_mix": 2.8914053440093994, "l_a": 2.9887068569660187, "l_c": 2.939475029706955, "z_loss": 16.4065580368042, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15344710648059845, "tokens_per_second": 1461963.719983523, "eta_seconds": 20648.924689006806, "data_wait_seconds": 1.722574234008789e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20765, "cosmopedia_v2": 0.2004, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.25856766, "gate_a2b/L1_absmax": 0.64943409, "gate_a2b/L1_rms": 0.28306657, "gate_a2b/L3_absmean": 0.31664333, "gate_a2b/L3_absmax": 0.49071401, "gate_a2b/L3_rms": 0.32151189, "gate_a2b/L5_absmean": 0.50480902, "gate_a2b/L5_absmax": 0.69190627, "gate_a2b/L5_rms": 0.51141131, "gate_c2b/L1_absmean": 0.01491879, "gate_c2b/L1_absmax": 0.47277629, "gate_c2b/L1_rms": 0.05589423, "gate_c2b/L3_absmean": 0.20940728, "gate_c2b/L3_absmax": 0.43897894, "gate_c2b/L3_rms": 0.21401556, "gate_c2b/L5_absmean": 0.53482956, "gate_c2b/L5_absmax": 0.72819322, "gate_c2b/L5_rms": 0.54708505, "gate_b2a/L7_absmean": 0.07839751, "gate_b2a/L7_absmax": 0.25412714, "gate_b2a/L7_rms": 0.081829, "gate_b2a/L11_absmean": 0.12542005, "gate_b2a/L11_absmax": 0.38062459, "gate_b2a/L11_rms": 0.1277729, "gate_b2a/L14_absmean": 0.30490363, "gate_b2a/L14_absmax": 0.48372179, "gate_b2a/L14_rms": 0.30608073, "gate_b2c/L3_absmean": 0.08044187, "gate_b2c/L3_absmax": 0.56799799, "gate_b2c/L3_rms": 0.1004862, "gate_b2c/L5_absmean": 0.11452803, "gate_b2c/L5_absmax": 0.67369169, "gate_b2c/L5_rms": 0.12079508, "gate_b2c/L6_absmean": 0.16930477, "gate_b2c/L6_absmax": 0.46394977, "gate_b2c/L6_rms": 0.17242612}, "streams": {"cos_a2b/round0": 0.380581, "cos_c2b/round0": 0.272276, "cos_b2a/round0": 0.159077, "cos_b2c/round0": 0.127297, "rms_a/round0": 0.902804, "rms_b/round0": 1.711553, "rms_c/round0": 1.289429, "cos_a2b/round1": 0.402071, "cos_c2b/round1": 0.276478, "cos_b2a/round1": 0.231397, "cos_b2c/round1": 0.139554, "rms_a/round1": 1.596501, "rms_b/round1": 3.234411, "rms_c/round1": 2.111944, "cos_a2b/round2": 0.525412, "cos_c2b/round2": 0.487285, "cos_b2a/round2": 0.085919, "cos_b2c/round2": 0.141664, "rms_a/round2": 3.462711, "rms_b/round2": 7.064539, "rms_c/round2": 3.383443}, "alpha_mean": 0.45261474952160086, "alpha_hist": [0.01569, 0.06207, 0.13431, 0.19958, 0.19666, 0.16327, 0.12714, 0.08206, 0.01881, 0.00041], "alpha_by_type": {"special": {"share": 0.00113, "mean": 0.57235, "std": 0.11145}, "whitespace": {"share": 0.02945, "mean": 0.53712, "std": 0.11992}, "digit": {"share": 0.03239, "mean": 0.52077, "std": 0.12783}, "word_start": {"share": 0.55054, "mean": 0.48023, "std": 0.17038}, "word_cont": {"share": 0.23198, "mean": 0.32186, "std": 0.14519}, "punct": {"share": 0.15451, "mean": 0.51925, "std": 0.16955}}, "alpha_p05": 0.16996, "alpha_p50": 0.42728, "alpha_p95": 0.74651}
{"step": 14910, "total_steps": 72479, "tokens": 7817134080, "loss": 4.682015597820282, "l_mix": 2.897063136100769, "l_a": 2.997670531272888, "l_c": 2.946686238050461, "z_loss": 16.45271587371826, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20930083096027374, "tokens_per_second": 1478294.7123469915, "eta_seconds": 20417.265664219856, "data_wait_seconds": 1.570582389831543e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20765, "cosmopedia_v2": 0.2004, "finemath": 0.08236}}
{"step": 14920, "total_steps": 72479, "tokens": 7822376960, "loss": 4.783481121063232, "l_mix": 2.961446702480316, "l_a": 3.059882938861847, "l_c": 3.008146971464157, "z_loss": 16.252893805503845, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19683904945850372, "tokens_per_second": 1445175.9698707326, "eta_seconds": 20881.53527400494, "data_wait_seconds": 1.6987323760986328e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.2004, "finemath": 0.08237}}
{"step": 14930, "total_steps": 72479, "tokens": 7827619840, "loss": 3.55160591006279, "l_mix": 2.182528793811798, "l_a": 2.310718983411789, "l_c": 2.2476591169834137, "z_loss": 15.636228561401367, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18785709142684937, "tokens_per_second": 1430074.2112560803, "eta_seconds": 21098.380681586266, "data_wait_seconds": 1.659989356994629e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20047, "finemath": 0.08232}}
{"step": 14940, "total_steps": 72479, "tokens": 7832862720, "loss": 4.810132801532745, "l_mix": 2.9780878126621246, "l_a": 3.071173667907715, "l_c": 3.0301437079906464, "z_loss": 16.498046398162842, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18914473056793213, "tokens_per_second": 1437617.2909418321, "eta_seconds": 20984.03199660778, "data_wait_seconds": 3.084540367126465e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.2077, "cosmopedia_v2": 0.2004, "finemath": 0.08233}}
{"step": 14950, "total_steps": 72479, "tokens": 7838105600, "loss": 4.921845614910126, "l_mix": 3.0477070808410645, "l_a": 3.1449851393699646, "l_c": 3.0965821146965027, "z_loss": 16.68249797821045, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20823253691196442, "tokens_per_second": 1405648.4324130279, "eta_seconds": 21457.54489991665, "data_wait_seconds": 2.3096799850463867e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20769, "cosmopedia_v2": 0.2004, "finemath": 0.08234}}
{"step": 14960, "total_steps": 72479, "tokens": 7843348480, "loss": 5.240415036678314, "l_mix": 3.2475299537181854, "l_a": 3.3405233323574066, "l_c": 3.2969392836093903, "z_loss": 16.46231508255005, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15745437145233154, "tokens_per_second": 1446776.3680112772, "eta_seconds": 20843.94114997387, "data_wait_seconds": 5.120038986206055e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20769, "cosmopedia_v2": 0.2004, "finemath": 0.08235}}
{"step": 14970, "total_steps": 72479, "tokens": 7848591360, "loss": 4.9882625341415405, "l_mix": 3.089903771877289, "l_a": 3.1815648674964905, "l_c": 3.1407993733882904, "z_loss": 16.49444603919983, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15249988436698914, "tokens_per_second": 1420874.6017777738, "eta_seconds": 21220.224891257287, "data_wait_seconds": 2.7298927307128906e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20768, "cosmopedia_v2": 0.2004, "finemath": 0.08236}}
{"step": 14980, "total_steps": 72479, "tokens": 7853834240, "loss": 3.18461149930954, "l_mix": 1.9666345864534378, "l_a": 2.0505419075489044, "l_c": 2.0029288977384567, "z_loss": 19.356772899627686, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22403742372989655, "tokens_per_second": 1389864.1958096323, "eta_seconds": 21689.914599490163, "data_wait_seconds": 3.460049629211426e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20768, "cosmopedia_v2": 0.2004, "finemath": 0.08238}}
{"step": 14990, "total_steps": 72479, "tokens": 7859077120, "loss": 4.7824766635894775, "l_mix": 2.961034804582596, "l_a": 3.054555356502533, "l_c": 3.0113688111305237, "z_loss": 16.645684719085693, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1998835802078247, "tokens_per_second": 1419824.0828491843, "eta_seconds": 21228.540349531173, "data_wait_seconds": 1.6421079635620117e-05, "mix": {"fineweb_hq": 0.50961, "dclm": 0.20767, "cosmopedia_v2": 0.2004, "finemath": 0.08232}}
{"step": 15000, "total_steps": 72479, "tokens": 7864320000, "loss": 4.738475799560547, "l_mix": 2.93106210231781, "l_a": 3.0350707471370697, "l_c": 2.9841274321079254, "z_loss": 16.54257297515869, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22936773300170898, "tokens_per_second": 1443014.1468041053, "eta_seconds": 20883.752261710168, "data_wait_seconds": 1.8596649169921875e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20767, "cosmopedia_v2": 0.2004, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.25865003, "gate_a2b/L1_absmax": 0.64491415, "gate_a2b/L1_rms": 0.28317311, "gate_a2b/L3_absmean": 0.31694314, "gate_a2b/L3_absmax": 0.49505815, "gate_a2b/L3_rms": 0.32184336, "gate_a2b/L5_absmean": 0.50432432, "gate_a2b/L5_absmax": 0.68305522, "gate_a2b/L5_rms": 0.51094717, "gate_c2b/L1_absmean": 0.0149668, "gate_c2b/L1_absmax": 0.4728522, "gate_c2b/L1_rms": 0.0556055, "gate_c2b/L3_absmean": 0.20903848, "gate_c2b/L3_absmax": 0.43881556, "gate_c2b/L3_rms": 0.21363644, "gate_c2b/L5_absmean": 0.53575587, "gate_c2b/L5_absmax": 0.73138106, "gate_c2b/L5_rms": 0.54789358, "gate_b2a/L7_absmean": 0.07858576, "gate_b2a/L7_absmax": 0.256264, "gate_b2a/L7_rms": 0.08192738, "gate_b2a/L11_absmean": 0.1253819, "gate_b2a/L11_absmax": 0.3794367, "gate_b2a/L11_rms": 0.12771031, "gate_b2a/L14_absmean": 0.3051976, "gate_b2a/L14_absmax": 0.48813349, "gate_b2a/L14_rms": 0.3063615, "gate_b2c/L3_absmean": 0.08014451, "gate_b2c/L3_absmax": 0.56675136, "gate_b2c/L3_rms": 0.10023613, "gate_b2c/L5_absmean": 0.11487725, "gate_b2c/L5_absmax": 0.67602617, "gate_b2c/L5_rms": 0.1211007, "gate_b2c/L6_absmean": 0.16941687, "gate_b2c/L6_absmax": 0.46361932, "gate_b2c/L6_rms": 0.17258404}, "streams": {"cos_a2b/round0": 0.379183, "cos_c2b/round0": 0.270042, "cos_b2a/round0": 0.152522, "cos_b2c/round0": 0.127974, "rms_a/round0": 0.888876, "rms_b/round0": 1.711274, "rms_c/round0": 1.282197, "cos_a2b/round1": 0.394445, "cos_c2b/round1": 0.262635, "cos_b2a/round1": 0.217099, "cos_b2c/round1": 0.134632, "rms_a/round1": 1.58687, "rms_b/round1": 3.258126, "rms_c/round1": 2.097882, "cos_a2b/round2": 0.518354, "cos_c2b/round2": 0.470558, "cos_b2a/round2": 0.072605, "cos_b2c/round2": 0.132076, "rms_a/round2": 3.445745, "rms_b/round2": 7.103434, "rms_c/round2": 3.348119}, "alpha_mean": 0.45663319805115066, "alpha_hist": [0.01707, 0.07101, 0.14064, 0.17494, 0.17883, 0.16321, 0.14145, 0.09221, 0.02023, 0.0004], "alpha_by_type": {"special": {"share": 0.00111, "mean": 0.57172, "std": 0.10921}, "whitespace": {"share": 0.02892, "mean": 0.56358, "std": 0.12645}, "digit": {"share": 0.04306, "mean": 0.53901, "std": 0.12203}, "word_start": {"share": 0.55133, "mean": 0.48207, "std": 0.17503}, "word_cont": {"share": 0.24956, "mean": 0.31579, "std": 0.14693}, "punct": {"share": 0.12602, "mean": 0.57053, "std": 0.16607}}, "alpha_p05": 0.16002, "alpha_p50": 0.46126, "alpha_p95": 0.75383}
{"step": 15010, "total_steps": 72479, "tokens": 7869562880, "loss": 4.761053025722504, "l_mix": 2.9465810656547546, "l_a": 3.0491213500499725, "l_c": 2.993785172700882, "z_loss": 15.999723672866821, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20519214868545532, "tokens_per_second": 1389968.1237044642, "eta_seconds": 21676.977017068864, "data_wait_seconds": 1.7344951629638672e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20766, "cosmopedia_v2": 0.2004, "finemath": 0.08235}}
{"step": 15020, "total_steps": 72479, "tokens": 7874805760, "loss": 4.741644740104675, "l_mix": 2.9349478781223297, "l_a": 3.0354283154010773, "l_c": 2.981375902891159, "z_loss": 16.556745290756226, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1886626034975052, "tokens_per_second": 1412615.9826731216, "eta_seconds": 21325.727983760833, "data_wait_seconds": 1.6450881958007812e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20766, "cosmopedia_v2": 0.2004, "finemath": 0.08236}}
{"step": 15030, "total_steps": 72479, "tokens": 7880048640, "loss": 4.642698705196381, "l_mix": 2.871727228164673, "l_a": 2.9762658774852753, "l_c": 2.921304941177368, "z_loss": 17.001482725143433, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20296505093574524, "tokens_per_second": 1465219.4011600364, "eta_seconds": 20556.526407003403, "data_wait_seconds": 1.5020370483398438e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.2004, "finemath": 0.08237}}
{"step": 15040, "total_steps": 72479, "tokens": 7885291520, "loss": 3.536026030778885, "l_mix": 2.173299938440323, "l_a": 2.302584618330002, "l_c": 2.234630972146988, "z_loss": 15.613571882247925, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2177397906780243, "tokens_per_second": 1473772.656955463, "eta_seconds": 20433.66613559723, "data_wait_seconds": 1.5616416931152344e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20047, "finemath": 0.08231}}
{"step": 15050, "total_steps": 72479, "tokens": 7890534400, "loss": 3.533766597509384, "l_mix": 2.1720940470695496, "l_a": 2.29705086350441, "l_c": 2.236595958471298, "z_loss": 15.78487491607666, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20169782638549805, "tokens_per_second": 1466786.9358437913, "eta_seconds": 20527.409139132502, "data_wait_seconds": 1.7076730728149414e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20764, "cosmopedia_v2": 0.20047, "finemath": 0.08233}}
{"step": 15060, "total_steps": 72479, "tokens": 7895777280, "loss": 3.51699498295784, "l_mix": 2.162436991930008, "l_a": 2.2838077545166016, "l_c": 2.2261112928390503, "z_loss": 15.822844386100769, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20366205275058746, "tokens_per_second": 1377208.6808897813, "eta_seconds": 21858.773539352416, "data_wait_seconds": 2.5153160095214844e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20764, "cosmopedia_v2": 0.20046, "finemath": 0.08234}}
{"step": 15070, "total_steps": 72479, "tokens": 7901020160, "loss": 3.4743448197841644, "l_mix": 2.135901689529419, "l_a": 2.2616899013519287, "l_c": 2.194478392601013, "z_loss": 15.926350593566895, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20209281146526337, "tokens_per_second": 1466843.3901107118, "eta_seconds": 20519.470582151414, "data_wait_seconds": 2.720952033996582e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20763, "cosmopedia_v2": 0.20046, "finemath": 0.08235}}
{"step": 15080, "total_steps": 72479, "tokens": 7906263040, "loss": 5.079330682754517, "l_mix": 3.1473975479602814, "l_a": 3.239268571138382, "l_c": 3.194962441921234, "z_loss": 16.63761830329895, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1529921591281891, "tokens_per_second": 1374632.7588037609, "eta_seconds": 21892.10661485195, "data_wait_seconds": 4.220008850097656e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20769, "cosmopedia_v2": 0.2004, "finemath": 0.08236}}
{"step": 15090, "total_steps": 72479, "tokens": 7911505920, "loss": 4.877628266811371, "l_mix": 3.0217444598674774, "l_a": 3.11352875828743, "l_c": 3.067306488752365, "z_loss": 16.332062244415283, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.256195992231369, "tokens_per_second": 1383446.0045929418, "eta_seconds": 21748.853176856042, "data_wait_seconds": 1.9758939743041992e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20769, "cosmopedia_v2": 0.2004, "finemath": 0.08237}}
{"step": 15100, "total_steps": 72479, "tokens": 7916748800, "loss": 4.689492881298065, "l_mix": 2.9024784862995148, "l_a": 3.0020447969436646, "l_c": 2.949146270751953, "z_loss": 16.57049798965454, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18517573177814484, "tokens_per_second": 1411509.680968166, "eta_seconds": 21312.72747018337, "data_wait_seconds": 2.1070241928100586e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20768, "cosmopedia_v2": 0.2004, "finemath": 0.08232}, "gates": {"gate_a2b/L1_absmean": 0.26001942, "gate_a2b/L1_absmax": 0.65278411, "gate_a2b/L1_rms": 0.28432629, "gate_a2b/L3_absmean": 0.3174634, "gate_a2b/L3_absmax": 0.49604675, "gate_a2b/L3_rms": 0.32241592, "gate_a2b/L5_absmean": 0.50437564, "gate_a2b/L5_absmax": 0.67637771, "gate_a2b/L5_rms": 0.51096743, "gate_c2b/L1_absmean": 0.01481105, "gate_c2b/L1_absmax": 0.47187036, "gate_c2b/L1_rms": 0.05558592, "gate_c2b/L3_absmean": 0.20848179, "gate_c2b/L3_absmax": 0.43476209, "gate_c2b/L3_rms": 0.21306543, "gate_c2b/L5_absmean": 0.53518021, "gate_c2b/L5_absmax": 0.72479802, "gate_c2b/L5_rms": 0.54734284, "gate_b2a/L7_absmean": 0.07876921, "gate_b2a/L7_absmax": 0.2562913, "gate_b2a/L7_rms": 0.08211961, "gate_b2a/L11_absmean": 0.12528741, "gate_b2a/L11_absmax": 0.38326156, "gate_b2a/L11_rms": 0.12771808, "gate_b2a/L14_absmean": 0.30491641, "gate_b2a/L14_absmax": 0.48341039, "gate_b2a/L14_rms": 0.30607313, "gate_b2c/L3_absmean": 0.08019111, "gate_b2c/L3_absmax": 0.56719476, "gate_b2c/L3_rms": 0.10045163, "gate_b2c/L5_absmean": 0.11487886, "gate_b2c/L5_absmax": 0.67623121, "gate_b2c/L5_rms": 0.1210587, "gate_b2c/L6_absmean": 0.16885599, "gate_b2c/L6_absmax": 0.46322882, "gate_b2c/L6_rms": 0.17211068}, "streams": {"cos_a2b/round0": 0.377062, "cos_c2b/round0": 0.280761, "cos_b2a/round0": 0.155779, "cos_b2c/round0": 0.130848, "rms_a/round0": 0.885446, "rms_b/round0": 1.682247, "rms_c/round0": 1.270952, "cos_a2b/round1": 0.395234, "cos_c2b/round1": 0.272177, "cos_b2a/round1": 0.222613, "cos_b2c/round1": 0.138424, "rms_a/round1": 1.582692, "rms_b/round1": 3.198019, "rms_c/round1": 2.075391, "cos_a2b/round2": 0.514159, "cos_c2b/round2": 0.494748, "cos_b2a/round2": 0.073126, "cos_b2c/round2": 0.138206, "rms_a/round2": 3.485564, "rms_b/round2": 7.280416, "rms_c/round2": 3.370317}, "alpha_mean": 0.418259908846764, "alpha_hist": [0.02322, 0.08751, 0.1432, 0.19319, 0.21919, 0.19151, 0.1035, 0.03574, 0.00288, 5e-05], "alpha_by_type": {"special": {"share": 0.00095, "mean": 0.55979, "std": 0.10218}, "whitespace": {"share": 0.03346, "mean": 0.5448, "std": 0.09132}, "digit": {"share": 0.06544, "mean": 0.25994, "std": 0.17172}, "word_start": {"share": 0.56169, "mean": 0.45106, "std": 0.14352}, "word_cont": {"share": 0.2274, "mean": 0.31229, "std": 0.13681}, "punct": {"share": 0.11105, "mean": 0.52333, "std": 0.15103}}, "alpha_p05": 0.1341, "alpha_p50": 0.42776, "alpha_p95": 0.68363}
{"step": 15110, "total_steps": 72479, "tokens": 7921991680, "loss": 4.712786257266998, "l_mix": 2.9166598320007324, "l_a": 3.018875867128372, "l_c": 2.9626805186271667, "z_loss": 16.59452199935913, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.160178080201149, "tokens_per_second": 1467896.956159942, "eta_seconds": 20490.456190252306, "data_wait_seconds": 2.372264862060547e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20768, "cosmopedia_v2": 0.2004, "finemath": 0.08233}}
{"step": 15120, "total_steps": 72479, "tokens": 7927234560, "loss": 4.759715139865875, "l_mix": 2.945715844631195, "l_a": 3.046857625246048, "l_c": 2.994366466999054, "z_loss": 16.31908869743347, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1495261788368225, "tokens_per_second": 1450050.065796892, "eta_seconds": 20739.032466077806, "data_wait_seconds": 1.913309097290039e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20767, "cosmopedia_v2": 0.2004, "finemath": 0.08234}}
{"step": 15130, "total_steps": 72479, "tokens": 7932477440, "loss": 4.735825181007385, "l_mix": 2.9304713904857635, "l_a": 3.0366083085536957, "l_c": 2.9756759703159332, "z_loss": 16.683584928512573, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.13965648412704468, "tokens_per_second": 1468208.3217701744, "eta_seconds": 20478.96886713505, "data_wait_seconds": 1.6450881958007812e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.2004, "finemath": 0.08235}}
{"step": 15140, "total_steps": 72479, "tokens": 7937720320, "loss": 4.677198350429535, "l_mix": 2.8945292234420776, "l_a": 2.9924311339855194, "l_c": 2.94426628947258, "z_loss": 16.59862732887268, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.14386452734470367, "tokens_per_second": 1380683.4665951908, "eta_seconds": 21773.38279144764, "data_wait_seconds": 2.6017427444458008e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.2004, "finemath": 0.08236}}
{"step": 15150, "total_steps": 72479, "tokens": 7942963200, "loss": 3.474124014377594, "l_mix": 2.1360092759132385, "l_a": 2.2617269158363342, "l_c": 2.193518191576004, "z_loss": 15.411506414413452, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2219148427248001, "tokens_per_second": 1441539.3192912238, "eta_seconds": 20850.563248443603, "data_wait_seconds": 2.2083520889282227e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20046, "finemath": 0.08231}}
{"step": 15160, "total_steps": 72479, "tokens": 7948206080, "loss": 3.4463304579257965, "l_mix": 2.117499679327011, "l_a": 2.244075894355774, "l_c": 2.180172950029373, "z_loss": 15.561012387275696, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22743380069732666, "tokens_per_second": 1443377.1909380762, "eta_seconds": 20820.38157501221, "data_wait_seconds": 2.9861927032470703e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20046, "finemath": 0.08232}}
{"step": 15170, "total_steps": 72479, "tokens": 7953448960, "loss": 3.4966356456279755, "l_mix": 2.149429678916931, "l_a": 2.274214804172516, "l_c": 2.2113557755947113, "z_loss": 15.34789264202118, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19126471877098083, "tokens_per_second": 1303098.4054972986, "eta_seconds": 23057.67612426281, "data_wait_seconds": 3.612041473388672e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20046, "finemath": 0.08233}}
{"step": 15180, "total_steps": 72479, "tokens": 7958691840, "loss": 3.5149334967136383, "l_mix": 2.160747677087784, "l_a": 2.287042200565338, "l_c": 2.221701830625534, "z_loss": 15.625545263290405, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1919470876455307, "tokens_per_second": 1418674.8900890187, "eta_seconds": 21175.519720458986, "data_wait_seconds": 2.3096799850463867e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20764, "cosmopedia_v2": 0.20046, "finemath": 0.08235}}
{"step": 15190, "total_steps": 72479, "tokens": 7963934720, "loss": 3.4895971715450287, "l_mix": 2.144938290119171, "l_a": 2.2691001892089844, "l_c": 2.207824945449829, "z_loss": 15.813384056091309, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21530094742774963, "tokens_per_second": 1469136.5373613036, "eta_seconds": 20444.61795630455, "data_wait_seconds": 3.403425216674805e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20764, "cosmopedia_v2": 0.20046, "finemath": 0.08236}}
{"step": 15200, "total_steps": 72479, "tokens": 7969177600, "loss": 3.550201863050461, "l_mix": 2.1824151277542114, "l_a": 2.3091453909873962, "l_c": 2.244968682527542, "z_loss": 15.52463448047638, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1998109221458435, "tokens_per_second": 1426032.1296439504, "eta_seconds": 21058.91706626415, "data_wait_seconds": 1.677870750427246e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20763, "cosmopedia_v2": 0.20046, "finemath": 0.08237}, "gates": {"gate_a2b/L1_absmean": 0.26116693, "gate_a2b/L1_absmax": 0.64567661, "gate_a2b/L1_rms": 0.2851679, "gate_a2b/L3_absmean": 0.31789234, "gate_a2b/L3_absmax": 0.49581227, "gate_a2b/L3_rms": 0.32289743, "gate_a2b/L5_absmean": 0.5047487, "gate_a2b/L5_absmax": 0.68069643, "gate_a2b/L5_rms": 0.51128221, "gate_c2b/L1_absmean": 0.0147628, "gate_c2b/L1_absmax": 0.46988738, "gate_c2b/L1_rms": 0.05534608, "gate_c2b/L3_absmean": 0.20889845, "gate_c2b/L3_absmax": 0.431456, "gate_c2b/L3_rms": 0.21347117, "gate_c2b/L5_absmean": 0.53531998, "gate_c2b/L5_absmax": 0.72789705, "gate_c2b/L5_rms": 0.54747993, "gate_b2a/L7_absmean": 0.07892692, "gate_b2a/L7_absmax": 0.25481379, "gate_b2a/L7_rms": 0.0822541, "gate_b2a/L11_absmean": 0.12597305, "gate_b2a/L11_absmax": 0.38746145, "gate_b2a/L11_rms": 0.12839481, "gate_b2a/L14_absmean": 0.30566147, "gate_b2a/L14_absmax": 0.48961574, "gate_b2a/L14_rms": 0.30681458, "gate_b2c/L3_absmean": 0.08157402, "gate_b2c/L3_absmax": 0.56754559, "gate_b2c/L3_rms": 0.10156506, "gate_b2c/L5_absmean": 0.11527656, "gate_b2c/L5_absmax": 0.67999375, "gate_b2c/L5_rms": 0.12155416, "gate_b2c/L6_absmean": 0.16974798, "gate_b2c/L6_absmax": 0.46920162, "gate_b2c/L6_rms": 0.17298646}, "streams": {"cos_a2b/round0": 0.367209, "cos_c2b/round0": 0.23606, "cos_b2a/round0": 0.138909, "cos_b2c/round0": 0.091363, "rms_a/round0": 0.911449, "rms_b/round0": 1.692674, "rms_c/round0": 1.283725, "cos_a2b/round1": 0.400569, "cos_c2b/round1": 0.267458, "cos_b2a/round1": 0.217596, "cos_b2c/round1": 0.139414, "rms_a/round1": 1.618903, "rms_b/round1": 3.295877, "rms_c/round1": 2.136364, "cos_a2b/round2": 0.497142, "cos_c2b/round2": 0.464691, "cos_b2a/round2": 0.036707, "cos_b2c/round2": 0.112885, "rms_a/round2": 3.469118, "rms_b/round2": 7.185258, "rms_c/round2": 3.448058}, "alpha_mean": 0.43272903664404794, "alpha_hist": [0.04131, 0.13206, 0.16214, 0.14804, 0.13007, 0.12419, 0.10956, 0.09789, 0.04933, 0.00542], "alpha_by_type": {"special": {"share": 0.0011, "mean": 0.59714, "std": 0.10827}, "whitespace": {"share": 0.0361, "mean": 0.65008, "std": 0.16265}, "digit": {"share": 0.00963, "mean": 0.40602, "std": 0.14406}, "word_start": {"share": 0.57814, "mean": 0.44523, "std": 0.20929}, "word_cont": {"share": 0.25836, "mean": 0.28578, "std": 0.15983}, "punct": {"share": 0.11667, "mean": 0.6296, "std": 0.18268}}, "alpha_p05": 0.10669, "alpha_p50": 0.40686, "alpha_p95": 0.78695}
{"step": 15210, "total_steps": 72479, "tokens": 7974420480, "loss": 4.7060229778289795, "l_mix": 2.913524329662323, "l_a": 3.006471663713455, "l_c": 2.9631106853485107, "z_loss": 16.23880136013031, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2154376208782196, "tokens_per_second": 1306128.872485335, "eta_seconds": 22988.12169649601, "data_wait_seconds": 1.7881393432617188e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20763, "cosmopedia_v2": 0.20046, "finemath": 0.08231}}
{"step": 15220, "total_steps": 72479, "tokens": 7979663360, "loss": 4.796227931976318, "l_mix": 2.968953847885132, "l_a": 3.0682495832443237, "l_c": 3.017121136188507, "z_loss": 16.62705969810486, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.14428465068340302, "tokens_per_second": 1411417.7259046778, "eta_seconds": 21269.54057683945, "data_wait_seconds": 1.6003847122192383e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20769, "cosmopedia_v2": 0.20039, "finemath": 0.08233}}
{"step": 15230, "total_steps": 72479, "tokens": 7984906240, "loss": 4.692025661468506, "l_mix": 2.903270125389099, "l_a": 3.005008190870285, "l_c": 2.9519443213939667, "z_loss": 16.697086811065674, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.153248593211174, "tokens_per_second": 1346095.6968744528, "eta_seconds": 22297.793375086785, "data_wait_seconds": 1.531839370727539e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20768, "cosmopedia_v2": 0.20039, "finemath": 0.08234}}
{"step": 15240, "total_steps": 72479, "tokens": 7990149120, "loss": 4.717527389526367, "l_mix": 2.9192066192626953, "l_a": 3.0201676189899445, "l_c": 2.9686509668827057, "z_loss": 16.751643896102905, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1570686399936676, "tokens_per_second": 1386070.6450814237, "eta_seconds": 21650.9316740036, "data_wait_seconds": 2.2858381271362305e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20768, "cosmopedia_v2": 0.20039, "finemath": 0.08235}}
{"step": 15250, "total_steps": 72479, "tokens": 7995392000, "loss": 4.68218719959259, "l_mix": 2.8975807428359985, "l_a": 3.0002799332141876, "l_c": 2.94289967417717, "z_loss": 16.52589225769043, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1647188663482666, "tokens_per_second": 1468152.5465480888, "eta_seconds": 20436.893987989424, "data_wait_seconds": 1.627206802368164e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20039, "finemath": 0.08236}}
{"step": 15260, "total_steps": 72479, "tokens": 8000634880, "loss": 3.494537830352783, "l_mix": 2.1486030519008636, "l_a": 2.273431211709976, "l_c": 2.207915633916855, "z_loss": 15.30645763874054, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.24341852962970734, "tokens_per_second": 1464211.1075464669, "eta_seconds": 20488.326387763023, "data_wait_seconds": 3.057718276977539e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20046, "finemath": 0.08231}}
{"step": 15270, "total_steps": 72479, "tokens": 8005877760, "loss": 3.5236235857009888, "l_mix": 2.1658587753772736, "l_a": 2.2927713692188263, "l_c": 2.2279845476150513, "z_loss": 15.38038432598114, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21888503432273865, "tokens_per_second": 1455428.5225710154, "eta_seconds": 20608.358106803895, "data_wait_seconds": 1.9878149032592773e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20046, "finemath": 0.08232}}
{"step": 15280, "total_steps": 72479, "tokens": 8011120640, "loss": 3.5019561648368835, "l_mix": 2.1524583995342255, "l_a": 2.2778464257717133, "l_c": 2.2152852416038513, "z_loss": 15.582031965255737, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19393758475780487, "tokens_per_second": 1387655.7842024348, "eta_seconds": 21611.086591792107, "data_wait_seconds": 1.6987323760986328e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20766, "cosmopedia_v2": 0.20046, "finemath": 0.08233}}
{"step": 15290, "total_steps": 72479, "tokens": 8016363520, "loss": 3.527365267276764, "l_mix": 2.168936997652054, "l_a": 2.2901783883571625, "l_c": 2.232809901237488, "z_loss": 15.31656789779663, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2544701099395752, "tokens_per_second": 1453163.0515308373, "eta_seconds": 20633.270575118062, "data_wait_seconds": 1.6868114471435547e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20765, "cosmopedia_v2": 0.20046, "finemath": 0.08234}}
{"step": 15300, "total_steps": 72479, "tokens": 8021606400, "loss": 3.524253100156784, "l_mix": 2.166228324174881, "l_a": 2.2966450452804565, "l_c": 2.224883884191513, "z_loss": 15.660098552703857, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1975194215774536, "tokens_per_second": 1355532.7190650855, "eta_seconds": 22115.485026931765, "data_wait_seconds": 4.342198371887207e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20765, "cosmopedia_v2": 0.20046, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.26228103, "gate_a2b/L1_absmax": 0.64750236, "gate_a2b/L1_rms": 0.28623033, "gate_a2b/L3_absmean": 0.31842709, "gate_a2b/L3_absmax": 0.49611583, "gate_a2b/L3_rms": 0.32333583, "gate_a2b/L5_absmean": 0.50536346, "gate_a2b/L5_absmax": 0.68282241, "gate_a2b/L5_rms": 0.51186252, "gate_c2b/L1_absmean": 0.0146138, "gate_c2b/L1_absmax": 0.46845311, "gate_c2b/L1_rms": 0.05527609, "gate_c2b/L3_absmean": 0.20903587, "gate_c2b/L3_absmax": 0.43449819, "gate_c2b/L3_rms": 0.21361484, "gate_c2b/L5_absmean": 0.53523684, "gate_c2b/L5_absmax": 0.72843224, "gate_c2b/L5_rms": 0.54744858, "gate_b2a/L7_absmean": 0.07911215, "gate_b2a/L7_absmax": 0.25533307, "gate_b2a/L7_rms": 0.08249674, "gate_b2a/L11_absmean": 0.12534277, "gate_b2a/L11_absmax": 0.3862845, "gate_b2a/L11_rms": 0.12779422, "gate_b2a/L14_absmean": 0.30562919, "gate_b2a/L14_absmax": 0.48996648, "gate_b2a/L14_rms": 0.30677348, "gate_b2c/L3_absmean": 0.08197831, "gate_b2c/L3_absmax": 0.56914341, "gate_b2c/L3_rms": 0.10183061, "gate_b2c/L5_absmean": 0.11547188, "gate_b2c/L5_absmax": 0.68439984, "gate_b2c/L5_rms": 0.12181944, "gate_b2c/L6_absmean": 0.17027216, "gate_b2c/L6_absmax": 0.46718055, "gate_b2c/L6_rms": 0.17348178}, "streams": {"cos_a2b/round0": 0.367969, "cos_c2b/round0": 0.244364, "cos_b2a/round0": 0.14224, "cos_b2c/round0": 0.095427, "rms_a/round0": 0.906614, "rms_b/round0": 1.69111, "rms_c/round0": 1.279573, "cos_a2b/round1": 0.400595, "cos_c2b/round1": 0.264694, "cos_b2a/round1": 0.215008, "cos_b2c/round1": 0.132044, "rms_a/round1": 1.611363, "rms_b/round1": 3.282415, "rms_c/round1": 2.158911, "cos_a2b/round2": 0.49523, "cos_c2b/round2": 0.464922, "cos_b2a/round2": 0.038881, "cos_b2c/round2": 0.114545, "rms_a/round2": 3.451386, "rms_b/round2": 7.18011, "rms_c/round2": 3.463233}, "alpha_mean": 0.4309772014463107, "alpha_hist": [0.04739, 0.14122, 0.17514, 0.14473, 0.11771, 0.09712, 0.09964, 0.09732, 0.06989, 0.00984], "alpha_by_type": {"special": {"share": 0.00099, "mean": 0.63598, "std": 0.11326}, "whitespace": {"share": 0.03667, "mean": 0.62976, "std": 0.20308}, "digit": {"share": 0.01228, "mean": 0.37354, "std": 0.17403}, "word_start": {"share": 0.55496, "mean": 0.45371, "std": 0.22064}, "word_cont": {"share": 0.26926, "mean": 0.27055, "std": 0.15988}, "punct": {"share": 0.12584, "mean": 0.62006, "std": 0.21338}}, "alpha_p05": 0.10195, "alpha_p50": 0.38862, "alpha_p95": 0.82047}
{"step": 15310, "total_steps": 72479, "tokens": 8026849280, "loss": 3.4923587441444397, "l_mix": 2.146847188472748, "l_a": 2.2710851430892944, "l_c": 2.2088311910629272, "z_loss": 15.366304993629456, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21040326356887817, "tokens_per_second": 1470675.7811165354, "eta_seconds": 20380.44078569412, "data_wait_seconds": 1.806020736694336e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20764, "cosmopedia_v2": 0.20046, "finemath": 0.08236}}
{"step": 15320, "total_steps": 72479, "tokens": 8032092160, "loss": 4.812137603759766, "l_mix": 2.979397237300873, "l_a": 3.079607665538788, "l_c": 3.0239712595939636, "z_loss": 16.66616725921631, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1984250545501709, "tokens_per_second": 1411438.9243344516, "eta_seconds": 21232.07549071312, "data_wait_seconds": 2.0265579223632812e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20764, "cosmopedia_v2": 0.20046, "finemath": 0.08231}}
{"step": 15330, "total_steps": 72479, "tokens": 8037335040, "loss": 4.691742360591888, "l_mix": 2.902866095304489, "l_a": 3.0048319697380066, "l_c": 2.952476888895035, "z_loss": 16.835721492767334, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19192640483379364, "tokens_per_second": 1475428.1450957863, "eta_seconds": 20307.68832192421, "data_wait_seconds": 2.205371856689453e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20763, "cosmopedia_v2": 0.20046, "finemath": 0.08232}}
{"step": 15340, "total_steps": 72479, "tokens": 8042577920, "loss": 4.74686586856842, "l_mix": 2.9381136298179626, "l_a": 3.039115995168686, "l_c": 2.984620153903961, "z_loss": 16.313471913337708, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19576025009155273, "tokens_per_second": 1413329.1374496575, "eta_seconds": 21196.260119605064, "data_wait_seconds": 1.773238182067871e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20763, "cosmopedia_v2": 0.20046, "finemath": 0.08233}}
{"step": 15350, "total_steps": 72479, "tokens": 8047820800, "loss": 4.6947489976882935, "l_mix": 2.9053471982479095, "l_a": 3.0075563788414, "l_c": 2.951686382293701, "z_loss": 16.288273096084595, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18610204756259918, "tokens_per_second": 1349200.8818318064, "eta_seconds": 22199.84403755665, "data_wait_seconds": 1.52587890625e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20762, "cosmopedia_v2": 0.20046, "finemath": 0.08235}}
{"step": 15360, "total_steps": 72479, "tokens": 8053063680, "loss": 4.9605631828308105, "l_mix": 3.0735469460487366, "l_a": 3.162283569574356, "l_c": 3.1223742365837097, "z_loss": 16.187620162963867, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1698540598154068, "tokens_per_second": 1399720.0943015637, "eta_seconds": 21394.85343813896, "data_wait_seconds": 2.3037195205688477e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20768, "cosmopedia_v2": 0.20039, "finemath": 0.08236}}
{"step": 15370, "total_steps": 72479, "tokens": 8058306560, "loss": 4.903998017311096, "l_mix": 3.038235902786255, "l_a": 3.1293010115623474, "l_c": 3.0844371616840363, "z_loss": 16.40563726425171, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18216781318187714, "tokens_per_second": 1406705.8767455937, "eta_seconds": 21284.878301119807, "data_wait_seconds": 2.8252601623535156e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.20039, "finemath": 0.08237}}
{"step": 15380, "total_steps": 72479, "tokens": 8063549440, "loss": 3.4694577157497406, "l_mix": 2.132265329360962, "l_a": 2.258092612028122, "l_c": 2.193969041109085, "z_loss": 15.737717747688293, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21865957975387573, "tokens_per_second": 1437083.6565007742, "eta_seconds": 20831.29981791973, "data_wait_seconds": 3.573298454284668e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20046, "finemath": 0.08231}}
{"step": 15390, "total_steps": 72479, "tokens": 8068792320, "loss": 3.51602703332901, "l_mix": 2.160929709672928, "l_a": 2.28889137506485, "l_c": 2.223005384206772, "z_loss": 15.282242894172668, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19211851060390472, "tokens_per_second": 1441366.786378592, "eta_seconds": 20765.760606431963, "data_wait_seconds": 2.5510787963867188e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20045, "finemath": 0.08233}}
{"step": 15400, "total_steps": 72479, "tokens": 8074035200, "loss": 3.4843413531780243, "l_mix": 2.142109602689743, "l_a": 2.2661812901496887, "l_c": 2.20285564661026, "z_loss": 15.206894636154175, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20831841230392456, "tokens_per_second": 1292990.3761445035, "eta_seconds": 23144.669368100163, "data_wait_seconds": 7.617473602294922e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20766, "cosmopedia_v2": 0.20045, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.26366481, "gate_a2b/L1_absmax": 0.65034038, "gate_a2b/L1_rms": 0.2874608, "gate_a2b/L3_absmean": 0.31917062, "gate_a2b/L3_absmax": 0.49453688, "gate_a2b/L3_rms": 0.32403862, "gate_a2b/L5_absmean": 0.5062983, "gate_a2b/L5_absmax": 0.68128902, "gate_a2b/L5_rms": 0.51280814, "gate_c2b/L1_absmean": 0.01493609, "gate_c2b/L1_absmax": 0.46502587, "gate_c2b/L1_rms": 0.054901, "gate_c2b/L3_absmean": 0.2087308, "gate_c2b/L3_absmax": 0.43410835, "gate_c2b/L3_rms": 0.21327467, "gate_c2b/L5_absmean": 0.53537649, "gate_c2b/L5_absmax": 0.72353709, "gate_c2b/L5_rms": 0.54749858, "gate_b2a/L7_absmean": 0.07922448, "gate_b2a/L7_absmax": 0.25703907, "gate_b2a/L7_rms": 0.08252539, "gate_b2a/L11_absmean": 0.12629139, "gate_b2a/L11_absmax": 0.38487178, "gate_b2a/L11_rms": 0.12868193, "gate_b2a/L14_absmean": 0.30573386, "gate_b2a/L14_absmax": 0.49310794, "gate_b2a/L14_rms": 0.30692169, "gate_b2c/L3_absmean": 0.08260324, "gate_b2c/L3_absmax": 0.57029915, "gate_b2c/L3_rms": 0.10235545, "gate_b2c/L5_absmean": 0.11580323, "gate_b2c/L5_absmax": 0.6843679, "gate_b2c/L5_rms": 0.12218584, "gate_b2c/L6_absmean": 0.17106354, "gate_b2c/L6_absmax": 0.46932852, "gate_b2c/L6_rms": 0.1742902}, "streams": {"cos_a2b/round0": 0.367423, "cos_c2b/round0": 0.234109, "cos_b2a/round0": 0.13824, "cos_b2c/round0": 0.090608, "rms_a/round0": 0.906878, "rms_b/round0": 1.687565, "rms_c/round0": 1.273881, "cos_a2b/round1": 0.400303, "cos_c2b/round1": 0.265034, "cos_b2a/round1": 0.216472, "cos_b2c/round1": 0.13991, "rms_a/round1": 1.614376, "rms_b/round1": 3.291886, "rms_c/round1": 2.157327, "cos_a2b/round2": 0.49702, "cos_c2b/round2": 0.462376, "cos_b2a/round2": 0.032289, "cos_b2c/round2": 0.112337, "rms_a/round2": 3.454567, "rms_b/round2": 7.149935, "rms_c/round2": 3.465334}, "alpha_mean": 0.4402903080663805, "alpha_hist": [0.04179, 0.13635, 0.16632, 0.14577, 0.11475, 0.10242, 0.10873, 0.10799, 0.06674, 0.00914], "alpha_by_type": {"special": {"share": 0.00111, "mean": 0.64084, "std": 0.12315}, "whitespace": {"share": 0.03265, "mean": 0.66342, "std": 0.18507}, "digit": {"share": 0.01013, "mean": 0.45599, "std": 0.18681}, "word_start": {"share": 0.56476, "mean": 0.45632, "std": 0.21931}, "word_cont": {"share": 0.26936, "mean": 0.28247, "std": 0.16127}, "punct": {"share": 0.12198, "mean": 0.65172, "std": 0.19096}}, "alpha_p05": 0.12379, "alpha_p50": 0.4168, "alpha_p95": 0.82104}
{"step": 15410, "total_steps": 72479, "tokens": 8079278080, "loss": 3.498218536376953, "l_mix": 2.1504472196102142, "l_a": 2.277252048254013, "l_c": 2.2101120054721832, "z_loss": 15.620108842849731, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20477570593357086, "tokens_per_second": 1233901.7949978297, "eta_seconds": 24248.76274051666, "data_wait_seconds": 3.355741500854492e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20766, "cosmopedia_v2": 0.20045, "finemath": 0.08235}}
{"step": 15420, "total_steps": 72479, "tokens": 8084520960, "loss": 3.5060126781463623, "l_mix": 2.155348688364029, "l_a": 2.2808177769184113, "l_c": 2.2163005471229553, "z_loss": 15.28485882282257, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.24658328294754028, "tokens_per_second": 1345102.6977507907, "eta_seconds": 22240.197006535527, "data_wait_seconds": 1.7851591110229492e-05, "mix": {"fineweb_hq": 0.50953, "dclm": 0.20765, "cosmopedia_v2": 0.20045, "finemath": 0.08236}}
{"step": 15430, "total_steps": 72479, "tokens": 8089763840, "loss": 4.7609864473342896, "l_mix": 2.946706384420395, "l_a": 3.046398401260376, "l_c": 2.995669901371002, "z_loss": 16.594598054885864, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18994951248168945, "tokens_per_second": 1469686.9777299664, "eta_seconds": 20351.344582366946, "data_wait_seconds": 1.609325408935547e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20765, "cosmopedia_v2": 0.20045, "finemath": 0.08231}}
{"step": 15440, "total_steps": 72479, "tokens": 8095006720, "loss": 4.71184515953064, "l_mix": 2.916674017906189, "l_a": 3.0142319798469543, "l_c": 2.964191436767578, "z_loss": 16.439969778060913, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20656874775886536, "tokens_per_second": 1295133.4391801697, "eta_seconds": 23090.17922580242, "data_wait_seconds": 2.3245811462402344e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20764, "cosmopedia_v2": 0.20045, "finemath": 0.08232}}
{"step": 15450, "total_steps": 72479, "tokens": 8100249600, "loss": 4.5991822481155396, "l_mix": 2.846569299697876, "l_a": 2.9411374032497406, "l_c": 2.895561307668686, "z_loss": 16.031789898872375, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18685458600521088, "tokens_per_second": 1464399.0999747813, "eta_seconds": 20417.671898674962, "data_wait_seconds": 1.5139579772949219e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20045, "finemath": 0.08233}}
{"step": 15460, "total_steps": 72479, "tokens": 8105492480, "loss": 4.6959593296051025, "l_mix": 2.9054016768932343, "l_a": 3.0066971480846405, "l_c": 2.956201493740082, "z_loss": 16.880964756011963, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1939513236284256, "tokens_per_second": 1419791.5398441656, "eta_seconds": 21055.469505953788, "data_wait_seconds": 2.1129846572875977e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20763, "cosmopedia_v2": 0.20045, "finemath": 0.08234}}
{"step": 15470, "total_steps": 72479, "tokens": 8110735360, "loss": 3.5383208096027374, "l_mix": 2.174945205450058, "l_a": 2.299884170293808, "l_c": 2.239594578742981, "z_loss": 15.320064663887024, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19344064593315125, "tokens_per_second": 1465696.0796690374, "eta_seconds": 20392.45039036274, "data_wait_seconds": 1.570582389831543e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20763, "cosmopedia_v2": 0.20045, "finemath": 0.08235}}
{"step": 15480, "total_steps": 72479, "tokens": 8115978240, "loss": 3.5140307247638702, "l_mix": 2.1609100997447968, "l_a": 2.2803571224212646, "l_c": 2.2248623967170715, "z_loss": 15.54763114452362, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2100108414888382, "tokens_per_second": 1447029.987916539, "eta_seconds": 20651.88141334057, "data_wait_seconds": 2.2739171981811523e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20762, "cosmopedia_v2": 0.20045, "finemath": 0.08236}}
{"step": 15490, "total_steps": 72479, "tokens": 8121221120, "loss": 3.484903186559677, "l_mix": 2.1426468193531036, "l_a": 2.2648647725582123, "l_c": 2.204019218683243, "z_loss": 15.91117537021637, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2502429783344269, "tokens_per_second": 1363314.648175546, "eta_seconds": 21916.179710960387, "data_wait_seconds": 1.722574234008789e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20762, "cosmopedia_v2": 0.20045, "finemath": 0.08238}}
{"step": 15500, "total_steps": 72479, "tokens": 8126464000, "loss": 3.5035282373428345, "l_mix": 2.1540321111679077, "l_a": 2.2784055173397064, "l_c": 2.2146372199058533, "z_loss": 15.832303285598755, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20684130489826202, "tokens_per_second": 1444672.9647470019, "eta_seconds": 20678.31729462147, "data_wait_seconds": 1.800060272216797e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20768, "cosmopedia_v2": 0.20045, "finemath": 0.08232}, "gates": {"gate_a2b/L1_absmean": 0.26490185, "gate_a2b/L1_absmax": 0.64332372, "gate_a2b/L1_rms": 0.28859243, "gate_a2b/L3_absmean": 0.31940705, "gate_a2b/L3_absmax": 0.4927758, "gate_a2b/L3_rms": 0.32427827, "gate_a2b/L5_absmean": 0.50678015, "gate_a2b/L5_absmax": 0.68198854, "gate_a2b/L5_rms": 0.51324892, "gate_c2b/L1_absmean": 0.01522084, "gate_c2b/L1_absmax": 0.46673897, "gate_c2b/L1_rms": 0.05518302, "gate_c2b/L3_absmean": 0.20892373, "gate_c2b/L3_absmax": 0.43207484, "gate_c2b/L3_rms": 0.21337773, "gate_c2b/L5_absmean": 0.53539914, "gate_c2b/L5_absmax": 0.72813427, "gate_c2b/L5_rms": 0.54749542, "gate_b2a/L7_absmean": 0.07950799, "gate_b2a/L7_absmax": 0.25712335, "gate_b2a/L7_rms": 0.08280892, "gate_b2a/L11_absmean": 0.1263583, "gate_b2a/L11_absmax": 0.39087519, "gate_b2a/L11_rms": 0.12880257, "gate_b2a/L14_absmean": 0.30605656, "gate_b2a/L14_absmax": 0.49452153, "gate_b2a/L14_rms": 0.30722246, "gate_b2c/L3_absmean": 0.08319542, "gate_b2c/L3_absmax": 0.57275677, "gate_b2c/L3_rms": 0.10285357, "gate_b2c/L5_absmean": 0.11631029, "gate_b2c/L5_absmax": 0.69181126, "gate_b2c/L5_rms": 0.12272975, "gate_b2c/L6_absmean": 0.17088383, "gate_b2c/L6_absmax": 0.47066006, "gate_b2c/L6_rms": 0.17415507}, "streams": {"cos_a2b/round0": 0.364902, "cos_c2b/round0": 0.240975, "cos_b2a/round0": 0.136475, "cos_b2c/round0": 0.092342, "rms_a/round0": 0.896846, "rms_b/round0": 1.677101, "rms_c/round0": 1.274254, "cos_a2b/round1": 0.399357, "cos_c2b/round1": 0.261471, "cos_b2a/round1": 0.214744, "cos_b2c/round1": 0.132705, "rms_a/round1": 1.607919, "rms_b/round1": 3.285796, "rms_c/round1": 2.130822, "cos_a2b/round2": 0.486645, "cos_c2b/round2": 0.474423, "cos_b2a/round2": 0.040416, "cos_b2c/round2": 0.109856, "rms_a/round2": 3.402279, "rms_b/round2": 7.227492, "rms_c/round2": 3.456231}, "alpha_mean": 0.4466146740140289, "alpha_hist": [0.04242, 0.12306, 0.1581, 0.14052, 0.13838, 0.11229, 0.09967, 0.10182, 0.07596, 0.00778], "alpha_by_type": {"special": {"share": 0.00099, "mean": 0.68456, "std": 0.09677}, "whitespace": {"share": 0.03874, "mean": 0.63911, "std": 0.1715}, "digit": {"share": 0.02118, "mean": 0.41656, "std": 0.12405}, "word_start": {"share": 0.5381, "mean": 0.46988, "std": 0.21994}, "word_cont": {"share": 0.28223, "mean": 0.29302, "std": 0.16528}, "punct": {"share": 0.11876, "mean": 0.6468, "std": 0.19744}}, "alpha_p05": 0.1132, "alpha_p50": 0.42179, "alpha_p95": 0.82446}
{"step": 15510, "total_steps": 72479, "tokens": 8131706880, "loss": 3.4628027379512787, "l_mix": 2.1286798417568207, "l_a": 2.255331963300705, "l_c": 2.186519533395767, "z_loss": 15.674248456954956, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20595985651016235, "tokens_per_second": 1431167.5284941855, "eta_seconds": 20869.788111686707, "data_wait_seconds": 2.0831823348999023e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20767, "cosmopedia_v2": 0.20045, "finemath": 0.08233}}
{"step": 15520, "total_steps": 72479, "tokens": 8136949760, "loss": 3.522726386785507, "l_mix": 2.165014088153839, "l_a": 2.2930315732955933, "l_c": 2.2275359332561493, "z_loss": 15.420050144195557, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.23164743185043335, "tokens_per_second": 1412234.234682462, "eta_seconds": 21145.869048213957, "data_wait_seconds": 1.6063451766967773e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20767, "cosmopedia_v2": 0.20045, "finemath": 0.08235}}
{"step": 15530, "total_steps": 72479, "tokens": 8142192640, "loss": 3.5000567734241486, "l_mix": 2.1510300636291504, "l_a": 2.277561604976654, "l_c": 2.2140738666057587, "z_loss": 15.360060453414917, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1878300905227661, "tokens_per_second": 1409516.1754224643, "eta_seconds": 21182.926335024833, "data_wait_seconds": 2.8520822525024414e-05, "mix": {"fineweb_hq": 0.50953, "dclm": 0.20766, "cosmopedia_v2": 0.20045, "finemath": 0.08236}}
{"step": 15540, "total_steps": 72479, "tokens": 8147435520, "loss": 4.718141615390778, "l_mix": 2.9200194776058197, "l_a": 3.0223337411880493, "l_c": 2.9658758640289307, "z_loss": 16.591607809066772, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16435390710830688, "tokens_per_second": 1410195.8167018774, "eta_seconds": 21168.99942436218, "data_wait_seconds": 2.327561378479004e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20766, "cosmopedia_v2": 0.20045, "finemath": 0.0823}}
{"step": 15550, "total_steps": 72479, "tokens": 8152678400, "loss": 4.6612823605537415, "l_mix": 2.8844196498394012, "l_a": 2.98392853140831, "l_c": 2.933411717414856, "z_loss": 16.60516917705536, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16209810972213745, "tokens_per_second": 1433745.3371508762, "eta_seconds": 20817.63809695244, "data_wait_seconds": 1.6003847122192383e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20045, "finemath": 0.08232}}
{"step": 15560, "total_steps": 72479, "tokens": 8157921280, "loss": 4.702083945274353, "l_mix": 2.909811943769455, "l_a": 3.0110901296138763, "l_c": 2.95761975646019, "z_loss": 16.59013080596924, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22101615369319916, "tokens_per_second": 1312127.4471914563, "eta_seconds": 22743.178443431854, "data_wait_seconds": 1.6927719116210938e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20045, "finemath": 0.08233}}
{"step": 15570, "total_steps": 72479, "tokens": 8163164160, "loss": 3.5110189616680145, "l_mix": 2.158364027738571, "l_a": 2.282216638326645, "l_c": 2.2213170528411865, "z_loss": 15.947834372520447, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20618367195129395, "tokens_per_second": 1376078.068514947, "eta_seconds": 21682.422294688226, "data_wait_seconds": 1.722574234008789e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20764, "cosmopedia_v2": 0.20045, "finemath": 0.08234}}
{"step": 15580, "total_steps": 72479, "tokens": 8168407040, "loss": 3.493799149990082, "l_mix": 2.1476171910762787, "l_a": 2.275279462337494, "l_c": 2.2068295776844025, "z_loss": 15.491894841194153, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19527113437652588, "tokens_per_second": 1444658.538660894, "eta_seconds": 20649.490598416327, "data_wait_seconds": 3.212690353393555e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20764, "cosmopedia_v2": 0.20045, "finemath": 0.08235}}
{"step": 15590, "total_steps": 72479, "tokens": 8173649920, "loss": 3.5104449093341827, "l_mix": 2.1569310128688812, "l_a": 2.2863230109214783, "l_c": 2.220304936170578, "z_loss": 15.254294395446777, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.25075313448905945, "tokens_per_second": 1437428.5946117064, "eta_seconds": 20749.705511498454, "data_wait_seconds": 2.1278858184814453e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20763, "cosmopedia_v2": 0.20045, "finemath": 0.08236}}
{"step": 15600, "total_steps": 72479, "tokens": 8178892800, "loss": 3.555204689502716, "l_mix": 2.1852328181266785, "l_a": 2.3150074779987335, "l_c": 2.246379017829895, "z_loss": 15.558658480644226, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22487904131412506, "tokens_per_second": 1333729.3775576903, "eta_seconds": 22359.091472220418, "data_wait_seconds": 2.2172927856445312e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20763, "cosmopedia_v2": 0.20045, "finemath": 0.08237}, "gates": {"gate_a2b/L1_absmean": 0.26538214, "gate_a2b/L1_absmax": 0.64150339, "gate_a2b/L1_rms": 0.28891909, "gate_a2b/L3_absmean": 0.31957865, "gate_a2b/L3_absmax": 0.4956834, "gate_a2b/L3_rms": 0.32456151, "gate_a2b/L5_absmean": 0.50681752, "gate_a2b/L5_absmax": 0.67980289, "gate_a2b/L5_rms": 0.51329333, "gate_c2b/L1_absmean": 0.01565667, "gate_c2b/L1_absmax": 0.46829319, "gate_c2b/L1_rms": 0.05503234, "gate_c2b/L3_absmean": 0.2086703, "gate_c2b/L3_absmax": 0.42680237, "gate_c2b/L3_rms": 0.21313344, "gate_c2b/L5_absmean": 0.53439409, "gate_c2b/L5_absmax": 0.71957922, "gate_c2b/L5_rms": 0.54642737, "gate_b2a/L7_absmean": 0.08016531, "gate_b2a/L7_absmax": 0.25673139, "gate_b2a/L7_rms": 0.08354394, "gate_b2a/L11_absmean": 0.12632313, "gate_b2a/L11_absmax": 0.38967213, "gate_b2a/L11_rms": 0.1288562, "gate_b2a/L14_absmean": 0.30647981, "gate_b2a/L14_absmax": 0.48899916, "gate_b2a/L14_rms": 0.3076703, "gate_b2c/L3_absmean": 0.08246574, "gate_b2c/L3_absmax": 0.5795173, "gate_b2c/L3_rms": 0.10229069, "gate_b2c/L5_absmean": 0.11540599, "gate_b2c/L5_absmax": 0.69378281, "gate_b2c/L5_rms": 0.12195783, "gate_b2c/L6_absmean": 0.17038783, "gate_b2c/L6_absmax": 0.46964186, "gate_b2c/L6_rms": 0.17374164}, "streams": {"cos_a2b/round0": 0.36407, "cos_c2b/round0": 0.235075, "cos_b2a/round0": 0.130655, "cos_b2c/round0": 0.087062, "rms_a/round0": 0.906255, "rms_b/round0": 1.693287, "rms_c/round0": 1.285712, "cos_a2b/round1": 0.39879, "cos_c2b/round1": 0.262845, "cos_b2a/round1": 0.213693, "cos_b2c/round1": 0.138964, "rms_a/round1": 1.618026, "rms_b/round1": 3.303045, "rms_c/round1": 2.15162, "cos_a2b/round2": 0.495435, "cos_c2b/round2": 0.46612, "cos_b2a/round2": 0.031681, "cos_b2c/round2": 0.109383, "rms_a/round2": 3.447168, "rms_b/round2": 7.215682, "rms_c/round2": 3.460517}, "alpha_mean": 0.4189073693448293, "alpha_hist": [0.05858, 0.15637, 0.16562, 0.13889, 0.116, 0.10126, 0.10141, 0.09579, 0.05978, 0.0063], "alpha_by_type": {"special": {"share": 0.0011, "mean": 0.63895, "std": 0.12791}, "whitespace": {"share": 0.03452, "mean": 0.60176, "std": 0.19847}, "digit": {"share": 0.01003, "mean": 0.30207, "std": 0.16194}, "word_start": {"share": 0.57333, "mean": 0.44259, "std": 0.22117}, "word_cont": {"share": 0.26466, "mean": 0.26164, "std": 0.16151}, "punct": {"share": 0.11636, "mean": 0.61368, "std": 0.21155}}, "alpha_p05": 0.1023, "alpha_p50": 0.40757, "alpha_p95": 0.80193}
{"step": 15610, "total_steps": 72479, "tokens": 8184135680, "loss": 4.975034832954407, "l_mix": 3.082554429769516, "l_a": 3.1728172600269318, "l_c": 3.129884958267212, "z_loss": 16.696752071380615, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.23780874907970428, "tokens_per_second": 1200485.8097701587, "eta_seconds": 24836.390425729754, "data_wait_seconds": 2.4944543838500977e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20769, "cosmopedia_v2": 0.20045, "finemath": 0.08232}}
{"step": 15620, "total_steps": 72479, "tokens": 8189378560, "loss": 5.4282286167144775, "l_mix": 3.365712434053421, "l_a": 3.4588561356067657, "l_c": 3.4108246862888336, "z_loss": 16.119860410690308, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20639543235301971, "tokens_per_second": 1357018.5941374332, "eta_seconds": 21967.636641669273, "data_wait_seconds": 2.4437904357910156e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20768, "cosmopedia_v2": 0.20045, "finemath": 0.08233}}
{"step": 15630, "total_steps": 72479, "tokens": 8194621440, "loss": 4.865367650985718, "l_mix": 3.012657254934311, "l_a": 3.1043379306793213, "l_c": 3.0660101771354675, "z_loss": 16.058409452438354, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.24409936368465424, "tokens_per_second": 1375693.6923403386, "eta_seconds": 21665.61399383545, "data_wait_seconds": 3.209710121154785e-05, "mix": {"fineweb_hq": 0.50953, "dclm": 0.20768, "cosmopedia_v2": 0.20045, "finemath": 0.08234}}
{"step": 15640, "total_steps": 72479, "tokens": 8199864320, "loss": 3.538212150335312, "l_mix": 2.1750394701957703, "l_a": 2.303492844104767, "l_c": 2.235299050807953, "z_loss": 15.350812315940857, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2596646547317505, "tokens_per_second": 1366835.8870447332, "eta_seconds": 21802.18262810707, "data_wait_seconds": 2.008676528930664e-05, "mix": {"fineweb_hq": 0.50953, "dclm": 0.20767, "cosmopedia_v2": 0.20045, "finemath": 0.08235}}
{"step": 15650, "total_steps": 72479, "tokens": 8205107200, "loss": 4.598025441169739, "l_mix": 2.845598816871643, "l_a": 2.9422548711299896, "l_c": 2.893691807985306, "z_loss": 16.42519199848175, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1594626009464264, "tokens_per_second": 1390844.6173883213, "eta_seconds": 21422.064247512815, "data_wait_seconds": 2.1696090698242188e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20038, "finemath": 0.08236}}
{"step": 15660, "total_steps": 72479, "tokens": 8210350080, "loss": 4.769328594207764, "l_mix": 2.951728403568268, "l_a": 3.0516563653945923, "l_c": 3.001255691051483, "z_loss": 17.26367712020874, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.211579829454422, "tokens_per_second": 1288106.524259083, "eta_seconds": 23126.596528291702, "data_wait_seconds": 2.5957822799682617e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20045, "finemath": 0.08231}}
{"step": 15670, "total_steps": 72479, "tokens": 8215592960, "loss": 4.648766040802002, "l_mix": 2.876564919948578, "l_a": 2.9792264699935913, "l_c": 2.922543168067932, "z_loss": 16.70189356803894, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1839337795972824, "tokens_per_second": 1395856.3218527534, "eta_seconds": 21337.63806898594, "data_wait_seconds": 2.1129846572875977e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20045, "finemath": 0.08232}}
{"step": 15680, "total_steps": 72479, "tokens": 8220835840, "loss": 3.4793024361133575, "l_mix": 2.1390265226364136, "l_a": 2.261492371559143, "l_c": 2.200927644968033, "z_loss": 15.498558402061462, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2094762623310089, "tokens_per_second": 1461499.2760717017, "eta_seconds": 20375.674897384644, "data_wait_seconds": 1.5735626220703125e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20045, "finemath": 0.08233}}
{"step": 15690, "total_steps": 72479, "tokens": 8226078720, "loss": 3.5046589970588684, "l_mix": 2.1537370085716248, "l_a": 2.2820523977279663, "l_c": 2.2159228324890137, "z_loss": 15.29286277294159, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22656108438968658, "tokens_per_second": 1466979.7005591874, "eta_seconds": 20295.980387902262, "data_wait_seconds": 1.6361474990844727e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20045, "finemath": 0.08235}}
{"step": 15700, "total_steps": 72479, "tokens": 8231321600, "loss": 3.5382707715034485, "l_mix": 2.174209713935852, "l_a": 2.3076626658439636, "l_c": 2.2341895401477814, "z_loss": 15.052965760231018, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.23574696481227875, "tokens_per_second": 1345022.8110471508, "eta_seconds": 22132.374341535568, "data_wait_seconds": 3.337860107421875e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20764, "cosmopedia_v2": 0.20045, "finemath": 0.08236}, "gates": {"gate_a2b/L1_absmean": 0.26543531, "gate_a2b/L1_absmax": 0.63460487, "gate_a2b/L1_rms": 0.28872299, "gate_a2b/L3_absmean": 0.3197616, "gate_a2b/L3_absmax": 0.5003376, "gate_a2b/L3_rms": 0.32471371, "gate_a2b/L5_absmean": 0.50636488, "gate_a2b/L5_absmax": 0.67939925, "gate_a2b/L5_rms": 0.51285893, "gate_c2b/L1_absmean": 0.01397416, "gate_c2b/L1_absmax": 0.4726671, "gate_c2b/L1_rms": 0.05509265, "gate_c2b/L3_absmean": 0.20833318, "gate_c2b/L3_absmax": 0.43412063, "gate_c2b/L3_rms": 0.21285999, "gate_c2b/L5_absmean": 0.5347302, "gate_c2b/L5_absmax": 0.72748864, "gate_c2b/L5_rms": 0.54672289, "gate_b2a/L7_absmean": 0.07986282, "gate_b2a/L7_absmax": 0.25389075, "gate_b2a/L7_rms": 0.08308913, "gate_b2a/L11_absmean": 0.12613434, "gate_b2a/L11_absmax": 0.38950494, "gate_b2a/L11_rms": 0.12858874, "gate_b2a/L14_absmean": 0.30641001, "gate_b2a/L14_absmax": 0.4891209, "gate_b2a/L14_rms": 0.30757397, "gate_b2c/L3_absmean": 0.08281666, "gate_b2c/L3_absmax": 0.57754648, "gate_b2c/L3_rms": 0.1026249, "gate_b2c/L5_absmean": 0.11531164, "gate_b2c/L5_absmax": 0.69254148, "gate_b2c/L5_rms": 0.12182253, "gate_b2c/L6_absmean": 0.1711069, "gate_b2c/L6_absmax": 0.4753893, "gate_b2c/L6_rms": 0.17442636}, "streams": {"cos_a2b/round0": 0.364169, "cos_c2b/round0": 0.241193, "cos_b2a/round0": 0.136288, "cos_b2c/round0": 0.091039, "rms_a/round0": 0.909875, "rms_b/round0": 1.678203, "rms_c/round0": 1.287321, "cos_a2b/round1": 0.400537, "cos_c2b/round1": 0.271567, "cos_b2a/round1": 0.217887, "cos_b2c/round1": 0.138426, "rms_a/round1": 1.611241, "rms_b/round1": 3.288146, "rms_c/round1": 2.166394, "cos_a2b/round2": 0.491574, "cos_c2b/round2": 0.468121, "cos_b2a/round2": 0.039303, "cos_b2c/round2": 0.107917, "rms_a/round2": 3.45337, "rms_b/round2": 7.191217, "rms_c/round2": 3.473832}, "alpha_mean": 0.412760467419254, "alpha_hist": [0.06609, 0.15958, 0.16539, 0.13702, 0.11229, 0.10376, 0.10435, 0.09059, 0.05255, 0.00838], "alpha_by_type": {"special": {"share": 0.0011, "mean": 0.57675, "std": 0.11224}, "whitespace": {"share": 0.03415, "mean": 0.64526, "std": 0.19497}, "digit": {"share": 0.0121, "mean": 0.42315, "std": 0.17996}, "word_start": {"share": 0.57666, "mean": 0.43167, "std": 0.21869}, "word_cont": {"share": 0.26112, "mean": 0.24727, "std": 0.15352}, "punct": {"share": 0.11487, "mean": 0.62225, "std": 0.20155}}, "alpha_p05": 0.08977, "alpha_p50": 0.36207, "alpha_p95": 0.8047}
{"step": 15710, "total_steps": 72479, "tokens": 8236564480, "loss": 3.535286605358124, "l_mix": 2.172265201807022, "l_a": 2.303337335586548, "l_c": 2.234923094511032, "z_loss": 15.433006048202515, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1973029226064682, "tokens_per_second": 1432826.186567895, "eta_seconds": 20772.44661705494, "data_wait_seconds": 2.065300941467285e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20764, "cosmopedia_v2": 0.20045, "finemath": 0.08237}}
{"step": 15720, "total_steps": 72479, "tokens": 8241807360, "loss": 4.916371583938599, "l_mix": 3.0445548593997955, "l_a": 3.1407022178173065, "l_c": 3.0931263267993927, "z_loss": 16.680664777755737, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2324722409248352, "tokens_per_second": 1445985.7096512443, "eta_seconds": 20579.776406764984, "data_wait_seconds": 1.9043684005737305e-05, "mix": {"fineweb_hq": 0.50961, "dclm": 0.20763, "cosmopedia_v2": 0.20045, "finemath": 0.08232}}
{"step": 15730, "total_steps": 72479, "tokens": 8247050240, "loss": 4.605470180511475, "l_mix": 2.8519811630249023, "l_a": 2.946450859308243, "l_c": 2.8929007053375244, "z_loss": 16.8351047039032, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.257480263710022, "tokens_per_second": 1460606.8764378838, "eta_seconds": 20370.17639172077, "data_wait_seconds": 1.6570091247558594e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20763, "cosmopedia_v2": 0.20045, "finemath": 0.08233}}
{"step": 15740, "total_steps": 72479, "tokens": 8252293120, "loss": 4.737972378730774, "l_mix": 2.9312840402126312, "l_a": 3.035459518432617, "l_c": 2.9811686277389526, "z_loss": 16.996870517730713, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2118239849805832, "tokens_per_second": 1359398.3446622542, "eta_seconds": 21882.89911401272, "data_wait_seconds": 4.079937934875488e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20762, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 15750, "total_steps": 72479, "tokens": 8257536000, "loss": 4.6270283460617065, "l_mix": 2.8621262907981873, "l_a": 2.9630945920944214, "l_c": 2.9143899977207184, "z_loss": 16.56664800643921, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.202500119805336, "tokens_per_second": 1462176.316087393, "eta_seconds": 20341.140548348427, "data_wait_seconds": 1.4841556549072266e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20762, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 15760, "total_steps": 72479, "tokens": 8262778880, "loss": 4.630626738071442, "l_mix": 2.8638142943382263, "l_a": 2.9698186218738556, "l_c": 2.9140343964099884, "z_loss": 16.563350915908813, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20289137959480286, "tokens_per_second": 1428704.9686909667, "eta_seconds": 20814.018095874784, "data_wait_seconds": 2.753734588623047e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20761, "cosmopedia_v2": 0.20044, "finemath": 0.08236}}
{"step": 15770, "total_steps": 72479, "tokens": 8268021760, "loss": 4.719817757606506, "l_mix": 2.9214033782482147, "l_a": 3.017146021127701, "l_c": 2.9719789028167725, "z_loss": 16.76771068572998, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.29131972789764404, "tokens_per_second": 1403523.3460153316, "eta_seconds": 21183.721864271163, "data_wait_seconds": 1.6063451766967773e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08231}}
{"step": 15780, "total_steps": 72479, "tokens": 8273264640, "loss": 4.70490288734436, "l_mix": 2.9120262265205383, "l_a": 3.007838398218155, "l_c": 2.962985575199127, "z_loss": 16.294026136398315, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.26174691319465637, "tokens_per_second": 1460560.1169232759, "eta_seconds": 20352.88035566807, "data_wait_seconds": 1.677870750427246e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08232}}
{"step": 15790, "total_steps": 72479, "tokens": 8278507520, "loss": 3.559879630804062, "l_mix": 2.187615841627121, "l_a": 2.319023549556732, "l_c": 2.250112682580948, "z_loss": 15.228808641433716, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.211968794465065, "tokens_per_second": 1379440.0094546156, "eta_seconds": 21545.96229505539, "data_wait_seconds": 4.604458808898926e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 15800, "total_steps": 72479, "tokens": 8283750400, "loss": 3.5034244060516357, "l_mix": 2.153033882379532, "l_a": 2.2803792357444763, "l_c": 2.215815484523773, "z_loss": 15.32110047340393, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20060615241527557, "tokens_per_second": 1405173.639701825, "eta_seconds": 21147.649452281, "data_wait_seconds": 2.7894973754882812e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.26710415, "gate_a2b/L1_absmax": 0.63776606, "gate_a2b/L1_rms": 0.29042152, "gate_a2b/L3_absmean": 0.31881484, "gate_a2b/L3_absmax": 0.4988592, "gate_a2b/L3_rms": 0.3237727, "gate_a2b/L5_absmean": 0.5062561, "gate_a2b/L5_absmax": 0.67781365, "gate_a2b/L5_rms": 0.51272446, "gate_c2b/L1_absmean": 0.01455554, "gate_c2b/L1_absmax": 0.46977919, "gate_c2b/L1_rms": 0.05510041, "gate_c2b/L3_absmean": 0.20784186, "gate_c2b/L3_absmax": 0.44042021, "gate_c2b/L3_rms": 0.21238989, "gate_c2b/L5_absmean": 0.5336048, "gate_c2b/L5_absmax": 0.72452515, "gate_c2b/L5_rms": 0.54575616, "gate_b2a/L7_absmean": 0.07971326, "gate_b2a/L7_absmax": 0.25585482, "gate_b2a/L7_rms": 0.08293431, "gate_b2a/L11_absmean": 0.12569487, "gate_b2a/L11_absmax": 0.3901836, "gate_b2a/L11_rms": 0.12820803, "gate_b2a/L14_absmean": 0.30678731, "gate_b2a/L14_absmax": 0.4919073, "gate_b2a/L14_rms": 0.30798438, "gate_b2c/L3_absmean": 0.0821782, "gate_b2c/L3_absmax": 0.57866651, "gate_b2c/L3_rms": 0.10205404, "gate_b2c/L5_absmean": 0.1155064, "gate_b2c/L5_absmax": 0.68737811, "gate_b2c/L5_rms": 0.12196337, "gate_b2c/L6_absmean": 0.17024302, "gate_b2c/L6_absmax": 0.47133851, "gate_b2c/L6_rms": 0.17356139}, "streams": {"cos_a2b/round0": 0.3672, "cos_c2b/round0": 0.241505, "cos_b2a/round0": 0.134706, "cos_b2c/round0": 0.091462, "rms_a/round0": 0.902417, "rms_b/round0": 1.674963, "rms_c/round0": 1.278004, "cos_a2b/round1": 0.399464, "cos_c2b/round1": 0.260338, "cos_b2a/round1": 0.214583, "cos_b2c/round1": 0.135346, "rms_a/round1": 1.614264, "rms_b/round1": 3.28828, "rms_c/round1": 2.147621, "cos_a2b/round2": 0.486978, "cos_c2b/round2": 0.470708, "cos_b2a/round2": 0.037136, "cos_b2c/round2": 0.113171, "rms_a/round2": 3.413722, "rms_b/round2": 7.167862, "rms_c/round2": 3.436599}, "alpha_mean": 0.4406582751716144, "alpha_hist": [0.05286, 0.12529, 0.15353, 0.14429, 0.12555, 0.10721, 0.11252, 0.10789, 0.06439, 0.00647], "alpha_by_type": {"special": {"share": 0.00105, "mean": 0.58255, "std": 0.1029}, "whitespace": {"share": 0.03793, "mean": 0.64477, "std": 0.1769}, "digit": {"share": 0.01218, "mean": 0.41192, "std": 0.15966}, "word_start": {"share": 0.54741, "mean": 0.46992, "std": 0.21627}, "word_cont": {"share": 0.27216, "mean": 0.26715, "std": 0.16033}, "punct": {"share": 0.12927, "mean": 0.62368, "std": 0.19147}}, "alpha_p05": 0.09009, "alpha_p50": 0.43015, "alpha_p95": 0.83114}
{"step": 15810, "total_steps": 72479, "tokens": 8288993280, "loss": 3.5269818007946014, "l_mix": 2.1675215363502502, "l_a": 2.297387510538101, "l_c": 2.229031026363373, "z_loss": 15.34611690044403, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.24710142612457275, "tokens_per_second": 1436025.0773117293, "eta_seconds": 20689.664227604866, "data_wait_seconds": 3.1620264053344727e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 15820, "total_steps": 72479, "tokens": 8294236160, "loss": 3.5218102633953094, "l_mix": 2.1641813218593597, "l_a": 2.295486271381378, "l_c": 2.224781095981598, "z_loss": 15.486614108085632, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.24452874064445496, "tokens_per_second": 1470517.0504963126, "eta_seconds": 20200.808812093735, "data_wait_seconds": 3.0279159545898438e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08236}}
{"step": 15830, "total_steps": 72479, "tokens": 8299479040, "loss": 4.768497109413147, "l_mix": 2.952790319919586, "l_a": 3.048998475074768, "l_c": 2.997866988182068, "z_loss": 16.47035264968872, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17339006066322327, "tokens_per_second": 1391399.1281745979, "eta_seconds": 21345.70182673931, "data_wait_seconds": 1.659989356994629e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08231}}
{"step": 15840, "total_steps": 72479, "tokens": 8304721920, "loss": 4.763613939285278, "l_mix": 2.947246730327606, "l_a": 3.053570121526718, "l_c": 2.995355486869812, "z_loss": 16.894556283950806, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22891657054424286, "tokens_per_second": 1473594.494907542, "eta_seconds": 20151.505814266202, "data_wait_seconds": 1.564621925354004e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08232}}
{"step": 15850, "total_steps": 72479, "tokens": 8309964800, "loss": 4.685560524463654, "l_mix": 2.9003285467624664, "l_a": 2.997232496738434, "l_c": 2.9480704963207245, "z_loss": 16.41133213043213, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1868629902601242, "tokens_per_second": 1470739.322306004, "eta_seconds": 20187.061501455308, "data_wait_seconds": 1.6123056411743164e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20763, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 15860, "total_steps": 72479, "tokens": 8315207680, "loss": 4.694871187210083, "l_mix": 2.9049842059612274, "l_a": 3.0037420094013214, "l_c": 2.956973522901535, "z_loss": 16.723185062408447, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20367756485939026, "tokens_per_second": 1418525.995844066, "eta_seconds": 20926.414009308817, "data_wait_seconds": 1.5228986740112305e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20763, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 15870, "total_steps": 72479, "tokens": 8320450560, "loss": 4.692157328128815, "l_mix": 2.903372049331665, "l_a": 3.0041555166244507, "l_c": 2.9528832137584686, "z_loss": 16.736729621887207, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21248117089271545, "tokens_per_second": 1390149.8371929866, "eta_seconds": 21349.79884753227, "data_wait_seconds": 1.6063451766967773e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20762, "cosmopedia_v2": 0.20044, "finemath": 0.08236}}
{"step": 15880, "total_steps": 72479, "tokens": 8325693440, "loss": 4.737655997276306, "l_mix": 2.9320459961891174, "l_a": 3.030540019273758, "l_c": 2.982553780078888, "z_loss": 16.81852078437805, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2257302701473236, "tokens_per_second": 1434873.862317903, "eta_seconds": 20680.686498856543, "data_wait_seconds": 2.282857894897461e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20762, "cosmopedia_v2": 0.20044, "finemath": 0.08237}}
{"step": 15890, "total_steps": 72479, "tokens": 8330936320, "loss": 4.931382954120636, "l_mix": 3.0546504855155945, "l_a": 3.145734876394272, "l_c": 3.104568302631378, "z_loss": 16.414085149765015, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1787448525428772, "tokens_per_second": 1425377.238432431, "eta_seconds": 20814.794029283523, "data_wait_seconds": 1.6301870346069336e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20768, "cosmopedia_v2": 0.20044, "finemath": 0.08232}}
{"step": 15900, "total_steps": 72479, "tokens": 8336179200, "loss": 4.91959422826767, "l_mix": 3.046681672334671, "l_a": 3.1387592554092407, "l_c": 3.0986196398735046, "z_loss": 16.98811411857605, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18388040363788605, "tokens_per_second": 1426781.7636458164, "eta_seconds": 20790.629308438303, "data_wait_seconds": 1.4960765838623047e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.26815835, "gate_a2b/L1_absmax": 0.64085716, "gate_a2b/L1_rms": 0.29099551, "gate_a2b/L3_absmean": 0.31952342, "gate_a2b/L3_absmax": 0.50456667, "gate_a2b/L3_rms": 0.32453468, "gate_a2b/L5_absmean": 0.50720292, "gate_a2b/L5_absmax": 0.67891467, "gate_a2b/L5_rms": 0.5136072, "gate_c2b/L1_absmean": 0.0150523, "gate_c2b/L1_absmax": 0.47145739, "gate_c2b/L1_rms": 0.05490502, "gate_c2b/L3_absmean": 0.20789929, "gate_c2b/L3_absmax": 0.43655354, "gate_c2b/L3_rms": 0.21236974, "gate_c2b/L5_absmean": 0.53317595, "gate_c2b/L5_absmax": 0.72871971, "gate_c2b/L5_rms": 0.54526258, "gate_b2a/L7_absmean": 0.08002594, "gate_b2a/L7_absmax": 0.255698, "gate_b2a/L7_rms": 0.08318675, "gate_b2a/L11_absmean": 0.12604469, "gate_b2a/L11_absmax": 0.38969022, "gate_b2a/L11_rms": 0.12856159, "gate_b2a/L14_absmean": 0.30702579, "gate_b2a/L14_absmax": 0.50253528, "gate_b2a/L14_rms": 0.30822638, "gate_b2c/L3_absmean": 0.08325587, "gate_b2c/L3_absmax": 0.57305527, "gate_b2c/L3_rms": 0.10260262, "gate_b2c/L5_absmean": 0.11585604, "gate_b2c/L5_absmax": 0.67921048, "gate_b2c/L5_rms": 0.12227888, "gate_b2c/L6_absmean": 0.17182136, "gate_b2c/L6_absmax": 0.47464013, "gate_b2c/L6_rms": 0.17518249}, "streams": {"cos_a2b/round0": 0.386665, "cos_c2b/round0": 0.280985, "cos_b2a/round0": 0.148997, "cos_b2c/round0": 0.141193, "rms_a/round0": 0.870345, "rms_b/round0": 1.663102, "rms_c/round0": 1.233467, "cos_a2b/round1": 0.40118, "cos_c2b/round1": 0.265454, "cos_b2a/round1": 0.228714, "cos_b2c/round1": 0.147717, "rms_a/round1": 1.547684, "rms_b/round1": 3.158659, "rms_c/round1": 2.04, "cos_a2b/round2": 0.532966, "cos_c2b/round2": 0.481905, "cos_b2a/round2": 0.094778, "cos_b2c/round2": 0.15315, "rms_a/round2": 3.385158, "rms_b/round2": 6.98312, "rms_c/round2": 3.242723}, "alpha_mean": 0.46114565604123214, "alpha_hist": [0.01324, 0.05779, 0.13406, 0.17961, 0.18369, 0.19293, 0.14107, 0.08182, 0.01535, 0.00044], "alpha_by_type": {"special": {"share": 0.00046, "mean": 0.5924, "std": 0.12036}, "whitespace": {"share": 0.07428, "mean": 0.54484, "std": 0.12262}, "digit": {"share": 0.05914, "mean": 0.47543, "std": 0.15865}, "word_start": {"share": 0.5192, "mean": 0.48219, "std": 0.15933}, "word_cont": {"share": 0.23799, "mean": 0.32304, "std": 0.14367}, "punct": {"share": 0.10893, "mean": 0.59722, "std": 0.16486}}, "alpha_p05": 0.19559, "alpha_p50": 0.47001, "alpha_p95": 0.74167}
{"step": 15910, "total_steps": 72479, "tokens": 8341422080, "loss": 3.539981782436371, "l_mix": 2.176234871149063, "l_a": 2.2975696325302124, "l_c": 2.2432693243026733, "z_loss": 14.952418446540833, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2794300615787506, "tokens_per_second": 1442021.609997806, "eta_seconds": 20567.27005085945, "data_wait_seconds": 2.4050474166870117e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 15920, "total_steps": 72479, "tokens": 8346664960, "loss": 3.551719307899475, "l_mix": 2.1831334233283997, "l_a": 2.312508225440979, "l_c": 2.2443685233592987, "z_loss": 15.22797429561615, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2287411093711853, "tokens_per_second": 1389372.6158450616, "eta_seconds": 21342.874225258827, "data_wait_seconds": 2.4259090423583984e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 15930, "total_steps": 72479, "tokens": 8351907840, "loss": 3.558852106332779, "l_mix": 2.1870435774326324, "l_a": 2.317846953868866, "l_c": 2.249663919210434, "z_loss": 15.552241921424866, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.23129113018512726, "tokens_per_second": 1358656.1269380415, "eta_seconds": 21821.534915399552, "data_wait_seconds": 1.6927719116210938e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08236}}
{"step": 15940, "total_steps": 72479, "tokens": 8357150720, "loss": 4.649769425392151, "l_mix": 2.8776543140411377, "l_a": 2.9762957394123077, "l_c": 2.924960196018219, "z_loss": 17.383777618408203, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.26121392846107483, "tokens_per_second": 1435272.1652904344, "eta_seconds": 20653.030100393295, "data_wait_seconds": 1.683831214904785e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08231}}
{"step": 15950, "total_steps": 72479, "tokens": 8362393600, "loss": 4.6896151304244995, "l_mix": 2.901901751756668, "l_a": 3.0031762421131134, "l_c": 2.9501935839653015, "z_loss": 17.02388310432434, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18428006768226624, "tokens_per_second": 1443433.3734886693, "eta_seconds": 20532.62512586117, "data_wait_seconds": 1.481175422668457e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08232}}
{"step": 15960, "total_steps": 72479, "tokens": 8367636480, "loss": 4.6949721574783325, "l_mix": 2.904638320207596, "l_a": 3.00657257437706, "l_c": 2.9556434750556946, "z_loss": 16.689517736434937, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1984138786792755, "tokens_per_second": 1440055.5267174952, "eta_seconds": 20577.146451807024, "data_wait_seconds": 1.609325408935547e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 15970, "total_steps": 72479, "tokens": 8372879360, "loss": 4.566255569458008, "l_mix": 2.8256540298461914, "l_a": 2.9223847687244415, "l_c": 2.8740064799785614, "z_loss": 16.84042000770569, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21769647300243378, "tokens_per_second": 1469171.6762410128, "eta_seconds": 20165.778493499754, "data_wait_seconds": 1.6063451766967773e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 15980, "total_steps": 72479, "tokens": 8378122240, "loss": 4.666242241859436, "l_mix": 2.886474907398224, "l_a": 2.9918715059757233, "l_c": 2.935102015733719, "z_loss": 16.75079321861267, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20474272966384888, "tokens_per_second": 1358976.7004716014, "eta_seconds": 21797.097552680967, "data_wait_seconds": 1.615285873413086e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20763, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 15990, "total_steps": 72479, "tokens": 8383365120, "loss": 4.719392895698547, "l_mix": 2.922065258026123, "l_a": 3.0177752673625946, "l_c": 2.967672735452652, "z_loss": 16.932838439941406, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.28430554270744324, "tokens_per_second": 1411259.93538436, "eta_seconds": 20985.86099514961, "data_wait_seconds": 3.635883331298828e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20763, "cosmopedia_v2": 0.20044, "finemath": 0.08236}}
{"step": 16000, "total_steps": 72479, "tokens": 8388608000, "loss": 4.864017069339752, "l_mix": 3.0133614540100098, "l_a": 3.101153641939163, "l_c": 3.062208831310272, "z_loss": 16.467427968978882, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20553629100322723, "tokens_per_second": 1408773.0175169688, "eta_seconds": 21019.185904192924, "data_wait_seconds": 2.0563602447509766e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20769, "cosmopedia_v2": 0.20044, "finemath": 0.08231}, "gates": {"gate_a2b/L1_absmean": 0.26925859, "gate_a2b/L1_absmax": 0.63976091, "gate_a2b/L1_rms": 0.29173699, "gate_a2b/L3_absmean": 0.3197026, "gate_a2b/L3_absmax": 0.50187188, "gate_a2b/L3_rms": 0.32463849, "gate_a2b/L5_absmean": 0.50737727, "gate_a2b/L5_absmax": 0.68007761, "gate_a2b/L5_rms": 0.51378918, "gate_c2b/L1_absmean": 0.01433547, "gate_c2b/L1_absmax": 0.46941561, "gate_c2b/L1_rms": 0.05468641, "gate_c2b/L3_absmean": 0.20801477, "gate_c2b/L3_absmax": 0.43243176, "gate_c2b/L3_rms": 0.21259175, "gate_c2b/L5_absmean": 0.53289241, "gate_c2b/L5_absmax": 0.72357637, "gate_c2b/L5_rms": 0.54500818, "gate_b2a/L7_absmean": 0.07978222, "gate_b2a/L7_absmax": 0.25700328, "gate_b2a/L7_rms": 0.08302996, "gate_b2a/L11_absmean": 0.1261519, "gate_b2a/L11_absmax": 0.39251396, "gate_b2a/L11_rms": 0.12865229, "gate_b2a/L14_absmean": 0.3063975, "gate_b2a/L14_absmax": 0.50099593, "gate_b2a/L14_rms": 0.30761856, "gate_b2c/L3_absmean": 0.08308835, "gate_b2c/L3_absmax": 0.5772168, "gate_b2c/L3_rms": 0.10283094, "gate_b2c/L5_absmean": 0.11597493, "gate_b2c/L5_absmax": 0.69010127, "gate_b2c/L5_rms": 0.12246338, "gate_b2c/L6_absmean": 0.17162716, "gate_b2c/L6_absmax": 0.46985933, "gate_b2c/L6_rms": 0.1749967}, "streams": {"cos_a2b/round0": 0.379534, "cos_c2b/round0": 0.271768, "cos_b2a/round0": 0.147309, "cos_b2c/round0": 0.137923, "rms_a/round0": 0.878282, "rms_b/round0": 1.681019, "rms_c/round0": 1.248225, "cos_a2b/round1": 0.400134, "cos_c2b/round1": 0.273585, "cos_b2a/round1": 0.22927, "cos_b2c/round1": 0.151911, "rms_a/round1": 1.565705, "rms_b/round1": 3.147581, "rms_c/round1": 2.086707, "cos_a2b/round2": 0.548015, "cos_c2b/round2": 0.470918, "cos_b2a/round2": 0.090088, "cos_b2c/round2": 0.157788, "rms_a/round2": 3.43622, "rms_b/round2": 6.849808, "rms_c/round2": 3.303186}, "alpha_mean": 0.47661634013827836, "alpha_hist": [0.01039, 0.04282, 0.10846, 0.15982, 0.20689, 0.22849, 0.155, 0.07651, 0.0112, 0.00043], "alpha_by_type": {"special": {"share": 0.00066, "mean": 0.53238, "std": 0.15107}, "whitespace": {"share": 0.06546, "mean": 0.54126, "std": 0.10962}, "digit": {"share": 0.03603, "mean": 0.47228, "std": 0.12864}, "word_start": {"share": 0.55937, "mean": 0.50499, "std": 0.14564}, "word_cont": {"share": 0.23997, "mean": 0.34426, "std": 0.14563}, "punct": {"share": 0.09851, "mean": 0.59619, "std": 0.15649}}, "alpha_p05": 0.20025, "alpha_p50": 0.48212, "alpha_p95": 0.75092}
{"step": 16010, "total_steps": 72479, "tokens": 8393850880, "loss": 4.96052211523056, "l_mix": 3.072810471057892, "l_a": 3.1650012135505676, "l_c": 3.121879756450653, "z_loss": 16.472053289413452, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15797337889671326, "tokens_per_second": 1396023.5369694738, "eta_seconds": 21207.392488718033, "data_wait_seconds": 1.8298625946044922e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.20044, "finemath": 0.08232}}
{"step": 16020, "total_steps": 72479, "tokens": 8399093760, "loss": 4.970148384571075, "l_mix": 3.0790427029132843, "l_a": 3.1677998900413513, "l_c": 3.130355089902878, "z_loss": 16.59079623222351, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20709633827209473, "tokens_per_second": 1404554.8968863857, "eta_seconds": 21074.844605660437, "data_wait_seconds": 1.823902130126953e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20768, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 16030, "total_steps": 72479, "tokens": 8404336640, "loss": 4.877957701683044, "l_mix": 3.020999163389206, "l_a": 3.113110214471817, "l_c": 3.071145921945572, "z_loss": 16.816723823547363, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1690281629562378, "tokens_per_second": 1359827.0617267767, "eta_seconds": 21764.189098000526, "data_wait_seconds": 2.110004425048828e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 16040, "total_steps": 72479, "tokens": 8409579520, "loss": 4.967248916625977, "l_mix": 3.0766710340976715, "l_a": 3.1683496832847595, "l_c": 3.127849280834198, "z_loss": 17.18117046356201, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17158429324626923, "tokens_per_second": 1411069.130267639, "eta_seconds": 20970.12102191448, "data_wait_seconds": 3.507733345031738e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08236}}
{"step": 16050, "total_steps": 72479, "tokens": 8414822400, "loss": 4.679058372974396, "l_mix": 2.8957669138908386, "l_a": 2.992218106985092, "l_c": 2.9465163350105286, "z_loss": 16.712192058563232, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16996356844902039, "tokens_per_second": 1344291.1088474877, "eta_seconds": 22007.917301011086, "data_wait_seconds": 1.665949821472168e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08231}}
{"step": 16060, "total_steps": 72479, "tokens": 8420065280, "loss": 4.68205726146698, "l_mix": 2.897997170686722, "l_a": 2.996164470911026, "l_c": 2.9452450573444366, "z_loss": 16.372009992599487, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18664507567882538, "tokens_per_second": 1475719.4412216223, "eta_seconds": 20044.328105831148, "data_wait_seconds": 1.615285873413086e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08232}}
{"step": 16070, "total_steps": 72479, "tokens": 8425308160, "loss": 4.726214349269867, "l_mix": 2.9244912564754486, "l_a": 3.0232962369918823, "l_c": 2.977022737264633, "z_loss": 16.272023916244507, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21808883547782898, "tokens_per_second": 1323100.2163820576, "eta_seconds": 22352.472946357728, "data_wait_seconds": 2.238154411315918e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 16080, "total_steps": 72479, "tokens": 8430551040, "loss": 4.634443700313568, "l_mix": 2.8681086897850037, "l_a": 2.967133969068527, "l_c": 2.9150599241256714, "z_loss": 16.768131494522095, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20519815385341644, "tokens_per_second": 1406951.7625080303, "eta_seconds": 21016.5832972765, "data_wait_seconds": 1.6808509826660156e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 16090, "total_steps": 72479, "tokens": 8435793920, "loss": 4.646410882472992, "l_mix": 2.875210464000702, "l_a": 2.974710375070572, "l_c": 2.923793286085129, "z_loss": 16.49246072769165, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2174818515777588, "tokens_per_second": 1430146.9416881474, "eta_seconds": 20672.0548568964, "data_wait_seconds": 1.659989356994629e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 16100, "total_steps": 72479, "tokens": 8441036800, "loss": 4.7616031765937805, "l_mix": 2.94756081700325, "l_a": 3.04581356048584, "l_c": 2.995360642671585, "z_loss": 16.900386333465576, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2876216769218445, "tokens_per_second": 1416899.71416944, "eta_seconds": 20861.626871967317, "data_wait_seconds": 3.7044286727905273e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20764, "cosmopedia_v2": 0.20043, "finemath": 0.08236}, "gates": {"gate_a2b/L1_absmean": 0.27024272, "gate_a2b/L1_absmax": 0.64567691, "gate_a2b/L1_rms": 0.29252616, "gate_a2b/L3_absmean": 0.32004732, "gate_a2b/L3_absmax": 0.50217617, "gate_a2b/L3_rms": 0.32498649, "gate_a2b/L5_absmean": 0.5078277, "gate_a2b/L5_absmax": 0.67810208, "gate_a2b/L5_rms": 0.51421285, "gate_c2b/L1_absmean": 0.0146769, "gate_c2b/L1_absmax": 0.4676148, "gate_c2b/L1_rms": 0.05488786, "gate_c2b/L3_absmean": 0.20787466, "gate_c2b/L3_absmax": 0.43375731, "gate_c2b/L3_rms": 0.21229249, "gate_c2b/L5_absmean": 0.53359061, "gate_c2b/L5_absmax": 0.72256202, "gate_c2b/L5_rms": 0.54568571, "gate_b2a/L7_absmean": 0.08005114, "gate_b2a/L7_absmax": 0.25675353, "gate_b2a/L7_rms": 0.08327565, "gate_b2a/L11_absmean": 0.12604704, "gate_b2a/L11_absmax": 0.39389104, "gate_b2a/L11_rms": 0.12855762, "gate_b2a/L14_absmean": 0.30707759, "gate_b2a/L14_absmax": 0.49936116, "gate_b2a/L14_rms": 0.30826825, "gate_b2c/L3_absmean": 0.08378446, "gate_b2c/L3_absmax": 0.57706547, "gate_b2c/L3_rms": 0.10345196, "gate_b2c/L5_absmean": 0.11649759, "gate_b2c/L5_absmax": 0.6990658, "gate_b2c/L5_rms": 0.12311659, "gate_b2c/L6_absmean": 0.17251548, "gate_b2c/L6_absmax": 0.46054575, "gate_b2c/L6_rms": 0.17579341}, "streams": {"cos_a2b/round0": 0.379811, "cos_c2b/round0": 0.275876, "cos_b2a/round0": 0.146611, "cos_b2c/round0": 0.139397, "rms_a/round0": 0.874144, "rms_b/round0": 1.688999, "rms_c/round0": 1.26151, "cos_a2b/round1": 0.403739, "cos_c2b/round1": 0.271424, "cos_b2a/round1": 0.224525, "cos_b2c/round1": 0.136237, "rms_a/round1": 1.558387, "rms_b/round1": 3.17878, "rms_c/round1": 2.081661, "cos_a2b/round2": 0.532848, "cos_c2b/round2": 0.478305, "cos_b2a/round2": 0.078597, "cos_b2c/round2": 0.138486, "rms_a/round2": 3.400902, "rms_b/round2": 6.999864, "rms_c/round2": 3.327659}, "alpha_mean": 0.4489198000187926, "alpha_hist": [0.01544, 0.061, 0.14178, 0.19019, 0.18991, 0.18645, 0.13657, 0.06764, 0.01085, 0.00017], "alpha_by_type": {"special": {"share": 0.00131, "mean": 0.55661, "std": 0.09012}, "whitespace": {"share": 0.03047, "mean": 0.52439, "std": 0.11923}, "digit": {"share": 0.03711, "mean": 0.44458, "std": 0.11373}, "word_start": {"share": 0.55945, "mean": 0.47427, "std": 0.15984}, "word_cont": {"share": 0.2691, "mean": 0.33313, "std": 0.14858}, "punct": {"share": 0.10255, "mean": 0.59221, "std": 0.15423}}, "alpha_p05": 0.17724, "alpha_p50": 0.45024, "alpha_p95": 0.75743}
{"step": 16110, "total_steps": 72479, "tokens": 8446279680, "loss": 3.3414966464042664, "l_mix": 2.0642860382795334, "l_a": 2.1392267048358917, "l_c": 2.1119506508111954, "z_loss": 18.57429814338684, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.4916796386241913, "tokens_per_second": 1469528.9514871456, "eta_seconds": 20110.92754728794, "data_wait_seconds": 1.8537044525146484e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20764, "cosmopedia_v2": 0.20043, "finemath": 0.08237}}
{"step": 16120, "total_steps": 72479, "tokens": 8451522560, "loss": 4.860545039176941, "l_mix": 3.010924458503723, "l_a": 3.1015798449516296, "l_c": 3.058448314666748, "z_loss": 16.12094807624817, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16144365072250366, "tokens_per_second": 1454998.1606291719, "eta_seconds": 20308.168210482596, "data_wait_seconds": 1.5735626220703125e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20769, "cosmopedia_v2": 0.20043, "finemath": 0.08232}}
{"step": 16130, "total_steps": 72479, "tokens": 8456765440, "loss": 4.93127578496933, "l_mix": 3.054747700691223, "l_a": 3.1470700800418854, "l_c": 3.1025761663913727, "z_loss": 16.341323733329773, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2002922147512436, "tokens_per_second": 1352718.976713057, "eta_seconds": 21839.79453277588, "data_wait_seconds": 2.2739171981811523e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20769, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 16140, "total_steps": 72479, "tokens": 8462008320, "loss": 4.8050801157951355, "l_mix": 2.976645439863205, "l_a": 3.0665546655654907, "l_c": 3.0226758420467377, "z_loss": 16.654239654541016, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17027364671230316, "tokens_per_second": 1445400.6218719468, "eta_seconds": 20435.76098213196, "data_wait_seconds": 2.4586915969848633e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20768, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 16150, "total_steps": 72479, "tokens": 8467251200, "loss": 4.906841993331909, "l_mix": 3.0400529503822327, "l_a": 3.127699851989746, "l_c": 3.0892776250839233, "z_loss": 16.95689558982849, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19175980985164642, "tokens_per_second": 1333536.4780236846, "eta_seconds": 22146.08991856575, "data_wait_seconds": 2.3990869522094727e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20768, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 16160, "total_steps": 72479, "tokens": 8472494080, "loss": 4.533401787281036, "l_mix": 2.80882166326046, "l_a": 2.8886477947235107, "l_c": 2.8546943813562393, "z_loss": 15.772720456123352, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.23368997871875763, "tokens_per_second": 1427498.6402242272, "eta_seconds": 20684.696321225165, "data_wait_seconds": 1.671910285949707e-05, "mix": {"fineweb_hq": 0.50953, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08236}}
{"step": 16170, "total_steps": 72479, "tokens": 8477736960, "loss": 4.484404683113098, "l_mix": 2.7738335728645325, "l_a": 2.87201851606369, "l_c": 2.824209213256836, "z_loss": 17.02667212486267, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15434563159942627, "tokens_per_second": 1383843.5212057775, "eta_seconds": 21333.432963776588, "data_wait_seconds": 2.0444393157958984e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08231}}
{"step": 16180, "total_steps": 72479, "tokens": 8482979840, "loss": 4.7916606068611145, "l_mix": 2.9666930735111237, "l_a": 3.0597929060459137, "l_c": 3.017908126115799, "z_loss": 16.571532726287842, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17160630226135254, "tokens_per_second": 1455692.9906125013, "eta_seconds": 20276.864903759957, "data_wait_seconds": 2.0205974578857422e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08232}}
{"step": 16190, "total_steps": 72479, "tokens": 8488222720, "loss": 4.6394882798194885, "l_mix": 2.871007889509201, "l_a": 2.9733498096466064, "l_c": 2.9159307777881622, "z_loss": 16.96081304550171, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19583715498447418, "tokens_per_second": 1408116.1158766774, "eta_seconds": 20958.24832856655, "data_wait_seconds": 2.008676528930664e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 16200, "total_steps": 72479, "tokens": 8493465600, "loss": 4.723243474960327, "l_mix": 2.9225718677043915, "l_a": 3.0200460255146027, "l_c": 2.9766102731227875, "z_loss": 16.746108531951904, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.28548896312713623, "tokens_per_second": 1405777.2008951362, "eta_seconds": 20989.388882684707, "data_wait_seconds": 2.2798776626586914e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.2711173, "gate_a2b/L1_absmax": 0.64769924, "gate_a2b/L1_rms": 0.29338324, "gate_a2b/L3_absmean": 0.32071891, "gate_a2b/L3_absmax": 0.50110471, "gate_a2b/L3_rms": 0.32557845, "gate_a2b/L5_absmean": 0.50770122, "gate_a2b/L5_absmax": 0.67684966, "gate_a2b/L5_rms": 0.51410651, "gate_c2b/L1_absmean": 0.01464786, "gate_c2b/L1_absmax": 0.46658412, "gate_c2b/L1_rms": 0.0547847, "gate_c2b/L3_absmean": 0.20885754, "gate_c2b/L3_absmax": 0.43739265, "gate_c2b/L3_rms": 0.21339558, "gate_c2b/L5_absmean": 0.53387469, "gate_c2b/L5_absmax": 0.72675133, "gate_c2b/L5_rms": 0.54598761, "gate_b2a/L7_absmean": 0.08044929, "gate_b2a/L7_absmax": 0.25677907, "gate_b2a/L7_rms": 0.08365826, "gate_b2a/L11_absmean": 0.12659736, "gate_b2a/L11_absmax": 0.39436105, "gate_b2a/L11_rms": 0.12908904, "gate_b2a/L14_absmean": 0.30743927, "gate_b2a/L14_absmax": 0.50185937, "gate_b2a/L14_rms": 0.30862466, "gate_b2c/L3_absmean": 0.0840718, "gate_b2c/L3_absmax": 0.5772019, "gate_b2c/L3_rms": 0.10352717, "gate_b2c/L5_absmean": 0.1164779, "gate_b2c/L5_absmax": 0.70688063, "gate_b2c/L5_rms": 0.12319402, "gate_b2c/L6_absmean": 0.17239401, "gate_b2c/L6_absmax": 0.47048163, "gate_b2c/L6_rms": 0.17575146}, "streams": {"cos_a2b/round0": 0.384803, "cos_c2b/round0": 0.282162, "cos_b2a/round0": 0.146312, "cos_b2c/round0": 0.138032, "rms_a/round0": 0.866312, "rms_b/round0": 1.667007, "rms_c/round0": 1.263765, "cos_a2b/round1": 0.410011, "cos_c2b/round1": 0.266324, "cos_b2a/round1": 0.225047, "cos_b2c/round1": 0.134906, "rms_a/round1": 1.565737, "rms_b/round1": 3.202892, "rms_c/round1": 2.06913, "cos_a2b/round2": 0.528849, "cos_c2b/round2": 0.482709, "cos_b2a/round2": 0.077068, "cos_b2c/round2": 0.120718, "rms_a/round2": 3.417941, "rms_b/round2": 7.047083, "rms_c/round2": 3.35136}, "alpha_mean": 0.4417157830798928, "alpha_hist": [0.01445, 0.07007, 0.15955, 0.19551, 0.17931, 0.16533, 0.12843, 0.07291, 0.01396, 0.00049], "alpha_by_type": {"special": {"share": 0.00117, "mean": 0.57703, "std": 0.12856}, "whitespace": {"share": 0.03386, "mean": 0.55557, "std": 0.13554}, "digit": {"share": 0.0401, "mean": 0.49973, "std": 0.12795}, "word_start": {"share": 0.57068, "mean": 0.46049, "std": 0.16729}, "word_cont": {"share": 0.25687, "mean": 0.32013, "std": 0.14057}, "punct": {"share": 0.09732, "mean": 0.58737, "std": 0.16418}}, "alpha_p05": 0.17639, "alpha_p50": 0.4383, "alpha_p95": 0.73867}
{"step": 16210, "total_steps": 72479, "tokens": 8498708480, "loss": 4.847602188587189, "l_mix": 3.0001161694526672, "l_a": 3.106375128030777, "l_c": 3.0462947487831116, "z_loss": 16.85054087638855, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.3167869448661804, "tokens_per_second": 1401699.011778864, "eta_seconds": 21046.71632361412, "data_wait_seconds": 2.79843807220459e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08236}}
{"step": 16220, "total_steps": 72479, "tokens": 8503951360, "loss": 3.170409232378006, "l_mix": 1.9587375223636627, "l_a": 2.0352771133184433, "l_c": 1.9973864257335663, "z_loss": 18.725555181503296, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.301288366317749, "tokens_per_second": 1435597.8647303774, "eta_seconds": 20546.086976480485, "data_wait_seconds": 2.7507543563842773e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20764, "cosmopedia_v2": 0.20043, "finemath": 0.08237}}
{"step": 16230, "total_steps": 72479, "tokens": 8509194240, "loss": 3.262485772371292, "l_mix": 2.016329988837242, "l_a": 2.0931702107191086, "l_c": 2.0546125322580338, "z_loss": 18.20901131629944, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.28259629011154175, "tokens_per_second": 1453886.600128422, "eta_seconds": 20284.02745399475, "data_wait_seconds": 1.7017126083374023e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20764, "cosmopedia_v2": 0.20043, "finemath": 0.08238}}
{"step": 16240, "total_steps": 72479, "tokens": 8514437120, "loss": 4.804346978664398, "l_mix": 2.974763184785843, "l_a": 3.0688607692718506, "l_c": 3.0242320597171783, "z_loss": 16.557698726654053, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16974186897277832, "tokens_per_second": 1432087.6292590317, "eta_seconds": 20589.126132774352, "data_wait_seconds": 1.9550323486328125e-05, "mix": {"fineweb_hq": 0.50961, "dclm": 0.20764, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 16250, "total_steps": 72479, "tokens": 8519680000, "loss": 4.631498634815216, "l_mix": 2.8659618496894836, "l_a": 2.9658804535865784, "l_c": 2.913676470518112, "z_loss": 16.696186780929565, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18177998065948486, "tokens_per_second": 1379673.5983721656, "eta_seconds": 21367.51039288044, "data_wait_seconds": 1.800060272216797e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20763, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 16260, "total_steps": 72479, "tokens": 8524922880, "loss": 4.64948695898056, "l_mix": 2.8767296075820923, "l_a": 2.976982533931732, "l_c": 2.9264053106307983, "z_loss": 17.409857749938965, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2120068520307541, "tokens_per_second": 1464809.1845109158, "eta_seconds": 20122.038681674003, "data_wait_seconds": 2.753734588623047e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20763, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 16270, "total_steps": 72479, "tokens": 8530165760, "loss": 4.539328038692474, "l_mix": 2.8099010586738586, "l_a": 2.905034989118576, "l_c": 2.8541654646396637, "z_loss": 16.66741132736206, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20381218194961548, "tokens_per_second": 1452549.1145337066, "eta_seconds": 20288.266948866843, "data_wait_seconds": 1.621246337890625e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20762, "cosmopedia_v2": 0.20043, "finemath": 0.08236}}
{"step": 16280, "total_steps": 72479, "tokens": 8535408640, "loss": 4.603402078151703, "l_mix": 2.8483184576034546, "l_a": 2.9482805728912354, "l_c": 2.896330714225769, "z_loss": 17.001381874084473, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1559186428785324, "tokens_per_second": 1439179.5072115613, "eta_seconds": 20473.096764063834, "data_wait_seconds": 1.9669532775878906e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20768, "cosmopedia_v2": 0.20043, "finemath": 0.08231}}
{"step": 16290, "total_steps": 72479, "tokens": 8540651520, "loss": 4.705496251583099, "l_mix": 2.9111343324184418, "l_a": 3.0140421092510223, "l_c": 2.9615143835544586, "z_loss": 16.94804310798645, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17317545413970947, "tokens_per_second": 1376263.1449589685, "eta_seconds": 21405.222206163406, "data_wait_seconds": 1.8358230590820312e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08232}}
{"step": 16300, "total_steps": 72479, "tokens": 8545894400, "loss": 4.64093291759491, "l_mix": 2.871605783700943, "l_a": 2.9733879268169403, "l_c": 2.9187150597572327, "z_loss": 16.962818384170532, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1817484349012375, "tokens_per_second": 1412697.7478418364, "eta_seconds": 20849.45318062305, "data_wait_seconds": 2.059340476989746e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.27165756, "gate_a2b/L1_absmax": 0.6454035, "gate_a2b/L1_rms": 0.29379669, "gate_a2b/L3_absmean": 0.3209455, "gate_a2b/L3_absmax": 0.49932152, "gate_a2b/L3_rms": 0.32578197, "gate_a2b/L5_absmean": 0.50817835, "gate_a2b/L5_absmax": 0.68747491, "gate_a2b/L5_rms": 0.51450694, "gate_c2b/L1_absmean": 0.01470184, "gate_c2b/L1_absmax": 0.46574125, "gate_c2b/L1_rms": 0.0549736, "gate_c2b/L3_absmean": 0.2086654, "gate_c2b/L3_absmax": 0.43314967, "gate_c2b/L3_rms": 0.21324377, "gate_c2b/L5_absmean": 0.53289908, "gate_c2b/L5_absmax": 0.7342211, "gate_c2b/L5_rms": 0.54494232, "gate_b2a/L7_absmean": 0.08012814, "gate_b2a/L7_absmax": 0.25703299, "gate_b2a/L7_rms": 0.08344303, "gate_b2a/L11_absmean": 0.12642297, "gate_b2a/L11_absmax": 0.39552978, "gate_b2a/L11_rms": 0.12898354, "gate_b2a/L14_absmean": 0.3061744, "gate_b2a/L14_absmax": 0.50252271, "gate_b2a/L14_rms": 0.3074908, "gate_b2c/L3_absmean": 0.08488271, "gate_b2c/L3_absmax": 0.58023369, "gate_b2c/L3_rms": 0.10427587, "gate_b2c/L5_absmean": 0.11707072, "gate_b2c/L5_absmax": 0.71263152, "gate_b2c/L5_rms": 0.12393176, "gate_b2c/L6_absmean": 0.17275052, "gate_b2c/L6_absmax": 0.47578558, "gate_b2c/L6_rms": 0.17619312}, "streams": {"cos_a2b/round0": 0.388402, "cos_c2b/round0": 0.286675, "cos_b2a/round0": 0.136583, "cos_b2c/round0": 0.143152, "rms_a/round0": 0.876423, "rms_b/round0": 1.674673, "rms_c/round0": 1.263834, "cos_a2b/round1": 0.408509, "cos_c2b/round1": 0.265058, "cos_b2a/round1": 0.22484, "cos_b2c/round1": 0.123579, "rms_a/round1": 1.564108, "rms_b/round1": 3.179695, "rms_c/round1": 2.076478, "cos_a2b/round2": 0.525486, "cos_c2b/round2": 0.488279, "cos_b2a/round2": 0.076814, "cos_b2c/round2": 0.126267, "rms_a/round2": 3.407359, "rms_b/round2": 7.047186, "rms_c/round2": 3.347925}, "alpha_mean": 0.4422863474390937, "alpha_hist": [0.01035, 0.07219, 0.15691, 0.19511, 0.18541, 0.16547, 0.13165, 0.07178, 0.01082, 0.00032], "alpha_by_type": {"special": {"share": 0.00085, "mean": 0.56165, "std": 0.09602}, "whitespace": {"share": 0.0312, "mean": 0.54682, "std": 0.12743}, "digit": {"share": 0.05371, "mean": 0.44197, "std": 0.13328}, "word_start": {"share": 0.56981, "mean": 0.46237, "std": 0.16763}, "word_cont": {"share": 0.25113, "mean": 0.32745, "std": 0.13756}, "punct": {"share": 0.09329, "mean": 0.5929, "std": 0.16316}}, "alpha_p05": 0.17246, "alpha_p50": 0.44324, "alpha_p95": 0.72115}
{"step": 16310, "total_steps": 72479, "tokens": 8551137280, "loss": 4.638347744941711, "l_mix": 2.8694899678230286, "l_a": 2.9702908396720886, "l_c": 2.920323461294174, "z_loss": 16.73437261581421, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2076936811208725, "tokens_per_second": 1452104.9194153026, "eta_seconds": 20280.030924940107, "data_wait_seconds": 1.8984079360961914e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 16320, "total_steps": 72479, "tokens": 8556380160, "loss": 4.625724613666534, "l_mix": 2.862644076347351, "l_a": 2.962402820587158, "l_c": 2.9089477360248566, "z_loss": 16.753732204437256, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1817668378353119, "tokens_per_second": 1448762.481066805, "eta_seconds": 20323.200094413754, "data_wait_seconds": 2.2619962692260742e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 16330, "total_steps": 72479, "tokens": 8561623040, "loss": 4.66231507062912, "l_mix": 2.8848282396793365, "l_a": 2.9862045645713806, "l_c": 2.9329715073108673, "z_loss": 17.338276863098145, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.24747730791568756, "tokens_per_second": 1391728.647017995, "eta_seconds": 21152.289259171488, "data_wait_seconds": 2.3365020751953125e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08236}}
{"step": 16340, "total_steps": 72479, "tokens": 8566865920, "loss": 3.3041343092918396, "l_mix": 2.042537122964859, "l_a": 2.121338978409767, "l_c": 2.0777582973241806, "z_loss": 18.679531574249268, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2020990252494812, "tokens_per_second": 1441691.1937673443, "eta_seconds": 20415.609222865107, "data_wait_seconds": 2.9653310775756836e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08237}}
{"step": 16350, "total_steps": 72479, "tokens": 8572108800, "loss": 4.572498798370361, "l_mix": 2.8291743397712708, "l_a": 2.928796887397766, "l_c": 2.876793712377548, "z_loss": 16.47257626056671, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18565873801708221, "tokens_per_second": 1457334.2606661625, "eta_seconds": 20192.86991753578, "data_wait_seconds": 3.889203071594238e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08232}}
{"step": 16360, "total_steps": 72479, "tokens": 8577351680, "loss": 4.606585085391998, "l_mix": 2.849801182746887, "l_a": 2.9507216215133667, "l_c": 2.8995876610279083, "z_loss": 16.909966230392456, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2069164663553238, "tokens_per_second": 1426493.64146827, "eta_seconds": 20625.761949920656, "data_wait_seconds": 2.238154411315918e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20764, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 16370, "total_steps": 72479, "tokens": 8582594560, "loss": 4.682787537574768, "l_mix": 2.8971122205257416, "l_a": 2.997655838727951, "l_c": 2.9488609433174133, "z_loss": 17.202975749969482, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19843946397304535, "tokens_per_second": 1338606.9805345228, "eta_seconds": 21976.036147856714, "data_wait_seconds": 2.09808349609375e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20764, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 16380, "total_steps": 72479, "tokens": 8587837440, "loss": 4.761679947376251, "l_mix": 2.9478860199451447, "l_a": 3.0414389967918396, "l_c": 2.99897301197052, "z_loss": 16.702662467956543, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17812800407409668, "tokens_per_second": 1212369.9324145368, "eta_seconds": 24259.948820590973, "data_wait_seconds": 1.7583370208740234e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20763, "cosmopedia_v2": 0.20043, "finemath": 0.08236}}
{"step": 16390, "total_steps": 72479, "tokens": 8593080320, "loss": 4.721587061882019, "l_mix": 2.9216522574424744, "l_a": 3.020890951156616, "l_c": 2.9731684625148773, "z_loss": 17.168097972869873, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18980669975280762, "tokens_per_second": 1247047.4411480927, "eta_seconds": 23581.131448316573, "data_wait_seconds": 2.5272369384765625e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20763, "cosmopedia_v2": 0.20043, "finemath": 0.08237}}
{"step": 16400, "total_steps": 72479, "tokens": 8598323200, "loss": 4.903463900089264, "l_mix": 3.0360551476478577, "l_a": 3.132203310728073, "l_c": 3.08704736828804, "z_loss": 16.33565402030945, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15735487639904022, "tokens_per_second": 1392369.991630039, "eta_seconds": 21116.188174653053, "data_wait_seconds": 2.03549861907959e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20768, "cosmopedia_v2": 0.20043, "finemath": 0.08232}, "gates": {"gate_a2b/L1_absmean": 0.27235374, "gate_a2b/L1_absmax": 0.64961958, "gate_a2b/L1_rms": 0.29456648, "gate_a2b/L3_absmean": 0.32025966, "gate_a2b/L3_absmax": 0.50320721, "gate_a2b/L3_rms": 0.32512286, "gate_a2b/L5_absmean": 0.50703728, "gate_a2b/L5_absmax": 0.67918795, "gate_a2b/L5_rms": 0.51340264, "gate_c2b/L1_absmean": 0.01456687, "gate_c2b/L1_absmax": 0.4673802, "gate_c2b/L1_rms": 0.05477681, "gate_c2b/L3_absmean": 0.20876308, "gate_c2b/L3_absmax": 0.43756911, "gate_c2b/L3_rms": 0.21325341, "gate_c2b/L5_absmean": 0.53304571, "gate_c2b/L5_absmax": 0.71924412, "gate_c2b/L5_rms": 0.5450905, "gate_b2a/L7_absmean": 0.08084729, "gate_b2a/L7_absmax": 0.25727651, "gate_b2a/L7_rms": 0.08410707, "gate_b2a/L11_absmean": 0.12717739, "gate_b2a/L11_absmax": 0.40235397, "gate_b2a/L11_rms": 0.12980735, "gate_b2a/L14_absmean": 0.30641246, "gate_b2a/L14_absmax": 0.5030632, "gate_b2a/L14_rms": 0.30765918, "gate_b2c/L3_absmean": 0.08418125, "gate_b2c/L3_absmax": 0.57821584, "gate_b2c/L3_rms": 0.10354809, "gate_b2c/L5_absmean": 0.11677425, "gate_b2c/L5_absmax": 0.7093395, "gate_b2c/L5_rms": 0.12355708, "gate_b2c/L6_absmean": 0.17232358, "gate_b2c/L6_absmax": 0.47046256, "gate_b2c/L6_rms": 0.17572396}, "streams": {"cos_a2b/round0": 0.383139, "cos_c2b/round0": 0.280526, "cos_b2a/round0": 0.143225, "cos_b2c/round0": 0.137408, "rms_a/round0": 0.870811, "rms_b/round0": 1.672619, "rms_c/round0": 1.252384, "cos_a2b/round1": 0.403952, "cos_c2b/round1": 0.276647, "cos_b2a/round1": 0.23327, "cos_b2c/round1": 0.140345, "rms_a/round1": 1.562418, "rms_b/round1": 3.157719, "rms_c/round1": 2.056006, "cos_a2b/round2": 0.537232, "cos_c2b/round2": 0.482375, "cos_b2a/round2": 0.080998, "cos_b2c/round2": 0.145135, "rms_a/round2": 3.444263, "rms_b/round2": 6.867766, "rms_c/round2": 3.28575}, "alpha_mean": 0.4719464634891892, "alpha_hist": [0.00951, 0.04434, 0.123, 0.17749, 0.20189, 0.19913, 0.14127, 0.08189, 0.02116, 0.00032], "alpha_by_type": {"special": {"share": 0.00049, "mean": 0.59261, "std": 0.11535}, "whitespace": {"share": 0.05818, "mean": 0.53071, "std": 0.12192}, "digit": {"share": 0.03548, "mean": 0.52363, "std": 0.13121}, "word_start": {"share": 0.55359, "mean": 0.48974, "std": 0.16168}, "word_cont": {"share": 0.2444, "mean": 0.35212, "std": 0.13784}, "punct": {"share": 0.10786, "mean": 0.60288, "std": 0.16257}}, "alpha_p05": 0.19012, "alpha_p50": 0.46442, "alpha_p95": 0.75455}
{"step": 16410, "total_steps": 72479, "tokens": 8603566080, "loss": 4.978919804096222, "l_mix": 3.0841744244098663, "l_a": 3.1763587594032288, "l_c": 3.1338517367839813, "z_loss": 16.821805477142334, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16637210547924042, "tokens_per_second": 1302708.2571292606, "eta_seconds": 22565.53123934269, "data_wait_seconds": 2.0772218704223633e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20768, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 16420, "total_steps": 72479, "tokens": 8608808960, "loss": 4.8966861963272095, "l_mix": 3.032733380794525, "l_a": 3.1218939125537872, "l_c": 3.085785001516342, "z_loss": 16.49038791656494, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18446753919124603, "tokens_per_second": 1453605.97260613, "eta_seconds": 20219.41402683258, "data_wait_seconds": 2.9146671295166016e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 16430, "total_steps": 72479, "tokens": 8614051840, "loss": 4.812350392341614, "l_mix": 2.9806261360645294, "l_a": 3.071953445672989, "l_c": 3.028285324573517, "z_loss": 16.527179956436157, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1729852557182312, "tokens_per_second": 1385741.965949422, "eta_seconds": 21205.83689754009, "data_wait_seconds": 2.5719404220581055e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 16440, "total_steps": 72479, "tokens": 8619294720, "loss": 4.888960361480713, "l_mix": 3.028230369091034, "l_a": 3.118364632129669, "l_c": 3.07842555642128, "z_loss": 16.928931951522827, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18262767791748047, "tokens_per_second": 1268032.588122864, "eta_seconds": 23170.20517232418, "data_wait_seconds": 1.8388032913208008e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08236}}
{"step": 16450, "total_steps": 72479, "tokens": 8624537600, "loss": 3.302911549806595, "l_mix": 2.0410579442977905, "l_a": 2.125094249844551, "l_c": 2.0748027116060257, "z_loss": 18.844401597976685, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2245643138885498, "tokens_per_second": 1447562.841140183, "eta_seconds": 20292.95828626156, "data_wait_seconds": 1.8596649169921875e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08237}}
{"step": 16460, "total_steps": 72479, "tokens": 8629780480, "loss": 4.674949467182159, "l_mix": 2.8929302990436554, "l_a": 2.994895190000534, "l_c": 2.9396820664405823, "z_loss": 16.458505868911743, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1670549362897873, "tokens_per_second": 1442460.6974955106, "eta_seconds": 20361.10205497742, "data_wait_seconds": 2.3424625396728516e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08232}}
{"step": 16470, "total_steps": 72479, "tokens": 8635023360, "loss": 4.603046417236328, "l_mix": 2.847318857908249, "l_a": 2.9490777254104614, "l_c": 2.897792249917984, "z_loss": 16.664881229400635, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17927858233451843, "tokens_per_second": 1441611.1414397655, "eta_seconds": 20369.46423892975, "data_wait_seconds": 1.8209218978881836e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 16480, "total_steps": 72479, "tokens": 8640266240, "loss": 4.509875953197479, "l_mix": 2.790456622838974, "l_a": 2.890844911336899, "l_c": 2.834996908903122, "z_loss": 16.66629385948181, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15413928031921387, "tokens_per_second": 1280154.930029998, "eta_seconds": 22934.414439439774, "data_wait_seconds": 0.05704924464225769, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 16490, "total_steps": 72479, "tokens": 8645509120, "loss": 4.673957407474518, "l_mix": 2.8915830552577972, "l_a": 2.993902325630188, "l_c": 2.941786766052246, "z_loss": 16.676249265670776, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17044080793857574, "tokens_per_second": 1481045.249630938, "eta_seconds": 19820.029698157312, "data_wait_seconds": 1.609325408935547e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 16500, "total_steps": 72479, "tokens": 8650752000, "loss": 4.566435098648071, "l_mix": 2.8265587389469147, "l_a": 2.9241995215415955, "l_c": 2.869820922613144, "z_loss": 16.70130681991577, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1803208887577057, "tokens_per_second": 1396325.0152771834, "eta_seconds": 21018.82987906933, "data_wait_seconds": 1.8268823623657227e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20042, "finemath": 0.08236}, "gates": {"gate_a2b/L1_absmean": 0.27398428, "gate_a2b/L1_absmax": 0.65276295, "gate_a2b/L1_rms": 0.29591784, "gate_a2b/L3_absmean": 0.32157728, "gate_a2b/L3_absmax": 0.50201946, "gate_a2b/L3_rms": 0.32646731, "gate_a2b/L5_absmean": 0.50841439, "gate_a2b/L5_absmax": 0.68096316, "gate_a2b/L5_rms": 0.51481658, "gate_c2b/L1_absmean": 0.01435186, "gate_c2b/L1_absmax": 0.46359286, "gate_c2b/L1_rms": 0.05425087, "gate_c2b/L3_absmean": 0.20845047, "gate_c2b/L3_absmax": 0.43400908, "gate_c2b/L3_rms": 0.21287528, "gate_c2b/L5_absmean": 0.534293, "gate_c2b/L5_absmax": 0.72120476, "gate_c2b/L5_rms": 0.54639041, "gate_b2a/L7_absmean": 0.08063237, "gate_b2a/L7_absmax": 0.25846696, "gate_b2a/L7_rms": 0.08380735, "gate_b2a/L11_absmean": 0.12746538, "gate_b2a/L11_absmax": 0.39693311, "gate_b2a/L11_rms": 0.12997942, "gate_b2a/L14_absmean": 0.30874205, "gate_b2a/L14_absmax": 0.50032127, "gate_b2a/L14_rms": 0.30993286, "gate_b2c/L3_absmean": 0.08555362, "gate_b2c/L3_absmax": 0.57767826, "gate_b2c/L3_rms": 0.10482419, "gate_b2c/L5_absmean": 0.11742197, "gate_b2c/L5_absmax": 0.71277237, "gate_b2c/L5_rms": 0.12416701, "gate_b2c/L6_absmean": 0.17324339, "gate_b2c/L6_absmax": 0.46930513, "gate_b2c/L6_rms": 0.17660064}, "streams": {"cos_a2b/round0": 0.385719, "cos_c2b/round0": 0.293377, "cos_b2a/round0": 0.152732, "cos_b2c/round0": 0.146014, "rms_a/round0": 0.858402, "rms_b/round0": 1.649835, "rms_c/round0": 1.246434, "cos_a2b/round1": 0.413686, "cos_c2b/round1": 0.276209, "cos_b2a/round1": 0.229669, "cos_b2c/round1": 0.134342, "rms_a/round1": 1.53106, "rms_b/round1": 3.10487, "rms_c/round1": 2.032771, "cos_a2b/round2": 0.544773, "cos_c2b/round2": 0.483705, "cos_b2a/round2": 0.08533, "cos_b2c/round2": 0.13952, "rms_a/round2": 3.439031, "rms_b/round2": 7.026115, "rms_c/round2": 3.273118}, "alpha_mean": 0.4544571803713495, "alpha_hist": [0.00784, 0.04797, 0.12633, 0.19443, 0.23051, 0.19624, 0.12766, 0.0612, 0.00777, 5e-05], "alpha_by_type": {"special": {"share": 0.00043, "mean": 0.55873, "std": 0.12252}, "whitespace": {"share": 0.02219, "mean": 0.55358, "std": 0.10589}, "digit": {"share": 0.0506, "mean": 0.38953, "std": 0.16623}, "word_start": {"share": 0.58388, "mean": 0.47348, "std": 0.14371}, "word_cont": {"share": 0.22421, "mean": 0.34011, "std": 0.13208}, "punct": {"share": 0.1187, "mean": 0.58567, "std": 0.13906}}, "alpha_p05": 0.20915, "alpha_p50": 0.43927, "alpha_p95": 0.72292}
{"step": 16510, "total_steps": 72479, "tokens": 8655994880, "loss": 4.8903802037239075, "l_mix": 3.0293060541152954, "l_a": 3.1187106668949127, "l_c": 3.079328864812851, "z_loss": 16.621944427490234, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16781550645828247, "tokens_per_second": 1174241.9721075892, "eta_seconds": 24989.632264065745, "data_wait_seconds": 2.956390380859375e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20769, "cosmopedia_v2": 0.20042, "finemath": 0.08231}}
{"step": 16520, "total_steps": 72479, "tokens": 8661237760, "loss": 4.996894419193268, "l_mix": 3.0950666666030884, "l_a": 3.187410145998001, "l_c": 3.1466103196144104, "z_loss": 16.21556806564331, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.23500914871692657, "tokens_per_second": 1217833.3304731075, "eta_seconds": 24090.84351518154, "data_wait_seconds": 3.045797348022461e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20769, "cosmopedia_v2": 0.20042, "finemath": 0.08232}}
{"step": 16530, "total_steps": 72479, "tokens": 8666480640, "loss": 4.906722962856293, "l_mix": 3.040328800678253, "l_a": 3.127894848585129, "l_c": 3.087867945432663, "z_loss": 16.652138710021973, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21820925176143646, "tokens_per_second": 1432855.3152355868, "eta_seconds": 20471.98276064396, "data_wait_seconds": 2.3484230041503906e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 16540, "total_steps": 72479, "tokens": 8671723520, "loss": 4.923945724964142, "l_mix": 3.0502084493637085, "l_a": 3.138281911611557, "l_c": 3.101928561925888, "z_loss": 16.741551160812378, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2080366015434265, "tokens_per_second": 1401710.1802360956, "eta_seconds": 20923.11723601818, "data_wait_seconds": 2.893805503845215e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20768, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 16550, "total_steps": 72479, "tokens": 8676966400, "loss": 5.009569048881531, "l_mix": 3.103977233171463, "l_a": 3.1925357282161713, "l_c": 3.1539406776428223, "z_loss": 16.487491011619568, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17368070781230927, "tokens_per_second": 1440137.8585396162, "eta_seconds": 20361.178187298774, "data_wait_seconds": 1.9997358322143555e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08236}}
{"step": 16560, "total_steps": 72479, "tokens": 8682209280, "loss": 3.231835752725601, "l_mix": 1.9974927008152008, "l_a": 2.0766032487154007, "l_c": 2.031862825155258, "z_loss": 18.031728267669678, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2729122042655945, "tokens_per_second": 1298501.2274822546, "eta_seconds": 22578.07697944641, "data_wait_seconds": 3.221631050109863e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08237}}
{"step": 16570, "total_steps": 72479, "tokens": 8687452160, "loss": 4.485400021076202, "l_mix": 2.774174749851227, "l_a": 2.8785434663295746, "l_c": 2.8199342787265778, "z_loss": 16.818262577056885, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.14380556344985962, "tokens_per_second": 1412256.2739267817, "eta_seconds": 20755.735579419135, "data_wait_seconds": 1.7583370208740234e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08232}}
{"step": 16580, "total_steps": 72479, "tokens": 8692695040, "loss": 4.580196499824524, "l_mix": 2.832977205514908, "l_a": 2.9375084936618805, "l_c": 2.880940556526184, "z_loss": 16.84402632713318, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15649722516536713, "tokens_per_second": 1449589.4327247648, "eta_seconds": 20217.569368529323, "data_wait_seconds": 1.990795135498047e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 16590, "total_steps": 72479, "tokens": 8697937920, "loss": 4.5698657631874084, "l_mix": 2.8276590704917908, "l_a": 2.9288388788700104, "l_c": 2.872966766357422, "z_loss": 16.650116443634033, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16431081295013428, "tokens_per_second": 1413165.016311312, "eta_seconds": 20734.96845293045, "data_wait_seconds": 1.7195940017700195e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 16600, "total_steps": 72479, "tokens": 8703180800, "loss": 4.4991448521614075, "l_mix": 2.7831127047538757, "l_a": 2.8835747838020325, "l_c": 2.830861896276474, "z_loss": 17.00963306427002, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15966898202896118, "tokens_per_second": 1432815.9171276898, "eta_seconds": 20446.931669163707, "data_wait_seconds": 1.6689300537109375e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.2752066, "gate_a2b/L1_absmax": 0.66088516, "gate_a2b/L1_rms": 0.29709169, "gate_a2b/L3_absmean": 0.32254291, "gate_a2b/L3_absmax": 0.50344235, "gate_a2b/L3_rms": 0.32740957, "gate_a2b/L5_absmean": 0.50918037, "gate_a2b/L5_absmax": 0.68150181, "gate_a2b/L5_rms": 0.51555318, "gate_c2b/L1_absmean": 0.01436383, "gate_c2b/L1_absmax": 0.4635067, "gate_c2b/L1_rms": 0.05442255, "gate_c2b/L3_absmean": 0.20889369, "gate_c2b/L3_absmax": 0.43624324, "gate_c2b/L3_rms": 0.21335809, "gate_c2b/L5_absmean": 0.5339027, "gate_c2b/L5_absmax": 0.72292197, "gate_c2b/L5_rms": 0.54594624, "gate_b2a/L7_absmean": 0.08051414, "gate_b2a/L7_absmax": 0.25829133, "gate_b2a/L7_rms": 0.08376982, "gate_b2a/L11_absmean": 0.12759978, "gate_b2a/L11_absmax": 0.39939651, "gate_b2a/L11_rms": 0.130135, "gate_b2a/L14_absmean": 0.30854014, "gate_b2a/L14_absmax": 0.5004583, "gate_b2a/L14_rms": 0.30975199, "gate_b2c/L3_absmean": 0.08580508, "gate_b2c/L3_absmax": 0.57824636, "gate_b2c/L3_rms": 0.10504017, "gate_b2c/L5_absmean": 0.11799693, "gate_b2c/L5_absmax": 0.70999295, "gate_b2c/L5_rms": 0.12473752, "gate_b2c/L6_absmean": 0.17367278, "gate_b2c/L6_absmax": 0.47581729, "gate_b2c/L6_rms": 0.17705779}, "streams": {"cos_a2b/round0": 0.389999, "cos_c2b/round0": 0.286148, "cos_b2a/round0": 0.144109, "cos_b2c/round0": 0.138815, "rms_a/round0": 0.867766, "rms_b/round0": 1.657301, "rms_c/round0": 1.265495, "cos_a2b/round1": 0.407843, "cos_c2b/round1": 0.262842, "cos_b2a/round1": 0.226247, "cos_b2c/round1": 0.132056, "rms_a/round1": 1.560574, "rms_b/round1": 3.175917, "rms_c/round1": 2.065676, "cos_a2b/round2": 0.521641, "cos_c2b/round2": 0.490138, "cos_b2a/round2": 0.072297, "cos_b2c/round2": 0.129863, "rms_a/round2": 3.44513, "rms_b/round2": 7.150794, "rms_c/round2": 3.382586}, "alpha_mean": 0.4402809784180022, "alpha_hist": [0.0161, 0.07085, 0.15862, 0.2038, 0.18588, 0.15147, 0.11311, 0.08038, 0.01932, 0.00047], "alpha_by_type": {"special": {"share": 0.0011, "mean": 0.54687, "std": 0.11016}, "whitespace": {"share": 0.03503, "mean": 0.53419, "std": 0.1273}, "digit": {"share": 0.05006, "mean": 0.44339, "std": 0.11472}, "word_start": {"share": 0.55005, "mean": 0.46396, "std": 0.17222}, "word_cont": {"share": 0.25488, "mean": 0.30966, "std": 0.13239}, "punct": {"share": 0.10887, "mean": 0.59371, "std": 0.17092}}, "alpha_p05": 0.17301, "alpha_p50": 0.42608, "alpha_p95": 0.74835}
{"step": 16610, "total_steps": 72479, "tokens": 8708423680, "loss": 4.5849661231040955, "l_mix": 2.837971568107605, "l_a": 2.9353930056095123, "l_c": 2.882404178380966, "z_loss": 16.551602602005005, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19428782165050507, "tokens_per_second": 1472063.328680886, "eta_seconds": 19898.22428240776, "data_wait_seconds": 1.5497207641601562e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08236}}
{"step": 16620, "total_steps": 72479, "tokens": 8713666560, "loss": 3.4643130600452423, "l_mix": 2.1409761160612106, "l_a": 2.222964659333229, "l_c": 2.1818223148584366, "z_loss": 19.007713556289673, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.25563517212867737, "tokens_per_second": 1407978.805086188, "eta_seconds": 20800.173472929, "data_wait_seconds": 3.9011240005493164e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20764, "cosmopedia_v2": 0.20042, "finemath": 0.08237}}
{"step": 16630, "total_steps": 72479, "tokens": 8718909440, "loss": 4.751190662384033, "l_mix": 2.942430794239044, "l_a": 3.0298416912555695, "l_c": 2.9937785267829895, "z_loss": 16.736563205718994, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20545384287834167, "tokens_per_second": 1215305.7802194413, "eta_seconds": 24093.49234454632, "data_wait_seconds": 1.767277717590332e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.2077, "cosmopedia_v2": 0.20042, "finemath": 0.08232}}
{"step": 16640, "total_steps": 72479, "tokens": 8724152320, "loss": 4.944717645645142, "l_mix": 3.063852936029434, "l_a": 3.1550627648830414, "l_c": 3.1089330911636353, "z_loss": 16.657206535339355, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18446052074432373, "tokens_per_second": 1385496.3663407709, "eta_seconds": 21130.129492378233, "data_wait_seconds": 2.5063753128051758e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20769, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 16650, "total_steps": 72479, "tokens": 8729395200, "loss": 5.0042948722839355, "l_mix": 3.1006689369678497, "l_a": 3.1879919171333313, "l_c": 3.1518391966819763, "z_loss": 16.76559829711914, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2065654993057251, "tokens_per_second": 1387322.7644842314, "eta_seconds": 21098.532728886606, "data_wait_seconds": 4.5418739318847656e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20769, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 16660, "total_steps": 72479, "tokens": 8734638080, "loss": 4.8999751806259155, "l_mix": 3.035280466079712, "l_a": 3.1260509490966797, "l_c": 3.084050267934799, "z_loss": 16.642855405807495, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.24167831242084503, "tokens_per_second": 1437232.9966330936, "eta_seconds": 20362.20427763462, "data_wait_seconds": 3.069639205932617e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20768, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 16670, "total_steps": 72479, "tokens": 8739880960, "loss": 4.528353244066238, "l_mix": 2.805307850241661, "l_a": 2.8871081471443176, "l_c": 2.8507528454065323, "z_loss": 16.870431780815125, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.23522666096687317, "tokens_per_second": 1396515.3123534794, "eta_seconds": 20952.14333360195, "data_wait_seconds": 1.9788742065429688e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20768, "cosmopedia_v2": 0.20042, "finemath": 0.08236}}
{"step": 16680, "total_steps": 72479, "tokens": 8745123840, "loss": 4.584336876869202, "l_mix": 2.836155444383621, "l_a": 2.937556743621826, "l_c": 2.88413605093956, "z_loss": 16.73437213897705, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17122259736061096, "tokens_per_second": 1402560.3986265387, "eta_seconds": 20858.100756764412, "data_wait_seconds": 1.564621925354004e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08231}}
{"step": 16690, "total_steps": 72479, "tokens": 8750366720, "loss": 4.56447035074234, "l_mix": 2.823931634426117, "l_a": 2.925964802503586, "l_c": 2.8702447414398193, "z_loss": 16.760176420211792, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.14933887124061584, "tokens_per_second": 1406821.4287486388, "eta_seconds": 20791.19825322628, "data_wait_seconds": 1.7136335372924805e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08232}}
{"step": 16700, "total_steps": 72479, "tokens": 8755609600, "loss": 4.49824795126915, "l_mix": 2.782817393541336, "l_a": 2.8854807317256927, "l_c": 2.8269759714603424, "z_loss": 16.93415904045105, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1673719435930252, "tokens_per_second": 1425094.8550031544, "eta_seconds": 20520.921992897987, "data_wait_seconds": 2.104043960571289e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.27496424, "gate_a2b/L1_absmax": 0.65667635, "gate_a2b/L1_rms": 0.29674545, "gate_a2b/L3_absmean": 0.32297316, "gate_a2b/L3_absmax": 0.50288808, "gate_a2b/L3_rms": 0.32780731, "gate_a2b/L5_absmean": 0.50930065, "gate_a2b/L5_absmax": 0.68537182, "gate_a2b/L5_rms": 0.51564741, "gate_c2b/L1_absmean": 0.01488079, "gate_c2b/L1_absmax": 0.46090102, "gate_c2b/L1_rms": 0.05425259, "gate_c2b/L3_absmean": 0.20839436, "gate_c2b/L3_absmax": 0.43322062, "gate_c2b/L3_rms": 0.21284358, "gate_c2b/L5_absmean": 0.53373313, "gate_c2b/L5_absmax": 0.71459138, "gate_c2b/L5_rms": 0.54576248, "gate_b2a/L7_absmean": 0.0809442, "gate_b2a/L7_absmax": 0.25780794, "gate_b2a/L7_rms": 0.08413503, "gate_b2a/L11_absmean": 0.12735954, "gate_b2a/L11_absmax": 0.39819053, "gate_b2a/L11_rms": 0.12991528, "gate_b2a/L14_absmean": 0.30836591, "gate_b2a/L14_absmax": 0.50221908, "gate_b2a/L14_rms": 0.30957475, "gate_b2c/L3_absmean": 0.0861389, "gate_b2c/L3_absmax": 0.57832241, "gate_b2c/L3_rms": 0.10516223, "gate_b2c/L5_absmean": 0.11797582, "gate_b2c/L5_absmax": 0.71411544, "gate_b2c/L5_rms": 0.12482453, "gate_b2c/L6_absmean": 0.17368089, "gate_b2c/L6_absmax": 0.47766477, "gate_b2c/L6_rms": 0.17713676}, "streams": {"cos_a2b/round0": 0.384784, "cos_c2b/round0": 0.289696, "cos_b2a/round0": 0.15018, "cos_b2c/round0": 0.144216, "rms_a/round0": 0.86117, "rms_b/round0": 1.647516, "rms_c/round0": 1.265917, "cos_a2b/round1": 0.417778, "cos_c2b/round1": 0.271891, "cos_b2a/round1": 0.228407, "cos_b2c/round1": 0.137497, "rms_a/round1": 1.565099, "rms_b/round1": 3.172705, "rms_c/round1": 2.087373, "cos_a2b/round2": 0.538799, "cos_c2b/round2": 0.479392, "cos_b2a/round2": 0.075454, "cos_b2c/round2": 0.137983, "rms_a/round2": 3.470547, "rms_b/round2": 6.970819, "rms_c/round2": 3.38803}, "alpha_mean": 0.4694598442668081, "alpha_hist": [0.01427, 0.05576, 0.1324, 0.17464, 0.18144, 0.17303, 0.14368, 0.10048, 0.024, 0.00031], "alpha_by_type": {"special": {"share": 0.00098, "mean": 0.56893, "std": 0.09154}, "whitespace": {"share": 0.02315, "mean": 0.54913, "std": 0.11093}, "digit": {"share": 0.02777, "mean": 0.49131, "std": 0.14496}, "word_start": {"share": 0.6216, "mean": 0.49168, "std": 0.16914}, "word_cont": {"share": 0.2271, "mean": 0.32923, "std": 0.14684}, "punct": {"share": 0.09941, "mean": 0.62527, "std": 0.16025}}, "alpha_p05": 0.16106, "alpha_p50": 0.46029, "alpha_p95": 0.76977}
{"step": 16710, "total_steps": 72479, "tokens": 8760852480, "loss": 4.587268352508545, "l_mix": 2.838927775621414, "l_a": 2.9368156492710114, "l_c": 2.885371685028076, "z_loss": 16.842859506607056, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.162313312292099, "tokens_per_second": 1439123.5611302007, "eta_seconds": 20317.23909032345, "data_wait_seconds": 1.8149614334106445e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 16720, "total_steps": 72479, "tokens": 8766095360, "loss": 4.508419930934906, "l_mix": 2.7873558402061462, "l_a": 2.891531676054001, "l_c": 2.8397408723831177, "z_loss": 16.82264494895935, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2454574555158615, "tokens_per_second": 1369723.1447887942, "eta_seconds": 21342.83464744091, "data_wait_seconds": 2.2679567337036133e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08236}}
{"step": 16730, "total_steps": 72479, "tokens": 8771338240, "loss": 3.0959203243255615, "l_mix": 1.9116175472736359, "l_a": 1.9967733770608902, "l_c": 1.9447424411773682, "z_loss": 18.479331493377686, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.4181691110134125, "tokens_per_second": 1453657.4769373962, "eta_seconds": 20106.89049911499, "data_wait_seconds": 1.627206802368164e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08237}}
{"step": 16740, "total_steps": 72479, "tokens": 8776581120, "loss": 3.2253264486789703, "l_mix": 1.992847129702568, "l_a": 2.0641762614250183, "l_c": 2.0382013469934464, "z_loss": 17.660365343093872, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.3529798090457916, "tokens_per_second": 1438560.4810170901, "eta_seconds": 20314.25805006027, "data_wait_seconds": 2.238154411315918e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08238}}
{"step": 16750, "total_steps": 72479, "tokens": 8781824000, "loss": 4.755318999290466, "l_mix": 2.945955276489258, "l_a": 3.0337584912776947, "l_c": 2.9919985234737396, "z_loss": 16.366098642349243, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16218777000904083, "tokens_per_second": 1392498.631295338, "eta_seconds": 20982.459368610384, "data_wait_seconds": 2.828240394592285e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.2077, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 16760, "total_steps": 72479, "tokens": 8787066880, "loss": 4.574688971042633, "l_mix": 2.830181747674942, "l_a": 2.9313769340515137, "l_c": 2.878016471862793, "z_loss": 16.890702724456787, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1791210025548935, "tokens_per_second": 1418729.8982660775, "eta_seconds": 20590.813732552528, "data_wait_seconds": 1.9729137420654297e-05, "mix": {"fineweb_hq": 0.50961, "dclm": 0.20764, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 16770, "total_steps": 72479, "tokens": 8792309760, "loss": 4.534103512763977, "l_mix": 2.804379642009735, "l_a": 2.906752735376358, "l_c": 2.85340279340744, "z_loss": 16.771934270858765, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18005654215812683, "tokens_per_second": 1431975.8156572364, "eta_seconds": 20396.685385775567, "data_wait_seconds": 2.2113323211669922e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20763, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 16780, "total_steps": 72479, "tokens": 8797552640, "loss": 4.538553893566132, "l_mix": 2.8085871040821075, "l_a": 2.905988872051239, "l_c": 2.8551573157310486, "z_loss": 16.23037850856781, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1635901778936386, "tokens_per_second": 1422145.629374919, "eta_seconds": 20533.985204339027, "data_wait_seconds": 1.6003847122192383e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20763, "cosmopedia_v2": 0.20042, "finemath": 0.08236}}
{"step": 16790, "total_steps": 72479, "tokens": 8802795520, "loss": 4.621830701828003, "l_mix": 2.8587354123592377, "l_a": 2.9643944203853607, "l_c": 2.9070034325122833, "z_loss": 16.757810831069946, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19323524832725525, "tokens_per_second": 1348680.2334768332, "eta_seconds": 21648.62634394169, "data_wait_seconds": 3.293156623840332e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20762, "cosmopedia_v2": 0.20042, "finemath": 0.08237}}
{"step": 16800, "total_steps": 72479, "tokens": 8808038400, "loss": 4.49719363451004, "l_mix": 2.7810146808624268, "l_a": 2.887835919857025, "l_c": 2.827088624238968, "z_loss": 17.01437258720398, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15891514718532562, "tokens_per_second": 1410138.3938097183, "eta_seconds": 20701.394756817816, "data_wait_seconds": 1.5050172805786133e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20768, "cosmopedia_v2": 0.20042, "finemath": 0.08232}, "gates": {"gate_a2b/L1_absmean": 0.27564535, "gate_a2b/L1_absmax": 0.64978403, "gate_a2b/L1_rms": 0.29723382, "gate_a2b/L3_absmean": 0.32291666, "gate_a2b/L3_absmax": 0.5045042, "gate_a2b/L3_rms": 0.32778811, "gate_a2b/L5_absmean": 0.50929016, "gate_a2b/L5_absmax": 0.67926478, "gate_a2b/L5_rms": 0.51555437, "gate_c2b/L1_absmean": 0.01471277, "gate_c2b/L1_absmax": 0.45990992, "gate_c2b/L1_rms": 0.05429034, "gate_c2b/L3_absmean": 0.20910957, "gate_c2b/L3_absmax": 0.43515369, "gate_c2b/L3_rms": 0.21359178, "gate_c2b/L5_absmean": 0.53359956, "gate_c2b/L5_absmax": 0.72568536, "gate_c2b/L5_rms": 0.54563564, "gate_b2a/L7_absmean": 0.08155276, "gate_b2a/L7_absmax": 0.25787866, "gate_b2a/L7_rms": 0.08471689, "gate_b2a/L11_absmean": 0.12753764, "gate_b2a/L11_absmax": 0.39987299, "gate_b2a/L11_rms": 0.13007741, "gate_b2a/L14_absmean": 0.30901682, "gate_b2a/L14_absmax": 0.50296038, "gate_b2a/L14_rms": 0.3102262, "gate_b2c/L3_absmean": 0.08616804, "gate_b2c/L3_absmax": 0.58059084, "gate_b2c/L3_rms": 0.10530681, "gate_b2c/L5_absmean": 0.1178491, "gate_b2c/L5_absmax": 0.71407807, "gate_b2c/L5_rms": 0.12477687, "gate_b2c/L6_absmean": 0.17299132, "gate_b2c/L6_absmax": 0.47572127, "gate_b2c/L6_rms": 0.17645153}, "streams": {"cos_a2b/round0": 0.375849, "cos_c2b/round0": 0.2823, "cos_b2a/round0": 0.142395, "cos_b2c/round0": 0.134573, "rms_a/round0": 0.863009, "rms_b/round0": 1.667933, "rms_c/round0": 1.269955, "cos_a2b/round1": 0.409646, "cos_c2b/round1": 0.271988, "cos_b2a/round1": 0.227611, "cos_b2c/round1": 0.141572, "rms_a/round1": 1.564583, "rms_b/round1": 3.193892, "rms_c/round1": 2.064564, "cos_a2b/round2": 0.528392, "cos_c2b/round2": 0.48165, "cos_b2a/round2": 0.078943, "cos_b2c/round2": 0.139423, "rms_a/round2": 3.442941, "rms_b/round2": 7.029695, "rms_c/round2": 3.363924}, "alpha_mean": 0.44837767373010706, "alpha_hist": [0.02464, 0.08952, 0.14587, 0.15999, 0.16087, 0.16277, 0.14778, 0.08939, 0.01881, 0.00035], "alpha_by_type": {"special": {"share": 0.00095, "mean": 0.55848, "std": 0.08767}, "whitespace": {"share": 0.02338, "mean": 0.55522, "std": 0.11181}, "digit": {"share": 0.02193, "mean": 0.57298, "std": 0.11719}, "word_start": {"share": 0.60683, "mean": 0.47333, "std": 0.17987}, "word_cont": {"share": 0.24748, "mean": 0.30387, "std": 0.15092}, "punct": {"share": 0.09944, "mean": 0.6021, "std": 0.17118}}, "alpha_p05": 0.13753, "alpha_p50": 0.45254, "alpha_p95": 0.77046}
{"step": 16810, "total_steps": 72479, "tokens": 8813281280, "loss": 4.514367699623108, "l_mix": 2.7922958731651306, "l_a": 2.8927252292633057, "l_c": 2.8419362604618073, "z_loss": 16.733540773391724, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17649342119693756, "tokens_per_second": 1444489.3383835843, "eta_seconds": 20205.471855306627, "data_wait_seconds": 1.5467405319213867e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 16820, "total_steps": 72479, "tokens": 8818524160, "loss": 4.76510763168335, "l_mix": 2.9488297402858734, "l_a": 3.048949807882309, "l_c": 2.999819189310074, "z_loss": 16.471113801002502, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21469874680042267, "tokens_per_second": 1396586.9753935917, "eta_seconds": 20894.75722324848, "data_wait_seconds": 1.5944242477416992e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 16830, "total_steps": 72479, "tokens": 8823767040, "loss": 4.591622471809387, "l_mix": 2.8402061462402344, "l_a": 2.9449897408485413, "l_c": 2.8875620365142822, "z_loss": 16.50701379776001, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18820317089557648, "tokens_per_second": 1400731.6949757133, "eta_seconds": 20829.187357330324, "data_wait_seconds": 1.8864870071411133e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 16840, "total_steps": 72479, "tokens": 8829009920, "loss": 3.1207745373249054, "l_mix": 1.9276476502418518, "l_a": 2.003087341785431, "l_c": 1.9679017961025238, "z_loss": 18.300204038619995, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2656092643737793, "tokens_per_second": 1413780.6444817106, "eta_seconds": 20633.229168796537, "data_wait_seconds": 2.4557113647460938e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08236}}
{"step": 16850, "total_steps": 72479, "tokens": 8834252800, "loss": 3.349863290786743, "l_mix": 2.0711972415447235, "l_a": 2.1489258855581284, "l_c": 2.1073340326547623, "z_loss": 17.88056516647339, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.3084186613559723, "tokens_per_second": 1432667.9601708185, "eta_seconds": 20357.555248546603, "data_wait_seconds": 1.806020736694336e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08237}}
{"step": 16860, "total_steps": 72479, "tokens": 8839495680, "loss": 4.524411618709564, "l_mix": 2.79945370554924, "l_a": 2.898534834384918, "l_c": 2.8456644415855408, "z_loss": 16.98054814338684, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15164147317409515, "tokens_per_second": 1428400.8526871807, "eta_seconds": 20414.699569201468, "data_wait_seconds": 1.96993350982666e-05, "mix": {"fineweb_hq": 0.50961, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 16870, "total_steps": 72479, "tokens": 8844738560, "loss": 4.712483108043671, "l_mix": 2.916359543800354, "l_a": 3.0183127522468567, "l_c": 2.9631520211696625, "z_loss": 16.839452743530273, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16787053644657135, "tokens_per_second": 1287214.0869908694, "eta_seconds": 22649.792048311236, "data_wait_seconds": 2.244114875793457e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20765, "cosmopedia_v2": 0.20041, "finemath": 0.08234}}
{"step": 16880, "total_steps": 72479, "tokens": 8849981440, "loss": 4.597669720649719, "l_mix": 2.843662679195404, "l_a": 2.9483937323093414, "l_c": 2.8925960958004, "z_loss": 17.102200746536255, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17041142284870148, "tokens_per_second": 1432590.399994684, "eta_seconds": 20347.678242230417, "data_wait_seconds": 3.197789192199707e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20764, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 16890, "total_steps": 72479, "tokens": 8855224320, "loss": 4.5395495891571045, "l_mix": 2.808450073003769, "l_a": 2.9086372554302216, "l_c": 2.8561961948871613, "z_loss": 16.494269609451294, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18019536137580872, "tokens_per_second": 1417588.8722139338, "eta_seconds": 20559.307570242883, "data_wait_seconds": 2.9146671295166016e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20764, "cosmopedia_v2": 0.20041, "finemath": 0.08236}}
{"step": 16900, "total_steps": 72479, "tokens": 8860467200, "loss": 4.509253919124603, "l_mix": 2.788652688264847, "l_a": 2.892986088991165, "l_c": 2.8368360698223114, "z_loss": 16.544544219970703, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20331327617168427, "tokens_per_second": 1321605.4394709307, "eta_seconds": 22048.48881649971, "data_wait_seconds": 2.1010637283325195e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20763, "cosmopedia_v2": 0.20041, "finemath": 0.08237}, "gates": {"gate_a2b/L1_absmean": 0.27698031, "gate_a2b/L1_absmax": 0.65156436, "gate_a2b/L1_rms": 0.29846844, "gate_a2b/L3_absmean": 0.32385102, "gate_a2b/L3_absmax": 0.50603777, "gate_a2b/L3_rms": 0.32872018, "gate_a2b/L5_absmean": 0.50967699, "gate_a2b/L5_absmax": 0.684448, "gate_a2b/L5_rms": 0.51600277, "gate_c2b/L1_absmean": 0.01441012, "gate_c2b/L1_absmax": 0.46042323, "gate_c2b/L1_rms": 0.05401953, "gate_c2b/L3_absmean": 0.20888345, "gate_c2b/L3_absmax": 0.43545261, "gate_c2b/L3_rms": 0.21336085, "gate_c2b/L5_absmean": 0.5334568, "gate_c2b/L5_absmax": 0.71854883, "gate_c2b/L5_rms": 0.54535443, "gate_b2a/L7_absmean": 0.0811753, "gate_b2a/L7_absmax": 0.25763103, "gate_b2a/L7_rms": 0.08433437, "gate_b2a/L11_absmean": 0.12740019, "gate_b2a/L11_absmax": 0.40111583, "gate_b2a/L11_rms": 0.12994312, "gate_b2a/L14_absmean": 0.30931872, "gate_b2a/L14_absmax": 0.50126737, "gate_b2a/L14_rms": 0.31051987, "gate_b2c/L3_absmean": 0.08694528, "gate_b2c/L3_absmax": 0.58204395, "gate_b2c/L3_rms": 0.10596675, "gate_b2c/L5_absmean": 0.11807472, "gate_b2c/L5_absmax": 0.71581525, "gate_b2c/L5_rms": 0.12496278, "gate_b2c/L6_absmean": 0.17394327, "gate_b2c/L6_absmax": 0.47674221, "gate_b2c/L6_rms": 0.17740622}, "streams": {"cos_a2b/round0": 0.385039, "cos_c2b/round0": 0.287452, "cos_b2a/round0": 0.148578, "cos_b2c/round0": 0.141355, "rms_a/round0": 0.860782, "rms_b/round0": 1.640357, "rms_c/round0": 1.248493, "cos_a2b/round1": 0.411848, "cos_c2b/round1": 0.265288, "cos_b2a/round1": 0.217511, "cos_b2c/round1": 0.13603, "rms_a/round1": 1.552095, "rms_b/round1": 3.180058, "rms_c/round1": 2.060863, "cos_a2b/round2": 0.526233, "cos_c2b/round2": 0.479757, "cos_b2a/round2": 0.061813, "cos_b2c/round2": 0.128806, "rms_a/round2": 3.401352, "rms_b/round2": 7.079164, "rms_c/round2": 3.354188}, "alpha_mean": 0.4321346806893871, "alpha_hist": [0.02164, 0.08234, 0.16325, 0.18994, 0.17842, 0.16043, 0.11687, 0.07054, 0.01627, 0.00031], "alpha_by_type": {"special": {"share": 0.00078, "mean": 0.56525, "std": 0.09991}, "whitespace": {"share": 0.02742, "mean": 0.52537, "std": 0.12221}, "digit": {"share": 0.05286, "mean": 0.36316, "std": 0.15988}, "word_start": {"share": 0.58423, "mean": 0.45461, "std": 0.17227}, "word_cont": {"share": 0.22539, "mean": 0.31545, "std": 0.15062}, "punct": {"share": 0.10933, "mean": 0.56162, "std": 0.18041}}, "alpha_p05": 0.15178, "alpha_p50": 0.41989, "alpha_p95": 0.74651}
{"step": 16910, "total_steps": 72479, "tokens": 8865710080, "loss": 4.592180132865906, "l_mix": 2.841549426317215, "l_a": 2.9421869814395905, "l_c": 2.8875635266304016, "z_loss": 17.054877042770386, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15275722742080688, "tokens_per_second": 1338378.9435670178, "eta_seconds": 21768.24434666634, "data_wait_seconds": 2.0056962966918945e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20769, "cosmopedia_v2": 0.20041, "finemath": 0.08232}}
{"step": 16920, "total_steps": 72479, "tokens": 8870952960, "loss": 4.872604310512543, "l_mix": 3.019632786512375, "l_a": 3.105276972055435, "l_c": 3.065703272819519, "z_loss": 16.773593187332153, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16277307271957397, "tokens_per_second": 1469856.0415571164, "eta_seconds": 19817.530539345742, "data_wait_seconds": 1.7583370208740234e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20768, "cosmopedia_v2": 0.20041, "finemath": 0.08233}}
{"step": 16930, "total_steps": 72479, "tokens": 8876195840, "loss": 4.696038544178009, "l_mix": 2.9094094038009644, "l_a": 2.9919404834508896, "l_c": 2.95792219042778, "z_loss": 16.70382523536682, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17859217524528503, "tokens_per_second": 1316229.0738412263, "eta_seconds": 22126.599913954735, "data_wait_seconds": 1.7851591110229492e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20768, "cosmopedia_v2": 0.20041, "finemath": 0.08234}}
{"step": 16940, "total_steps": 72479, "tokens": 8881438720, "loss": 4.491323545575142, "l_mix": 2.7826934456825256, "l_a": 2.8265537321567535, "l_c": 2.8633085042238235, "z_loss": 16.714792490005493, "lr": 0.0023, "wd": 0.1, "grad_norm": 1.751602292060852, "tokens_per_second": 1443382.1173948876, "eta_seconds": 20173.750859928132, "data_wait_seconds": 1.8090009689331055e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 16950, "total_steps": 72479, "tokens": 8886681600, "loss": 4.923481225967407, "l_mix": 3.051064223051071, "l_a": 3.1349420249462128, "l_c": 3.1008572578430176, "z_loss": 16.77187156677246, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19622637331485748, "tokens_per_second": 1354190.6804412904, "eta_seconds": 21498.588620114326, "data_wait_seconds": 1.9818544387817383e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08236}}
{"step": 16960, "total_steps": 72479, "tokens": 8891924480, "loss": 3.226975381374359, "l_mix": 1.993778482079506, "l_a": 2.0705007314682007, "l_c": 2.0337988883256912, "z_loss": 19.069561004638672, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2265140861272812, "tokens_per_second": 1426882.0274669614, "eta_seconds": 20399.686106967925, "data_wait_seconds": 2.5153160095214844e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08237}}
{"step": 16970, "total_steps": 72479, "tokens": 8897167360, "loss": 4.429964125156403, "l_mix": 2.7396998703479767, "l_a": 2.842404991388321, "l_c": 2.7861229479312897, "z_loss": 17.05892515182495, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1425364911556244, "tokens_per_second": 1400893.2976864227, "eta_seconds": 20774.389198708534, "data_wait_seconds": 2.9087066650390625e-05, "mix": {"fineweb_hq": 0.50961, "dclm": 0.20766, "cosmopedia_v2": 0.20041, "finemath": 0.08232}}
{"step": 16980, "total_steps": 72479, "tokens": 8902410240, "loss": 4.547518193721771, "l_mix": 2.8124105036258698, "l_a": 2.916781783103943, "l_c": 2.861411392688751, "z_loss": 16.49625587463379, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15844793617725372, "tokens_per_second": 1444608.3343309937, "eta_seconds": 20142.10981655121, "data_wait_seconds": 3.275275230407715e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20766, "cosmopedia_v2": 0.20041, "finemath": 0.08233}}
{"step": 16990, "total_steps": 72479, "tokens": 8907653120, "loss": 4.600339591503143, "l_mix": 2.8467802703380585, "l_a": 2.9446786642074585, "l_c": 2.8949472308158875, "z_loss": 16.715250492095947, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1468057632446289, "tokens_per_second": 1460121.383043688, "eta_seconds": 19924.519406294825, "data_wait_seconds": 2.7239322662353516e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20765, "cosmopedia_v2": 0.20041, "finemath": 0.08234}}
{"step": 17000, "total_steps": 72479, "tokens": 8912896000, "loss": 4.546385049819946, "l_mix": 2.8136253356933594, "l_a": 2.9131976664066315, "l_c": 2.8570649325847626, "z_loss": 16.809224128723145, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.14696964621543884, "tokens_per_second": 1375389.8729669822, "eta_seconds": 21148.166438984874, "data_wait_seconds": 2.0295381546020508e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20765, "cosmopedia_v2": 0.20041, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.27657506, "gate_a2b/L1_absmax": 0.65286338, "gate_a2b/L1_rms": 0.2978487, "gate_a2b/L3_absmean": 0.32351986, "gate_a2b/L3_absmax": 0.50464332, "gate_a2b/L3_rms": 0.32836261, "gate_a2b/L5_absmean": 0.50977165, "gate_a2b/L5_absmax": 0.68855566, "gate_a2b/L5_rms": 0.51605207, "gate_c2b/L1_absmean": 0.01414372, "gate_c2b/L1_absmax": 0.46276611, "gate_c2b/L1_rms": 0.05393675, "gate_c2b/L3_absmean": 0.20898817, "gate_c2b/L3_absmax": 0.43420216, "gate_c2b/L3_rms": 0.21346201, "gate_c2b/L5_absmean": 0.53312361, "gate_c2b/L5_absmax": 0.71931726, "gate_c2b/L5_rms": 0.54513389, "gate_b2a/L7_absmean": 0.08118047, "gate_b2a/L7_absmax": 0.25879359, "gate_b2a/L7_rms": 0.0844693, "gate_b2a/L11_absmean": 0.12783208, "gate_b2a/L11_absmax": 0.40177649, "gate_b2a/L11_rms": 0.13039806, "gate_b2a/L14_absmean": 0.30946577, "gate_b2a/L14_absmax": 0.50156301, "gate_b2a/L14_rms": 0.31064129, "gate_b2c/L3_absmean": 0.08708291, "gate_b2c/L3_absmax": 0.58421886, "gate_b2c/L3_rms": 0.10613412, "gate_b2c/L5_absmean": 0.11832225, "gate_b2c/L5_absmax": 0.71983385, "gate_b2c/L5_rms": 0.12525603, "gate_b2c/L6_absmean": 0.17409007, "gate_b2c/L6_absmax": 0.48135027, "gate_b2c/L6_rms": 0.17759828}, "streams": {"cos_a2b/round0": 0.380226, "cos_c2b/round0": 0.285125, "cos_b2a/round0": 0.15452, "cos_b2c/round0": 0.138997, "rms_a/round0": 0.863547, "rms_b/round0": 1.654355, "rms_c/round0": 1.25729, "cos_a2b/round1": 0.41549, "cos_c2b/round1": 0.27884, "cos_b2a/round1": 0.235224, "cos_b2c/round1": 0.14176, "rms_a/round1": 1.548167, "rms_b/round1": 3.136934, "rms_c/round1": 2.067669, "cos_a2b/round2": 0.54307, "cos_c2b/round2": 0.481298, "cos_b2a/round2": 0.079971, "cos_b2c/round2": 0.150374, "rms_a/round2": 3.436542, "rms_b/round2": 6.887219, "rms_c/round2": 3.326357}, "alpha_mean": 0.47390756100021747, "alpha_hist": [0.0117, 0.05292, 0.12146, 0.16841, 0.18311, 0.20152, 0.15063, 0.08759, 0.02205, 0.00061], "alpha_by_type": {"special": {"share": 0.00093, "mean": 0.57273, "std": 0.10388}, "whitespace": {"share": 0.0352, "mean": 0.57177, "std": 0.09826}, "digit": {"share": 0.02637, "mean": 0.48438, "std": 0.13248}, "word_start": {"share": 0.5697, "mean": 0.49134, "std": 0.16473}, "word_cont": {"share": 0.24017, "mean": 0.3519, "std": 0.14703}, "punct": {"share": 0.12762, "mean": 0.59581, "std": 0.17329}}, "alpha_p05": 0.17896, "alpha_p50": 0.49806, "alpha_p95": 0.78826}
{"step": 17010, "total_steps": 72479, "tokens": 8918138880, "loss": 4.511370837688446, "l_mix": 2.790433794260025, "l_a": 2.8936474919319153, "l_c": 2.837188333272934, "z_loss": 16.86086916923523, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1823783814907074, "tokens_per_second": 1466576.6159145823, "eta_seconds": 19829.67051050663, "data_wait_seconds": 1.4841556549072266e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20041, "finemath": 0.08236}}
{"step": 17020, "total_steps": 72479, "tokens": 8923381760, "loss": 3.364876300096512, "l_mix": 2.079805850982666, "l_a": 2.161721169948578, "l_c": 2.1156404316425323, "z_loss": 18.618976354599, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20662643015384674, "tokens_per_second": 1362484.9924094465, "eta_seconds": 21340.776855516433, "data_wait_seconds": 1.6897916793823242e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20041, "finemath": 0.08237}}
{"step": 17030, "total_steps": 72479, "tokens": 8928624640, "loss": 4.828152656555176, "l_mix": 2.991311013698578, "l_a": 3.0790963768959045, "l_c": 3.0381388068199158, "z_loss": 16.70894694328308, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16554510593414307, "tokens_per_second": 1404365.8107227401, "eta_seconds": 20700.62165429592, "data_wait_seconds": 3.591179847717285e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20769, "cosmopedia_v2": 0.20041, "finemath": 0.08233}}
{"step": 17040, "total_steps": 72479, "tokens": 8933867520, "loss": 4.795693874359131, "l_mix": 2.970233589410782, "l_a": 3.058687448501587, "l_c": 3.0205487310886383, "z_loss": 16.892727375030518, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20043669641017914, "tokens_per_second": 1454491.5670158665, "eta_seconds": 19983.617018580437, "data_wait_seconds": 1.6421079635620117e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20769, "cosmopedia_v2": 0.20041, "finemath": 0.08234}}
{"step": 17050, "total_steps": 72479, "tokens": 8939110400, "loss": 4.916359603404999, "l_mix": 3.0455127954483032, "l_a": 3.133846938610077, "l_c": 3.09685081243515, "z_loss": 16.37382936477661, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17013414204120636, "tokens_per_second": 1436237.7940195533, "eta_seconds": 20233.947103333474, "data_wait_seconds": 2.3305416107177734e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 17060, "total_steps": 72479, "tokens": 8944353280, "loss": 4.8603150844573975, "l_mix": 3.011029303073883, "l_a": 3.1017342805862427, "l_c": 3.0569789111614227, "z_loss": 16.717403411865234, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16374465823173523, "tokens_per_second": 1394596.4528614578, "eta_seconds": 20834.354348444936, "data_wait_seconds": 8.210539817810059e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20768, "cosmopedia_v2": 0.20041, "finemath": 0.08236}}
{"step": 17070, "total_steps": 72479, "tokens": 8949596160, "loss": 4.802113175392151, "l_mix": 2.975459396839142, "l_a": 3.058510482311249, "l_c": 3.0247366428375244, "z_loss": 16.796552419662476, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17030778527259827, "tokens_per_second": 1457768.3251784686, "eta_seconds": 19927.908495640757, "data_wait_seconds": 3.832578659057617e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08237}}
{"step": 17080, "total_steps": 72479, "tokens": 8954839040, "loss": 4.507754921913147, "l_mix": 2.788742274045944, "l_a": 2.8860857486724854, "l_c": 2.8383804857730865, "z_loss": 16.727017879486084, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18121986091136932, "tokens_per_second": 1411163.9377513228, "eta_seconds": 20582.322248315813, "data_wait_seconds": 2.4616718292236328e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08232}}
{"step": 17090, "total_steps": 72479, "tokens": 8960081920, "loss": 4.492151379585266, "l_mix": 2.7780606746673584, "l_a": 2.879148244857788, "l_c": 2.8289045095443726, "z_loss": 16.74714970588684, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17440475523471832, "tokens_per_second": 1437129.2066477144, "eta_seconds": 20206.80388212204, "data_wait_seconds": 1.6421079635620117e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08233}}
{"step": 17100, "total_steps": 72479, "tokens": 8965324800, "loss": 4.545909285545349, "l_mix": 2.811341643333435, "l_a": 2.916462928056717, "l_c": 2.859907865524292, "z_loss": 16.564634561538696, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18646714091300964, "tokens_per_second": 1322552.0254240045, "eta_seconds": 21953.423830485342, "data_wait_seconds": 1.683831214904785e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20766, "cosmopedia_v2": 0.20041, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.27736098, "gate_a2b/L1_absmax": 0.64636981, "gate_a2b/L1_rms": 0.29865399, "gate_a2b/L3_absmean": 0.32454845, "gate_a2b/L3_absmax": 0.50791955, "gate_a2b/L3_rms": 0.32936063, "gate_a2b/L5_absmean": 0.51034302, "gate_a2b/L5_absmax": 0.68613237, "gate_a2b/L5_rms": 0.51654053, "gate_c2b/L1_absmean": 0.01410946, "gate_c2b/L1_absmax": 0.45942277, "gate_c2b/L1_rms": 0.05377334, "gate_c2b/L3_absmean": 0.20855688, "gate_c2b/L3_absmax": 0.43461224, "gate_c2b/L3_rms": 0.21302451, "gate_c2b/L5_absmean": 0.53382027, "gate_c2b/L5_absmax": 0.72209835, "gate_c2b/L5_rms": 0.54580426, "gate_b2a/L7_absmean": 0.08139521, "gate_b2a/L7_absmax": 0.2581408, "gate_b2a/L7_rms": 0.08464828, "gate_b2a/L11_absmean": 0.12815228, "gate_b2a/L11_absmax": 0.40022013, "gate_b2a/L11_rms": 0.13070111, "gate_b2a/L14_absmean": 0.30976814, "gate_b2a/L14_absmax": 0.50894439, "gate_b2a/L14_rms": 0.31098306, "gate_b2c/L3_absmean": 0.08738343, "gate_b2c/L3_absmax": 0.58346915, "gate_b2c/L3_rms": 0.10638292, "gate_b2c/L5_absmean": 0.11898415, "gate_b2c/L5_absmax": 0.71587318, "gate_b2c/L5_rms": 0.12582518, "gate_b2c/L6_absmean": 0.17496614, "gate_b2c/L6_absmax": 0.48427415, "gate_b2c/L6_rms": 0.17851067}, "streams": {"cos_a2b/round0": 0.376777, "cos_c2b/round0": 0.27991, "cos_b2a/round0": 0.143481, "cos_b2c/round0": 0.132973, "rms_a/round0": 0.860488, "rms_b/round0": 1.655353, "rms_c/round0": 1.265383, "cos_a2b/round1": 0.408372, "cos_c2b/round1": 0.27248, "cos_b2a/round1": 0.23071, "cos_b2c/round1": 0.135433, "rms_a/round1": 1.541435, "rms_b/round1": 3.17846, "rms_c/round1": 2.055828, "cos_a2b/round2": 0.527239, "cos_c2b/round2": 0.483201, "cos_b2a/round2": 0.074874, "cos_b2c/round2": 0.133555, "rms_a/round2": 3.385073, "rms_b/round2": 7.033984, "rms_c/round2": 3.297873}, "alpha_mean": 0.4289782144701775, "alpha_hist": [0.02415, 0.0726, 0.15355, 0.19521, 0.19995, 0.17651, 0.11316, 0.05519, 0.00941, 0.00026], "alpha_by_type": {"special": {"share": 0.00076, "mean": 0.57992, "std": 0.09739}, "whitespace": {"share": 0.03027, "mean": 0.54362, "std": 0.1178}, "digit": {"share": 0.02556, "mean": 0.53666, "std": 0.1131}, "word_start": {"share": 0.55951, "mean": 0.44725, "std": 0.15878}, "word_cont": {"share": 0.2791, "mean": 0.31486, "std": 0.14373}, "punct": {"share": 0.1048, "mean": 0.57489, "std": 0.15996}}, "alpha_p05": 0.16858, "alpha_p50": 0.44348, "alpha_p95": 0.72584}
{"step": 17110, "total_steps": 72479, "tokens": 8970567680, "loss": 4.66115665435791, "l_mix": 2.8842205703258514, "l_a": 2.98649999499321, "l_c": 2.9311667680740356, "z_loss": 16.360050916671753, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16334876418113708, "tokens_per_second": 1391820.256670714, "eta_seconds": 20857.077006077765, "data_wait_seconds": 2.181529998779297e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20766, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 17120, "total_steps": 72479, "tokens": 8975810560, "loss": 4.500512719154358, "l_mix": 2.784690409898758, "l_a": 2.882644832134247, "l_c": 2.8311043679714203, "z_loss": 16.97404909133911, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17777076363563538, "tokens_per_second": 1398549.921663589, "eta_seconds": 20752.966299176216, "data_wait_seconds": 2.0176172256469727e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20041, "finemath": 0.08236}}
{"step": 17130, "total_steps": 72479, "tokens": 8981053440, "loss": 3.2720671892166138, "l_mix": 2.0222961753606796, "l_a": 2.1003854274749756, "l_c": 2.0593772530555725, "z_loss": 18.421926736831665, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.3756963014602661, "tokens_per_second": 1357475.2237477007, "eta_seconds": 21377.050574731828, "data_wait_seconds": 3.0994415283203125e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20041, "finemath": 0.08237}}
{"step": 17140, "total_steps": 72479, "tokens": 8986296320, "loss": 3.4700770676136017, "l_mix": 2.1325881481170654, "l_a": 2.2586643993854523, "l_c": 2.194411873817444, "z_loss": 15.659911036491394, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2045375555753708, "tokens_per_second": 1448792.738556289, "eta_seconds": 20026.034683823586, "data_wait_seconds": 1.621246337890625e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20764, "cosmopedia_v2": 0.20047, "finemath": 0.08232}}
{"step": 17150, "total_steps": 72479, "tokens": 8991539200, "loss": 3.508516490459442, "l_mix": 2.1569930613040924, "l_a": 2.2815899550914764, "l_c": 2.2183437049388885, "z_loss": 15.433034539222717, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21790385246276855, "tokens_per_second": 1312133.945531921, "eta_seconds": 22107.75115664005, "data_wait_seconds": 1.7344951629638672e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20764, "cosmopedia_v2": 0.20047, "finemath": 0.08233}}
{"step": 17160, "total_steps": 72479, "tokens": 8996782080, "loss": 4.841476380825043, "l_mix": 2.9985387325286865, "l_a": 3.09093376994133, "l_c": 3.0466303527355194, "z_loss": 16.683781147003174, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15480844676494598, "tokens_per_second": 1391936.812150271, "eta_seconds": 20836.49747519493, "data_wait_seconds": 2.3484230041503906e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20769, "cosmopedia_v2": 0.20041, "finemath": 0.08234}}
{"step": 17170, "total_steps": 72479, "tokens": 9002024960, "loss": 4.821111679077148, "l_mix": 2.9875957369804382, "l_a": 3.071479022502899, "l_c": 3.034685969352722, "z_loss": 16.664698362350464, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18560642004013062, "tokens_per_second": 1322501.4387400537, "eta_seconds": 21926.513002228738, "data_wait_seconds": 3.3855438232421875e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20769, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 17180, "total_steps": 72479, "tokens": 9007267840, "loss": 4.853686213493347, "l_mix": 3.006454885005951, "l_a": 3.095829725265503, "l_c": 3.056212991476059, "z_loss": 16.184394121170044, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18457481265068054, "tokens_per_second": 1364499.3873170905, "eta_seconds": 21247.794159150126, "data_wait_seconds": 2.7120113372802734e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20768, "cosmopedia_v2": 0.20041, "finemath": 0.08236}}
{"step": 17190, "total_steps": 72479, "tokens": 9012510720, "loss": 4.476342082023621, "l_mix": 2.768494814634323, "l_a": 2.8711869418621063, "l_c": 2.815893441438675, "z_loss": 17.231489419937134, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15054656565189362, "tokens_per_second": 1431869.7068031975, "eta_seconds": 20244.411271691322, "data_wait_seconds": 1.6033649444580078e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20768, "cosmopedia_v2": 0.20041, "finemath": 0.08232}}
{"step": 17200, "total_steps": 72479, "tokens": 9017753600, "loss": 4.545496463775635, "l_mix": 2.812631666660309, "l_a": 2.912813812494278, "l_c": 2.8579187989234924, "z_loss": 16.448440551757812, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17388683557510376, "tokens_per_second": 1455952.4457140397, "eta_seconds": 19905.949838757515, "data_wait_seconds": 1.5139579772949219e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.27847585, "gate_a2b/L1_absmax": 0.64043134, "gate_a2b/L1_rms": 0.29964364, "gate_a2b/L3_absmean": 0.32481974, "gate_a2b/L3_absmax": 0.50549716, "gate_a2b/L3_rms": 0.3295773, "gate_a2b/L5_absmean": 0.51080149, "gate_a2b/L5_absmax": 0.68324387, "gate_a2b/L5_rms": 0.51692951, "gate_c2b/L1_absmean": 0.01417807, "gate_c2b/L1_absmax": 0.45584604, "gate_c2b/L1_rms": 0.05369063, "gate_c2b/L3_absmean": 0.20820065, "gate_c2b/L3_absmax": 0.43062714, "gate_c2b/L3_rms": 0.21256529, "gate_c2b/L5_absmean": 0.53294259, "gate_c2b/L5_absmax": 0.72142953, "gate_c2b/L5_rms": 0.54486716, "gate_b2a/L7_absmean": 0.08175416, "gate_b2a/L7_absmax": 0.2598553, "gate_b2a/L7_rms": 0.08505196, "gate_b2a/L11_absmean": 0.12821686, "gate_b2a/L11_absmax": 0.40212494, "gate_b2a/L11_rms": 0.13073668, "gate_b2a/L14_absmean": 0.30993772, "gate_b2a/L14_absmax": 0.50739193, "gate_b2a/L14_rms": 0.31113389, "gate_b2c/L3_absmean": 0.08792308, "gate_b2c/L3_absmax": 0.58512336, "gate_b2c/L3_rms": 0.10698383, "gate_b2c/L5_absmean": 0.11904216, "gate_b2c/L5_absmax": 0.70964074, "gate_b2c/L5_rms": 0.12583841, "gate_b2c/L6_absmean": 0.17486458, "gate_b2c/L6_absmax": 0.48041138, "gate_b2c/L6_rms": 0.17838974}, "streams": {"cos_a2b/round0": 0.380461, "cos_c2b/round0": 0.284227, "cos_b2a/round0": 0.143352, "cos_b2c/round0": 0.134791, "rms_a/round0": 0.85493, "rms_b/round0": 1.650971, "rms_c/round0": 1.265247, "cos_a2b/round1": 0.413988, "cos_c2b/round1": 0.264978, "cos_b2a/round1": 0.225896, "cos_b2c/round1": 0.130269, "rms_a/round1": 1.559395, "rms_b/round1": 3.196542, "rms_c/round1": 2.070105, "cos_a2b/round2": 0.519655, "cos_c2b/round2": 0.496358, "cos_b2a/round2": 0.077573, "cos_b2c/round2": 0.127194, "rms_a/round2": 3.419348, "rms_b/round2": 7.135083, "rms_c/round2": 3.379201}, "alpha_mean": 0.4413032585304819, "alpha_hist": [0.01006, 0.06387, 0.16556, 0.20168, 0.18646, 0.16304, 0.1282, 0.06792, 0.01292, 0.00029], "alpha_by_type": {"special": {"share": 0.00081, "mean": 0.54237, "std": 0.08715}, "whitespace": {"share": 0.02448, "mean": 0.53826, "std": 0.1167}, "digit": {"share": 0.03638, "mean": 0.55951, "std": 0.12094}, "word_start": {"share": 0.55122, "mean": 0.45276, "std": 0.16745}, "word_cont": {"share": 0.27405, "mean": 0.33371, "std": 0.13159}, "punct": {"share": 0.11307, "mean": 0.58648, "std": 0.15108}}, "alpha_p05": 0.16212, "alpha_p50": 0.41419, "alpha_p95": 0.72933}
{"step": 17210, "total_steps": 72479, "tokens": 9022996480, "loss": 4.506188094615936, "l_mix": 2.786452293395996, "l_a": 2.89150270819664, "l_c": 2.8353922963142395, "z_loss": 16.67210102081299, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15068626403808594, "tokens_per_second": 1411313.9171485985, "eta_seconds": 20531.841371297836, "data_wait_seconds": 1.659989356994629e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08234}}
{"step": 17220, "total_steps": 72479, "tokens": 9028239360, "loss": 4.533219993114471, "l_mix": 2.8042533099651337, "l_a": 2.9064759612083435, "l_c": 2.8511343896389008, "z_loss": 16.834373474121094, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15509997308254242, "tokens_per_second": 1307979.3363767273, "eta_seconds": 22149.91459441185, "data_wait_seconds": 2.2083520889282227e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 17230, "total_steps": 72479, "tokens": 9033482240, "loss": 4.544037878513336, "l_mix": 2.8117666244506836, "l_a": 2.910799503326416, "l_c": 2.8579083681106567, "z_loss": 16.587456464767456, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15371684730052948, "tokens_per_second": 1399431.397468615, "eta_seconds": 20698.683597064017, "data_wait_seconds": 2.637505531311035e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20041, "finemath": 0.08236}}
{"step": 17240, "total_steps": 72479, "tokens": 9038725120, "loss": 3.1286436915397644, "l_mix": 1.9325141310691833, "l_a": 2.0107611268758774, "l_c": 1.970142275094986, "z_loss": 18.584587574005127, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2897893488407135, "tokens_per_second": 1443271.6591515385, "eta_seconds": 20066.315754461288, "data_wait_seconds": 1.901388168334961e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20041, "finemath": 0.08237}}
{"step": 17250, "total_steps": 72479, "tokens": 9043968000, "loss": 3.5158880949020386, "l_mix": 2.161626487970352, "l_a": 2.2875413298606873, "l_c": 2.221476674079895, "z_loss": 15.561737775802612, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20604296028614044, "tokens_per_second": 1469049.1882116029, "eta_seconds": 19710.641539001466, "data_wait_seconds": 2.130866050720215e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20046, "finemath": 0.08232}}
{"step": 17260, "total_steps": 72479, "tokens": 9049210880, "loss": 3.4693803191184998, "l_mix": 2.133312612771988, "l_a": 2.25345316529274, "l_c": 2.195024222135544, "z_loss": 15.245089411735535, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18539586663246155, "tokens_per_second": 1440366.79613805, "eta_seconds": 20099.5046189785, "data_wait_seconds": 1.5228986740112305e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20046, "finemath": 0.08233}}
{"step": 17270, "total_steps": 72479, "tokens": 9054453760, "loss": 3.500269055366516, "l_mix": 2.1517347395420074, "l_a": 2.279078871011734, "l_c": 2.2110016345977783, "z_loss": 15.10142970085144, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18189787864685059, "tokens_per_second": 1423524.6223472487, "eta_seconds": 20333.625240898135, "data_wait_seconds": 2.714991569519043e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20764, "cosmopedia_v2": 0.20046, "finemath": 0.08234}}
{"step": 17280, "total_steps": 72479, "tokens": 9059696640, "loss": 3.53363174200058, "l_mix": 2.1726514101028442, "l_a": 2.299489736557007, "l_c": 2.23185932636261, "z_loss": 15.755897641181946, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19146962463855743, "tokens_per_second": 1403603.7035801602, "eta_seconds": 20618.47887561321, "data_wait_seconds": 1.659989356994629e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20764, "cosmopedia_v2": 0.20046, "finemath": 0.08235}}
{"step": 17290, "total_steps": 72479, "tokens": 9064939520, "loss": 4.520878374576569, "l_mix": 2.7968190908432007, "l_a": 2.8999381363391876, "l_c": 2.8412336111068726, "z_loss": 17.076815605163574, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.13952405750751495, "tokens_per_second": 1469617.9291437792, "eta_seconds": 19688.743487811087, "data_wait_seconds": 1.609325408935547e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20763, "cosmopedia_v2": 0.2004, "finemath": 0.08236}}
{"step": 17300, "total_steps": 72479, "tokens": 9070182400, "loss": 4.66983163356781, "l_mix": 2.890282690525055, "l_a": 2.9878245294094086, "l_c": 2.938343793153763, "z_loss": 16.98398232460022, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21515873074531555, "tokens_per_second": 1369530.014251307, "eta_seconds": 21123.806890654563, "data_wait_seconds": 1.71661376953125e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20763, "cosmopedia_v2": 0.2004, "finemath": 0.08237}, "gates": {"gate_a2b/L1_absmean": 0.27983651, "gate_a2b/L1_absmax": 0.64627409, "gate_a2b/L1_rms": 0.3008495, "gate_a2b/L3_absmean": 0.32481995, "gate_a2b/L3_absmax": 0.50500786, "gate_a2b/L3_rms": 0.32969621, "gate_a2b/L5_absmean": 0.51096559, "gate_a2b/L5_absmax": 0.68190151, "gate_a2b/L5_rms": 0.51718676, "gate_c2b/L1_absmean": 0.01476467, "gate_c2b/L1_absmax": 0.45666707, "gate_c2b/L1_rms": 0.05383131, "gate_c2b/L3_absmean": 0.20833302, "gate_c2b/L3_absmax": 0.43878892, "gate_c2b/L3_rms": 0.21276051, "gate_c2b/L5_absmean": 0.53328419, "gate_c2b/L5_absmax": 0.7260645, "gate_c2b/L5_rms": 0.54520786, "gate_b2a/L7_absmean": 0.08186376, "gate_b2a/L7_absmax": 0.25643918, "gate_b2a/L7_rms": 0.0850799, "gate_b2a/L11_absmean": 0.1284191, "gate_b2a/L11_absmax": 0.40408811, "gate_b2a/L11_rms": 0.13093916, "gate_b2a/L14_absmean": 0.30999929, "gate_b2a/L14_absmax": 0.51178992, "gate_b2a/L14_rms": 0.31125087, "gate_b2c/L3_absmean": 0.08868724, "gate_b2c/L3_absmax": 0.58344012, "gate_b2c/L3_rms": 0.10740436, "gate_b2c/L5_absmean": 0.11900567, "gate_b2c/L5_absmax": 0.71850669, "gate_b2c/L5_rms": 0.1258989, "gate_b2c/L6_absmean": 0.17483035, "gate_b2c/L6_absmax": 0.47824743, "gate_b2c/L6_rms": 0.17835774}, "streams": {"cos_a2b/round0": 0.380781, "cos_c2b/round0": 0.287979, "cos_b2a/round0": 0.147072, "cos_b2c/round0": 0.137811, "rms_a/round0": 0.857883, "rms_b/round0": 1.648028, "rms_c/round0": 1.255848, "cos_a2b/round1": 0.413841, "cos_c2b/round1": 0.26937, "cos_b2a/round1": 0.227989, "cos_b2c/round1": 0.135199, "rms_a/round1": 1.545071, "rms_b/round1": 3.133981, "rms_c/round1": 2.068721, "cos_a2b/round2": 0.53657, "cos_c2b/round2": 0.488271, "cos_b2a/round2": 0.075714, "cos_b2c/round2": 0.136634, "rms_a/round2": 3.408822, "rms_b/round2": 6.939963, "rms_c/round2": 3.34541}, "alpha_mean": 0.4543791844660632, "alpha_hist": [0.00999, 0.06061, 0.14854, 0.18864, 0.18826, 0.17276, 0.13278, 0.08194, 0.01625, 0.00021], "alpha_by_type": {"special": {"share": 0.0014, "mean": 0.57559, "std": 0.10431}, "whitespace": {"share": 0.02739, "mean": 0.55121, "std": 0.11804}, "digit": {"share": 0.02628, "mean": 0.51374, "std": 0.12087}, "word_start": {"share": 0.58333, "mean": 0.46251, "std": 0.17047}, "word_cont": {"share": 0.25058, "mean": 0.35029, "std": 0.14185}, "punct": {"share": 0.11102, "mean": 0.6071, "std": 0.15424}}, "alpha_p05": 0.19378, "alpha_p50": 0.45665, "alpha_p95": 0.76629}
{"step": 17310, "total_steps": 72479, "tokens": 9075425280, "loss": 4.479735612869263, "l_mix": 2.770957261323929, "l_a": 2.8697134256362915, "l_c": 2.8205932676792145, "z_loss": 16.862186670303345, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18587103486061096, "tokens_per_second": 1431462.294257625, "eta_seconds": 20206.22183904648, "data_wait_seconds": 2.0295381546020508e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20768, "cosmopedia_v2": 0.2004, "finemath": 0.08232}}
{"step": 17320, "total_steps": 72479, "tokens": 9080668160, "loss": 4.5907251834869385, "l_mix": 2.839944303035736, "l_a": 2.9411821961402893, "l_c": 2.889173209667206, "z_loss": 16.74118733406067, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1618018001317978, "tokens_per_second": 1438955.654821288, "eta_seconds": 20097.354421663284, "data_wait_seconds": 1.9162893295288086e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20768, "cosmopedia_v2": 0.2004, "finemath": 0.08233}}
{"step": 17330, "total_steps": 72479, "tokens": 9085911040, "loss": 4.572245419025421, "l_mix": 2.828193426132202, "l_a": 2.932960957288742, "l_c": 2.8749203085899353, "z_loss": 16.874821424484253, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.14487089216709137, "tokens_per_second": 1413680.6683765228, "eta_seconds": 20452.96335926056, "data_wait_seconds": 2.199411392211914e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.2004, "finemath": 0.08234}}
{"step": 17340, "total_steps": 72479, "tokens": 9091153920, "loss": 4.514185309410095, "l_mix": 2.7917478680610657, "l_a": 2.8970119953155518, "l_c": 2.8388373851776123, "z_loss": 16.825371265411377, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15473948419094086, "tokens_per_second": 1459927.7994347303, "eta_seconds": 19801.4696639061, "data_wait_seconds": 3.001093864440918e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.2004, "finemath": 0.08235}}
{"step": 17350, "total_steps": 72479, "tokens": 9096396800, "loss": 3.2276968359947205, "l_mix": 1.994316890835762, "l_a": 2.073011577129364, "l_c": 2.03204146027565, "z_loss": 18.640458822250366, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.24673397839069366, "tokens_per_second": 1328377.3655782705, "eta_seconds": 21758.480610227587, "data_wait_seconds": 2.664327621459961e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.2004, "finemath": 0.08236}}
{"step": 17360, "total_steps": 72479, "tokens": 9101639680, "loss": 3.459615707397461, "l_mix": 2.126266986131668, "l_a": 2.252936840057373, "l_c": 2.186412811279297, "z_loss": 15.438071131706238, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17550493776798248, "tokens_per_second": 1392466.094412002, "eta_seconds": 20753.273913073543, "data_wait_seconds": 1.6927719116210938e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20046, "finemath": 0.08232}}
{"step": 17370, "total_steps": 72479, "tokens": 9106882560, "loss": 3.4806899428367615, "l_mix": 2.1394031941890717, "l_a": 2.2629414498806, "l_c": 2.2027534544467926, "z_loss": 15.78243112564087, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20972265303134918, "tokens_per_second": 1474799.0133450797, "eta_seconds": 19591.135558509828, "data_wait_seconds": 1.9878149032592773e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20046, "finemath": 0.08233}}
{"step": 17380, "total_steps": 72479, "tokens": 9112125440, "loss": 3.4848313331604004, "l_mix": 2.1419357657432556, "l_a": 2.2689066529273987, "l_c": 2.2022045850753784, "z_loss": 15.621195912361145, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1757931113243103, "tokens_per_second": 1379644.6875627195, "eta_seconds": 20938.53930103779, "data_wait_seconds": 2.065300941467285e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20765, "cosmopedia_v2": 0.20046, "finemath": 0.08234}}
{"step": 17390, "total_steps": 72479, "tokens": 9117368320, "loss": 3.498537451028824, "l_mix": 2.1513496041297913, "l_a": 2.276706576347351, "l_c": 2.2088806331157684, "z_loss": 15.116138935089111, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20322726666927338, "tokens_per_second": 1441271.750292315, "eta_seconds": 20039.594633102417, "data_wait_seconds": 1.6182661056518555e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20765, "cosmopedia_v2": 0.20046, "finemath": 0.08235}}
{"step": 17400, "total_steps": 72479, "tokens": 9122611200, "loss": 4.55417788028717, "l_mix": 2.8181344866752625, "l_a": 2.9151570200920105, "l_c": 2.8661616444587708, "z_loss": 16.47623860836029, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1509314775466919, "tokens_per_second": 1467902.2473899785, "eta_seconds": 19672.46715736389, "data_wait_seconds": 2.35140323638916e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20764, "cosmopedia_v2": 0.2004, "finemath": 0.08236}, "gates": {"gate_a2b/L1_absmean": 0.28086293, "gate_a2b/L1_absmax": 0.63541698, "gate_a2b/L1_rms": 0.30156443, "gate_a2b/L3_absmean": 0.32591704, "gate_a2b/L3_absmax": 0.50237, "gate_a2b/L3_rms": 0.33067596, "gate_a2b/L5_absmean": 0.51119465, "gate_a2b/L5_absmax": 0.68094122, "gate_a2b/L5_rms": 0.51739073, "gate_c2b/L1_absmean": 0.01399661, "gate_c2b/L1_absmax": 0.45249876, "gate_c2b/L1_rms": 0.05340592, "gate_c2b/L3_absmean": 0.20852385, "gate_c2b/L3_absmax": 0.43621913, "gate_c2b/L3_rms": 0.21290395, "gate_c2b/L5_absmean": 0.53241563, "gate_c2b/L5_absmax": 0.71083909, "gate_c2b/L5_rms": 0.54430234, "gate_b2a/L7_absmean": 0.08166653, "gate_b2a/L7_absmax": 0.25730336, "gate_b2a/L7_rms": 0.08480079, "gate_b2a/L11_absmean": 0.12857199, "gate_b2a/L11_absmax": 0.40408209, "gate_b2a/L11_rms": 0.13108917, "gate_b2a/L14_absmean": 0.30983451, "gate_b2a/L14_absmax": 0.51228648, "gate_b2a/L14_rms": 0.31105989, "gate_b2c/L3_absmean": 0.08868867, "gate_b2c/L3_absmax": 0.58257085, "gate_b2c/L3_rms": 0.10734805, "gate_b2c/L5_absmean": 0.11941534, "gate_b2c/L5_absmax": 0.72070473, "gate_b2c/L5_rms": 0.12631263, "gate_b2c/L6_absmean": 0.17506087, "gate_b2c/L6_absmax": 0.48639244, "gate_b2c/L6_rms": 0.17861119}, "streams": {"cos_a2b/round0": 0.379095, "cos_c2b/round0": 0.281331, "cos_b2a/round0": 0.141382, "cos_b2c/round0": 0.126858, "rms_a/round0": 0.857378, "rms_b/round0": 1.631636, "rms_c/round0": 1.279238, "cos_a2b/round1": 0.402356, "cos_c2b/round1": 0.266193, "cos_b2a/round1": 0.225785, "cos_b2c/round1": 0.127169, "rms_a/round1": 1.528858, "rms_b/round1": 3.154797, "rms_c/round1": 2.07354, "cos_a2b/round2": 0.50749, "cos_c2b/round2": 0.501964, "cos_b2a/round2": 0.073275, "cos_b2c/round2": 0.134018, "rms_a/round2": 3.321895, "rms_b/round2": 7.11321, "rms_c/round2": 3.327468}, "alpha_mean": 0.41867418898022635, "alpha_hist": [0.02293, 0.09135, 0.17014, 0.20384, 0.18506, 0.14142, 0.1104, 0.06499, 0.00977, 0.00011], "alpha_by_type": {"special": {"share": 0.00113, "mean": 0.53054, "std": 0.09683}, "whitespace": {"share": 0.02847, "mean": 0.51853, "std": 0.12187}, "digit": {"share": 0.04208, "mean": 0.43862, "std": 0.12449}, "word_start": {"share": 0.5323, "mean": 0.44656, "std": 0.1742}, "word_cont": {"share": 0.2579, "mean": 0.29533, "std": 0.13585}, "punct": {"share": 0.13811, "mean": 0.51395, "std": 0.17326}}, "alpha_p05": 0.15896, "alpha_p50": 0.42036, "alpha_p95": 0.73335}
{"step": 17410, "total_steps": 72479, "tokens": 9127854080, "loss": 4.501518607139587, "l_mix": 2.785998910665512, "l_a": 2.8849925696849823, "l_c": 2.8277917206287384, "z_loss": 16.844467520713806, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18285611271858215, "tokens_per_second": 1407161.0851266503, "eta_seconds": 20517.91808142662, "data_wait_seconds": 2.631545066833496e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20764, "cosmopedia_v2": 0.2004, "finemath": 0.08237}}
{"step": 17420, "total_steps": 72479, "tokens": 9133096960, "loss": 4.611809492111206, "l_mix": 2.854879230260849, "l_a": 2.9521137475967407, "l_c": 2.8988112807273865, "z_loss": 16.52675986289978, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18800139427185059, "tokens_per_second": 1337344.913449339, "eta_seconds": 21585.136864614487, "data_wait_seconds": 1.683831214904785e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20763, "cosmopedia_v2": 0.20046, "finemath": 0.08232}}
{"step": 17430, "total_steps": 72479, "tokens": 9138339840, "loss": 4.594316482543945, "l_mix": 2.8435014188289642, "l_a": 2.9423379600048065, "l_c": 2.8881225883960724, "z_loss": 16.769120931625366, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1429651379585266, "tokens_per_second": 1368955.977326494, "eta_seconds": 21082.87672505379, "data_wait_seconds": 1.6748905181884766e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20769, "cosmopedia_v2": 0.2004, "finemath": 0.08233}}
{"step": 17440, "total_steps": 72479, "tokens": 9143582720, "loss": 4.989352703094482, "l_mix": 3.0920515060424805, "l_a": 3.177673876285553, "l_c": 3.1412281692028046, "z_loss": 16.305299043655396, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15976345539093018, "tokens_per_second": 1454510.4232210165, "eta_seconds": 19839.175279402734, "data_wait_seconds": 1.5467405319213867e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20768, "cosmopedia_v2": 0.2004, "finemath": 0.08234}}
{"step": 17450, "total_steps": 72479, "tokens": 9148825600, "loss": 4.837974667549133, "l_mix": 2.997050791978836, "l_a": 3.087726354598999, "l_c": 3.043097734451294, "z_loss": 16.76566219329834, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.14486995339393616, "tokens_per_second": 1350825.3987577206, "eta_seconds": 21358.085492420196, "data_wait_seconds": 2.7805566787719727e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20768, "cosmopedia_v2": 0.2004, "finemath": 0.08235}}
{"step": 17460, "total_steps": 72479, "tokens": 9154068480, "loss": 5.098592042922974, "l_mix": 3.159676730632782, "l_a": 3.2517040073871613, "l_c": 3.206000328063965, "z_loss": 16.038745164871216, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18873393535614014, "tokens_per_second": 1442999.2803794725, "eta_seconds": 19990.170379304887, "data_wait_seconds": 2.1755695343017578e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.2004, "finemath": 0.08236}}
{"step": 17470, "total_steps": 72479, "tokens": 9159311360, "loss": 3.4459524154663086, "l_mix": 2.11921489238739, "l_a": 2.2396583557128906, "l_c": 2.1775562167167664, "z_loss": 15.730849266052246, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.23250612616539001, "tokens_per_second": 1395560.094125376, "eta_seconds": 20665.938151574133, "data_wait_seconds": 3.126263618469238e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20046, "finemath": 0.08231}}
{"step": 17480, "total_steps": 72479, "tokens": 9164554240, "loss": 3.5091456472873688, "l_mix": 2.1585536003112793, "l_a": 2.2771069407463074, "l_c": 2.219489574432373, "z_loss": 16.1301486492157, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2635640501976013, "tokens_per_second": 1442838.9886481464, "eta_seconds": 19985.12373096943, "data_wait_seconds": 3.400444984436035e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20046, "finemath": 0.08232}}
{"step": 17490, "total_steps": 72479, "tokens": 9169797120, "loss": 3.4644000232219696, "l_mix": 2.1294838190078735, "l_a": 2.2563214898109436, "l_c": 2.188229560852051, "z_loss": 15.508787870407104, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21551959216594696, "tokens_per_second": 1467087.7492343236, "eta_seconds": 19651.225938630105, "data_wait_seconds": 1.6242265701293945e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20766, "cosmopedia_v2": 0.20046, "finemath": 0.08233}}
{"step": 17500, "total_steps": 72479, "tokens": 9175040000, "loss": 3.4870653450489044, "l_mix": 2.143970936536789, "l_a": 2.2678622007369995, "l_c": 2.203925460577011, "z_loss": 15.5807546377182, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20505307614803314, "tokens_per_second": 1440055.149502862, "eta_seconds": 20016.476425886154, "data_wait_seconds": 2.1964311599731445e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20766, "cosmopedia_v2": 0.20046, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.28109038, "gate_a2b/L1_absmax": 0.63784528, "gate_a2b/L1_rms": 0.30187818, "gate_a2b/L3_absmean": 0.32572132, "gate_a2b/L3_absmax": 0.50461578, "gate_a2b/L3_rms": 0.33057499, "gate_a2b/L5_absmean": 0.51116085, "gate_a2b/L5_absmax": 0.67932755, "gate_a2b/L5_rms": 0.51735222, "gate_c2b/L1_absmean": 0.01436483, "gate_c2b/L1_absmax": 0.45835605, "gate_c2b/L1_rms": 0.05320522, "gate_c2b/L3_absmean": 0.20912495, "gate_c2b/L3_absmax": 0.4363721, "gate_c2b/L3_rms": 0.21354204, "gate_c2b/L5_absmean": 0.5334717, "gate_c2b/L5_absmax": 0.72670573, "gate_c2b/L5_rms": 0.54546905, "gate_b2a/L7_absmean": 0.08213776, "gate_b2a/L7_absmax": 0.26083297, "gate_b2a/L7_rms": 0.08538599, "gate_b2a/L11_absmean": 0.12863581, "gate_b2a/L11_absmax": 0.40394118, "gate_b2a/L11_rms": 0.13119037, "gate_b2a/L14_absmean": 0.31049296, "gate_b2a/L14_absmax": 0.51156217, "gate_b2a/L14_rms": 0.31174389, "gate_b2c/L3_absmean": 0.08887914, "gate_b2c/L3_absmax": 0.58084822, "gate_b2c/L3_rms": 0.10749627, "gate_b2c/L5_absmean": 0.11904913, "gate_b2c/L5_absmax": 0.72698885, "gate_b2c/L5_rms": 0.1261148, "gate_b2c/L6_absmean": 0.17538115, "gate_b2c/L6_absmax": 0.48931497, "gate_b2c/L6_rms": 0.17897305}, "streams": {"cos_a2b/round0": 0.368535, "cos_c2b/round0": 0.245354, "cos_b2a/round0": 0.127166, "cos_b2c/round0": 0.096199, "rms_a/round0": 0.889807, "rms_b/round0": 1.641476, "rms_c/round0": 1.273315, "cos_a2b/round1": 0.404362, "cos_c2b/round1": 0.270754, "cos_b2a/round1": 0.219336, "cos_b2c/round1": 0.135098, "rms_a/round1": 1.584432, "rms_b/round1": 3.262448, "rms_c/round1": 2.131728, "cos_a2b/round2": 0.496743, "cos_c2b/round2": 0.475441, "cos_b2a/round2": 0.040431, "cos_b2c/round2": 0.112338, "rms_a/round2": 3.397381, "rms_b/round2": 7.161435, "rms_c/round2": 3.452658}, "alpha_mean": 0.4222447007841055, "alpha_hist": [0.05429, 0.14548, 0.17052, 0.14503, 0.11549, 0.10435, 0.10501, 0.09816, 0.0556, 0.00606], "alpha_by_type": {"special": {"share": 0.00107, "mean": 0.66375, "std": 0.09347}, "whitespace": {"share": 0.03914, "mean": 0.6438, "std": 0.17793}, "digit": {"share": 0.00844, "mean": 0.36093, "std": 0.15606}, "word_start": {"share": 0.5681, "mean": 0.44034, "std": 0.21555}, "word_cont": {"share": 0.26288, "mean": 0.25905, "std": 0.15785}, "punct": {"share": 0.12038, "mean": 0.62334, "std": 0.19008}}, "alpha_p05": 0.09501, "alpha_p50": 0.37731, "alpha_p95": 0.81287}
{"step": 17510, "total_steps": 72479, "tokens": 9180282880, "loss": 4.456193029880524, "l_mix": 2.756756842136383, "l_a": 2.8548682034015656, "l_c": 2.804301381111145, "z_loss": 16.85260033607483, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17021536827087402, "tokens_per_second": 1427452.030622839, "eta_seconds": 20189.531034135816, "data_wait_seconds": 2.148747444152832e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20765, "cosmopedia_v2": 0.2004, "finemath": 0.08235}}
{"step": 17520, "total_steps": 72479, "tokens": 9185525760, "loss": 4.546557128429413, "l_mix": 2.8129156827926636, "l_a": 2.9142847657203674, "l_c": 2.8589735329151154, "z_loss": 16.63736081123352, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17887401580810547, "tokens_per_second": 1466192.6197698016, "eta_seconds": 19652.49572496414, "data_wait_seconds": 1.576542854309082e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20765, "cosmopedia_v2": 0.2004, "finemath": 0.08236}}
{"step": 17530, "total_steps": 72479, "tokens": 9190768640, "loss": 4.482513427734375, "l_mix": 2.7730657160282135, "l_a": 2.8763587474823, "l_c": 2.81610643863678, "z_loss": 17.079400300979614, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.182540163397789, "tokens_per_second": 1420952.5512529386, "eta_seconds": 20274.499163675307, "data_wait_seconds": 1.7434358596801758e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.2004, "finemath": 0.08237}}
{"step": 17540, "total_steps": 72479, "tokens": 9196011520, "loss": 3.4489444494247437, "l_mix": 2.1197363436222076, "l_a": 2.2432596683502197, "l_c": 2.1820962131023407, "z_loss": 16.01243031024933, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22480818629264832, "tokens_per_second": 1339468.2577716357, "eta_seconds": 21503.949992752077, "data_wait_seconds": 1.9371509552001953e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20046, "finemath": 0.08233}}
{"step": 17550, "total_steps": 72479, "tokens": 9201254400, "loss": 3.479579597711563, "l_mix": 2.1395913660526276, "l_a": 2.261110931634903, "l_c": 2.200254440307617, "z_loss": 15.785071730613708, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20378842949867249, "tokens_per_second": 1461370.974470272, "eta_seconds": 19706.574206757545, "data_wait_seconds": 2.60770320892334e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20764, "cosmopedia_v2": 0.20046, "finemath": 0.08234}}
{"step": 17560, "total_steps": 72479, "tokens": 9206497280, "loss": 3.4720264077186584, "l_mix": 2.1344816982746124, "l_a": 2.2569994628429413, "l_c": 2.196311503648758, "z_loss": 15.513697743415833, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22618255019187927, "tokens_per_second": 1439623.0856223633, "eta_seconds": 20000.63277642727, "data_wait_seconds": 1.6480684280395508e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20763, "cosmopedia_v2": 0.20046, "finemath": 0.08235}}
{"step": 17570, "total_steps": 72479, "tokens": 9211740160, "loss": 4.857525944709778, "l_mix": 3.0084036588668823, "l_a": 3.102242201566696, "l_c": 3.05594402551651, "z_loss": 16.663854598999023, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18567706644535065, "tokens_per_second": 1433461.0303183906, "eta_seconds": 20082.95250663757, "data_wait_seconds": 2.8759241104125977e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.2004, "finemath": 0.08236}}
{"step": 17580, "total_steps": 72479, "tokens": 9216983040, "loss": 4.909419536590576, "l_mix": 3.0418058931827545, "l_a": 3.13057142496109, "l_c": 3.0893435180187225, "z_loss": 16.391462087631226, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16223542392253876, "tokens_per_second": 1261804.6953281702, "eta_seconds": 22810.88905324936, "data_wait_seconds": 3.075599670410156e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.2004, "finemath": 0.08237}}
{"step": 17590, "total_steps": 72479, "tokens": 9222225920, "loss": 3.4173262119293213, "l_mix": 2.1004957258701324, "l_a": 2.2235476970672607, "l_c": 2.1607457399368286, "z_loss": 15.424564123153687, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1995578557252884, "tokens_per_second": 1413187.8111571765, "eta_seconds": 20363.637306237222, "data_wait_seconds": 2.276897430419922e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20045, "finemath": 0.08232}}
{"step": 17600, "total_steps": 72479, "tokens": 9227468800, "loss": 3.4510285556316376, "l_mix": 2.122365266084671, "l_a": 2.2461987137794495, "l_c": 2.1774182319641113, "z_loss": 15.782043695449829, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2000165730714798, "tokens_per_second": 1377013.0026800435, "eta_seconds": 20894.792638850213, "data_wait_seconds": 1.77919864654541e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20045, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.28144154, "gate_a2b/L1_absmax": 0.64670974, "gate_a2b/L1_rms": 0.30215871, "gate_a2b/L3_absmean": 0.3268829, "gate_a2b/L3_absmax": 0.50152075, "gate_a2b/L3_rms": 0.33161038, "gate_a2b/L5_absmean": 0.5109176, "gate_a2b/L5_absmax": 0.67973602, "gate_a2b/L5_rms": 0.51704645, "gate_c2b/L1_absmean": 0.01492668, "gate_c2b/L1_absmax": 0.45867658, "gate_c2b/L1_rms": 0.05328135, "gate_c2b/L3_absmean": 0.20945434, "gate_c2b/L3_absmax": 0.42945051, "gate_c2b/L3_rms": 0.2137823, "gate_c2b/L5_absmean": 0.53310364, "gate_c2b/L5_absmax": 0.72385103, "gate_c2b/L5_rms": 0.54495126, "gate_b2a/L7_absmean": 0.08270475, "gate_b2a/L7_absmax": 0.25953084, "gate_b2a/L7_rms": 0.08590791, "gate_b2a/L11_absmean": 0.12925541, "gate_b2a/L11_absmax": 0.40508193, "gate_b2a/L11_rms": 0.13179174, "gate_b2a/L14_absmean": 0.31076971, "gate_b2a/L14_absmax": 0.50921559, "gate_b2a/L14_rms": 0.31201226, "gate_b2c/L3_absmean": 0.08949222, "gate_b2c/L3_absmax": 0.58461046, "gate_b2c/L3_rms": 0.10801902, "gate_b2c/L5_absmean": 0.12020794, "gate_b2c/L5_absmax": 0.72597927, "gate_b2c/L5_rms": 0.12724875, "gate_b2c/L6_absmean": 0.17574567, "gate_b2c/L6_absmax": 0.48905256, "gate_b2c/L6_rms": 0.17943081}, "streams": {"cos_a2b/round0": 0.371312, "cos_c2b/round0": 0.253073, "cos_b2a/round0": 0.128235, "cos_b2c/round0": 0.098156, "rms_a/round0": 0.883989, "rms_b/round0": 1.632966, "rms_c/round0": 1.276792, "cos_a2b/round1": 0.406383, "cos_c2b/round1": 0.275509, "cos_b2a/round1": 0.223217, "cos_b2c/round1": 0.13815, "rms_a/round1": 1.571517, "rms_b/round1": 3.216862, "rms_c/round1": 2.131917, "cos_a2b/round2": 0.506028, "cos_c2b/round2": 0.468909, "cos_b2a/round2": 0.044872, "cos_b2c/round2": 0.114012, "rms_a/round2": 3.415569, "rms_b/round2": 7.117914, "rms_c/round2": 3.440361}, "alpha_mean": 0.44171466744887766, "alpha_hist": [0.05219, 0.12419, 0.16049, 0.14297, 0.12013, 0.10733, 0.10921, 0.10767, 0.06653, 0.00929], "alpha_by_type": {"special": {"share": 0.00116, "mean": 0.63751, "std": 0.09531}, "whitespace": {"share": 0.0352, "mean": 0.65261, "std": 0.18786}, "digit": {"share": 0.0092, "mean": 0.49753, "std": 0.17271}, "word_start": {"share": 0.57942, "mean": 0.46302, "std": 0.21657}, "word_cont": {"share": 0.2578, "mean": 0.26756, "std": 0.16436}, "punct": {"share": 0.11722, "mean": 0.64976, "std": 0.18477}}, "alpha_p05": 0.08632, "alpha_p50": 0.42274, "alpha_p95": 0.83468}
{"step": 17610, "total_steps": 72479, "tokens": 9232711680, "loss": 3.49409419298172, "l_mix": 2.1493498980998993, "l_a": 2.2714954018592834, "l_c": 2.2057681679725647, "z_loss": 15.652241587638855, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.263256698846817, "tokens_per_second": 1420993.9625209214, "eta_seconds": 20244.391623568536, "data_wait_seconds": 1.7434358596801758e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20767, "cosmopedia_v2": 0.20045, "finemath": 0.08234}}
{"step": 17620, "total_steps": 72479, "tokens": 9237954560, "loss": 4.475242912769318, "l_mix": 2.768428474664688, "l_a": 2.871334493160248, "l_c": 2.812477022409439, "z_loss": 16.710286378860474, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1589459925889969, "tokens_per_second": 1443427.5939688338, "eta_seconds": 19926.122731876374, "data_wait_seconds": 1.5616416931152344e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20766, "cosmopedia_v2": 0.2004, "finemath": 0.08235}}
{"step": 17630, "total_steps": 72479, "tokens": 9243197440, "loss": 4.642269968986511, "l_mix": 2.8737531900405884, "l_a": 2.9711746871471405, "l_c": 2.918299287557602, "z_loss": 16.744949102401733, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18588408827781677, "tokens_per_second": 1422695.7460824847, "eta_seconds": 20212.80557785034, "data_wait_seconds": 2.041459083557129e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20766, "cosmopedia_v2": 0.2004, "finemath": 0.08236}}
{"step": 17640, "total_steps": 72479, "tokens": 9248440320, "loss": 4.546061336994171, "l_mix": 2.8127076029777527, "l_a": 2.914682924747467, "l_c": 2.8576608896255493, "z_loss": 16.505898237228394, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18867269158363342, "tokens_per_second": 1280367.2836791696, "eta_seconds": 22455.61097857952, "data_wait_seconds": 0.05724552273750305, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.2004, "finemath": 0.08237}}
{"step": 17650, "total_steps": 72479, "tokens": 9253683200, "loss": 3.465275913476944, "l_mix": 2.1297221779823303, "l_a": 2.2582241892814636, "l_c": 2.1884894371032715, "z_loss": 15.39583933353424, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21396249532699585, "tokens_per_second": 1429278.008760556, "eta_seconds": 20112.383018422126, "data_wait_seconds": 2.0116567611694336e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20045, "finemath": 0.08232}}
{"step": 17660, "total_steps": 72479, "tokens": 9258926080, "loss": 3.448850780725479, "l_mix": 2.1206798553466797, "l_a": 2.240443527698517, "l_c": 2.1815838515758514, "z_loss": 15.626607537269592, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19423474371433258, "tokens_per_second": 1425533.0267369861, "eta_seconds": 20161.541916561127, "data_wait_seconds": 2.7954578399658203e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20764, "cosmopedia_v2": 0.20045, "finemath": 0.08233}}
{"step": 17670, "total_steps": 72479, "tokens": 9264168960, "loss": 3.4625288546085358, "l_mix": 2.127891093492508, "l_a": 2.2553273141384125, "l_c": 2.188287764787674, "z_loss": 15.532409906387329, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21107089519500732, "tokens_per_second": 1408572.9593696014, "eta_seconds": 20400.576910734177, "data_wait_seconds": 2.041459083557129e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20764, "cosmopedia_v2": 0.20045, "finemath": 0.08234}}
{"step": 17680, "total_steps": 72479, "tokens": 9269411840, "loss": 3.52485591173172, "l_mix": 2.1676396429538727, "l_a": 2.2912970185279846, "l_c": 2.2275601029396057, "z_loss": 15.590221881866455, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.23574161529541016, "tokens_per_second": 1473350.1353451083, "eta_seconds": 19500.088555169106, "data_wait_seconds": 1.621246337890625e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20764, "cosmopedia_v2": 0.20045, "finemath": 0.08235}}
{"step": 17690, "total_steps": 72479, "tokens": 9274654720, "loss": 3.487109810113907, "l_mix": 2.143893390893936, "l_a": 2.267584443092346, "l_c": 2.204595059156418, "z_loss": 15.62541389465332, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18249265849590302, "tokens_per_second": 1327058.0999396343, "eta_seconds": 21645.78569190502, "data_wait_seconds": 1.8775463104248047e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20763, "cosmopedia_v2": 0.20045, "finemath": 0.08236}}
{"step": 17700, "total_steps": 72479, "tokens": 9279897600, "loss": 4.861915946006775, "l_mix": 3.012088567018509, "l_a": 3.10207599401474, "l_c": 3.0584560930728912, "z_loss": 16.67666244506836, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22920867800712585, "tokens_per_second": 1402474.5253636972, "eta_seconds": 20478.070604920387, "data_wait_seconds": 1.9341707229614258e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20768, "cosmopedia_v2": 0.20045, "finemath": 0.08232}, "gates": {"gate_a2b/L1_absmean": 0.28136054, "gate_a2b/L1_absmax": 0.641734, "gate_a2b/L1_rms": 0.30199924, "gate_a2b/L3_absmean": 0.32631055, "gate_a2b/L3_absmax": 0.50278395, "gate_a2b/L3_rms": 0.33108613, "gate_a2b/L5_absmean": 0.51116562, "gate_a2b/L5_absmax": 0.67874247, "gate_a2b/L5_rms": 0.51727307, "gate_c2b/L1_absmean": 0.01443469, "gate_c2b/L1_absmax": 0.4622882, "gate_c2b/L1_rms": 0.05352413, "gate_c2b/L3_absmean": 0.20951489, "gate_c2b/L3_absmax": 0.42931977, "gate_c2b/L3_rms": 0.21383238, "gate_c2b/L5_absmean": 0.53356045, "gate_c2b/L5_absmax": 0.71696544, "gate_c2b/L5_rms": 0.54542524, "gate_b2a/L7_absmean": 0.08269106, "gate_b2a/L7_absmax": 0.26001203, "gate_b2a/L7_rms": 0.08580648, "gate_b2a/L11_absmean": 0.12900111, "gate_b2a/L11_absmax": 0.40245697, "gate_b2a/L11_rms": 0.13152292, "gate_b2a/L14_absmean": 0.31169555, "gate_b2a/L14_absmax": 0.51046807, "gate_b2a/L14_rms": 0.31288439, "gate_b2c/L3_absmean": 0.08989002, "gate_b2c/L3_absmax": 0.58503354, "gate_b2c/L3_rms": 0.1083568, "gate_b2c/L5_absmean": 0.11979458, "gate_b2c/L5_absmax": 0.72345811, "gate_b2c/L5_rms": 0.12673038, "gate_b2c/L6_absmean": 0.17571387, "gate_b2c/L6_absmax": 0.48726249, "gate_b2c/L6_rms": 0.17937252}, "streams": {"cos_a2b/round0": 0.385165, "cos_c2b/round0": 0.301336, "cos_b2a/round0": 0.150093, "cos_b2c/round0": 0.15112, "rms_a/round0": 0.846119, "rms_b/round0": 1.611425, "rms_c/round0": 1.240137, "cos_a2b/round1": 0.417073, "cos_c2b/round1": 0.289044, "cos_b2a/round1": 0.237363, "cos_b2c/round1": 0.138227, "rms_a/round1": 1.510075, "rms_b/round1": 3.061205, "rms_c/round1": 2.029701, "cos_a2b/round2": 0.545605, "cos_c2b/round2": 0.498608, "cos_b2a/round2": 0.089271, "cos_b2c/round2": 0.151506, "rms_a/round2": 3.37593, "rms_b/round2": 6.851787, "rms_c/round2": 3.309926}, "alpha_mean": 0.4633013581057668, "alpha_hist": [0.00574, 0.03459, 0.1162, 0.19119, 0.23888, 0.21556, 0.13214, 0.05954, 0.00609, 8e-05], "alpha_by_type": {"special": {"share": 0.0007, "mean": 0.54555, "std": 0.08402}, "whitespace": {"share": 0.05472, "mean": 0.52632, "std": 0.10672}, "digit": {"share": 0.03926, "mean": 0.46341, "std": 0.12363}, "word_start": {"share": 0.59958, "mean": 0.48014, "std": 0.14374}, "word_cont": {"share": 0.20967, "mean": 0.34873, "std": 0.12719}, "punct": {"share": 0.09607, "mean": 0.57174, "std": 0.14771}}, "alpha_p05": 0.22989, "alpha_p50": 0.46599, "alpha_p95": 0.71482}
{"step": 17710, "total_steps": 72479, "tokens": 9285140480, "loss": 3.4394776225090027, "l_mix": 2.1141943633556366, "l_a": 2.240483343601227, "l_c": 2.171793609857559, "z_loss": 16.001073002815247, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21555982530117035, "tokens_per_second": 1325152.4198732676, "eta_seconds": 21669.00127213001, "data_wait_seconds": 1.9669532775878906e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20768, "cosmopedia_v2": 0.20045, "finemath": 0.08233}}
{"step": 17720, "total_steps": 72479, "tokens": 9290383360, "loss": 3.48084032535553, "l_mix": 2.14007431268692, "l_a": 2.2646886706352234, "l_c": 2.1991936564445496, "z_loss": 16.012425541877747, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2721133530139923, "tokens_per_second": 1358402.5796601411, "eta_seconds": 21134.74092428684, "data_wait_seconds": 3.254413604736328e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20767, "cosmopedia_v2": 0.20045, "finemath": 0.08234}}
{"step": 17730, "total_steps": 72479, "tokens": 9295626240, "loss": 4.690781950950623, "l_mix": 2.9034152925014496, "l_a": 3.002841830253601, "l_c": 2.9494435787200928, "z_loss": 16.80988335609436, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18289311230182648, "tokens_per_second": 1439731.289372126, "eta_seconds": 19937.222955346107, "data_wait_seconds": 2.810359001159668e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20767, "cosmopedia_v2": 0.20039, "finemath": 0.08235}}
{"step": 17740, "total_steps": 72479, "tokens": 9300869120, "loss": 4.736538231372833, "l_mix": 2.9317162930965424, "l_a": 3.0334786474704742, "l_c": 2.977069139480591, "z_loss": 16.57571244239807, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15609624981880188, "tokens_per_second": 1455665.5277105742, "eta_seconds": 19715.381236743924, "data_wait_seconds": 1.958012580871582e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20039, "finemath": 0.08236}}
{"step": 17750, "total_steps": 72479, "tokens": 9306112000, "loss": 4.680328726768494, "l_mix": 2.8967796862125397, "l_a": 2.993621349334717, "l_c": 2.9459310173988342, "z_loss": 16.83406352996826, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2363058626651764, "tokens_per_second": 1432752.7166320048, "eta_seconds": 20027.013467788696, "data_wait_seconds": 2.244114875793457e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20039, "finemath": 0.08237}}
{"step": 17760, "total_steps": 72479, "tokens": 9311354880, "loss": 3.517855703830719, "l_mix": 2.1617947220802307, "l_a": 2.2897612154483795, "l_c": 2.225242257118225, "z_loss": 15.598376393318176, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21400879323482513, "tokens_per_second": 1312859.7374069486, "eta_seconds": 21851.926945877076, "data_wait_seconds": 3.0606985092163086e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20045, "finemath": 0.08232}}
{"step": 17770, "total_steps": 72479, "tokens": 9316597760, "loss": 3.4752745032310486, "l_mix": 2.137088268995285, "l_a": 2.2576864361763, "l_c": 2.1977197229862213, "z_loss": 15.64361298084259, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.32702913880348206, "tokens_per_second": 1468967.442743107, "eta_seconds": 19526.145615887643, "data_wait_seconds": 2.4527311325073242e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20045, "finemath": 0.08233}}
{"step": 17780, "total_steps": 72479, "tokens": 9321840640, "loss": 3.515179932117462, "l_mix": 2.1602362394332886, "l_a": 2.2947697937488556, "l_c": 2.2167199552059174, "z_loss": 14.967485427856445, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.26400721073150635, "tokens_per_second": 1377255.6036311565, "eta_seconds": 20822.59040107727, "data_wait_seconds": 1.677870750427246e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20045, "finemath": 0.08234}}
{"step": 17790, "total_steps": 72479, "tokens": 9327083520, "loss": 3.496513068675995, "l_mix": 2.149594396352768, "l_a": 2.275286912918091, "l_c": 2.209213614463806, "z_loss": 15.683991074562073, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20227856934070587, "tokens_per_second": 1404121.8141342937, "eta_seconds": 20420.44083595276, "data_wait_seconds": 2.4974346160888672e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20764, "cosmopedia_v2": 0.20045, "finemath": 0.08235}}
{"step": 17800, "total_steps": 72479, "tokens": 9332326400, "loss": 3.4890196323394775, "l_mix": 2.145233601331711, "l_a": 2.273223400115967, "l_c": 2.2009157240390778, "z_loss": 15.442485451698303, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.23388133943080902, "tokens_per_second": 1389572.5252202535, "eta_seconds": 20630.47666220665, "data_wait_seconds": 2.1159648895263672e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20764, "cosmopedia_v2": 0.20045, "finemath": 0.08236}, "gates": {"gate_a2b/L1_absmean": 0.28143507, "gate_a2b/L1_absmax": 0.63951623, "gate_a2b/L1_rms": 0.30190659, "gate_a2b/L3_absmean": 0.32572195, "gate_a2b/L3_absmax": 0.49239919, "gate_a2b/L3_rms": 0.33047551, "gate_a2b/L5_absmean": 0.50949353, "gate_a2b/L5_absmax": 0.68905962, "gate_a2b/L5_rms": 0.5156517, "gate_c2b/L1_absmean": 0.01582123, "gate_c2b/L1_absmax": 0.46529663, "gate_c2b/L1_rms": 0.05475768, "gate_c2b/L3_absmean": 0.21001692, "gate_c2b/L3_absmax": 0.42431572, "gate_c2b/L3_rms": 0.21438625, "gate_c2b/L5_absmean": 0.53053248, "gate_c2b/L5_absmax": 0.71588737, "gate_c2b/L5_rms": 0.54251397, "gate_b2a/L7_absmean": 0.08265005, "gate_b2a/L7_absmax": 0.27244705, "gate_b2a/L7_rms": 0.08638792, "gate_b2a/L11_absmean": 0.12892932, "gate_b2a/L11_absmax": 0.41252381, "gate_b2a/L11_rms": 0.13161583, "gate_b2a/L14_absmean": 0.30861217, "gate_b2a/L14_absmax": 0.52439392, "gate_b2a/L14_rms": 0.30997065, "gate_b2c/L3_absmean": 0.08853094, "gate_b2c/L3_absmax": 0.59222513, "gate_b2c/L3_rms": 0.10742848, "gate_b2c/L5_absmean": 0.11921221, "gate_b2c/L5_absmax": 0.73328894, "gate_b2c/L5_rms": 0.12654893, "gate_b2c/L6_absmean": 0.17426828, "gate_b2c/L6_absmax": 0.47603071, "gate_b2c/L6_rms": 0.17803584}, "streams": {"cos_a2b/round0": 0.370607, "cos_c2b/round0": 0.249749, "cos_b2a/round0": 0.125023, "cos_b2c/round0": 0.099253, "rms_a/round0": 0.908887, "rms_b/round0": 1.674538, "rms_c/round0": 1.279749, "cos_a2b/round1": 0.407745, "cos_c2b/round1": 0.267402, "cos_b2a/round1": 0.222575, "cos_b2c/round1": 0.137455, "rms_a/round1": 1.604306, "rms_b/round1": 3.306822, "rms_c/round1": 2.165851, "cos_a2b/round2": 0.504611, "cos_c2b/round2": 0.468506, "cos_b2a/round2": 0.040046, "cos_b2c/round2": 0.110292, "rms_a/round2": 3.466251, "rms_b/round2": 7.275733, "rms_c/round2": 3.458699}, "alpha_mean": 0.43779340491217766, "alpha_hist": [0.04663, 0.12949, 0.16187, 0.14272, 0.12051, 0.11407, 0.11563, 0.10777, 0.05618, 0.00513], "alpha_by_type": {"special": {"share": 0.00111, "mean": 0.62059, "std": 0.12986}, "whitespace": {"share": 0.04189, "mean": 0.63974, "std": 0.166}, "digit": {"share": 0.01036, "mean": 0.45689, "std": 0.19394}, "word_start": {"share": 0.55792, "mean": 0.45229, "std": 0.21146}, "word_cont": {"share": 0.26332, "mean": 0.27264, "std": 0.15934}, "punct": {"share": 0.1254, "mean": 0.64944, "std": 0.17447}}, "alpha_p05": 0.09138, "alpha_p50": 0.43614, "alpha_p95": 0.80867}
{"step": 17810, "total_steps": 72479, "tokens": 9337569280, "loss": 4.2579856514930725, "l_mix": 2.6344028413295746, "l_a": 2.750858396291733, "l_c": 2.654901623725891, "z_loss": 18.548508524894714, "lr": 0.0023, "wd": 0.1, "grad_norm": 1.5375622510910034, "tokens_per_second": 1391738.6882568053, "eta_seconds": 20594.599340987206, "data_wait_seconds": 1.996755599975586e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20764, "cosmopedia_v2": 0.20045, "finemath": 0.08237}}
{"step": 17820, "total_steps": 72479, "tokens": 9342812160, "loss": 4.783064424991608, "l_mix": 2.9605312645435333, "l_a": 3.06318399310112, "l_c": 3.0063362419605255, "z_loss": 16.768807411193848, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16109983623027802, "tokens_per_second": 1383341.048183983, "eta_seconds": 20715.829859614372, "data_wait_seconds": 2.98917293548584e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20763, "cosmopedia_v2": 0.20045, "finemath": 0.08232}}
{"step": 17830, "total_steps": 72479, "tokens": 9348055040, "loss": 4.785577178001404, "l_mix": 2.96129047870636, "l_a": 3.0597930550575256, "l_c": 3.0155972242355347, "z_loss": 16.696491241455078, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1912594437599182, "tokens_per_second": 1474249.086566535, "eta_seconds": 19434.853426790236, "data_wait_seconds": 1.58846378326416e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20763, "cosmopedia_v2": 0.20045, "finemath": 0.08233}}
{"step": 17840, "total_steps": 72479, "tokens": 9353297920, "loss": 4.800368547439575, "l_mix": 2.971101313829422, "l_a": 3.0738610923290253, "l_c": 3.0180916786193848, "z_loss": 16.8130521774292, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19205299019813538, "tokens_per_second": 1422468.3423619336, "eta_seconds": 20138.63590414524, "data_wait_seconds": 2.4259090423583984e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20762, "cosmopedia_v2": 0.20045, "finemath": 0.08234}}
{"step": 17850, "total_steps": 72479, "tokens": 9358540800, "loss": 4.711688816547394, "l_mix": 2.9157210886478424, "l_a": 3.017926901578903, "l_c": 2.963114619255066, "z_loss": 16.55409836769104, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1559925228357315, "tokens_per_second": 1436906.9304849552, "eta_seconds": 19932.626494002343, "data_wait_seconds": 2.0384788513183594e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20768, "cosmopedia_v2": 0.20039, "finemath": 0.08235}}
{"step": 17860, "total_steps": 72479, "tokens": 9363783680, "loss": 4.844270050525665, "l_mix": 2.999199151992798, "l_a": 3.093078136444092, "l_c": 3.0516814291477203, "z_loss": 16.428598642349243, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2185899168252945, "tokens_per_second": 1387529.78870145, "eta_seconds": 20638.177648639677, "data_wait_seconds": 1.7642974853515625e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20039, "finemath": 0.08236}}
{"step": 17870, "total_steps": 72479, "tokens": 9369026560, "loss": 3.4878748655319214, "l_mix": 2.144284874200821, "l_a": 2.27207088470459, "l_c": 2.2012756764888763, "z_loss": 15.859856724739075, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.24203947186470032, "tokens_per_second": 1462087.5567438889, "eta_seconds": 19582.16747002602, "data_wait_seconds": 2.2292137145996094e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20045, "finemath": 0.08232}}
{"step": 17880, "total_steps": 72479, "tokens": 9374269440, "loss": 3.485872834920883, "l_mix": 2.1433395743370056, "l_a": 2.266567200422287, "l_c": 2.203330785036087, "z_loss": 15.638662219047546, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2515442669391632, "tokens_per_second": 1448510.5434854208, "eta_seconds": 19762.093303871156, "data_wait_seconds": 2.4557113647460938e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20045, "finemath": 0.08233}}
{"step": 17890, "total_steps": 72479, "tokens": 9379512320, "loss": 3.505518078804016, "l_mix": 2.155582308769226, "l_a": 2.278439939022064, "l_c": 2.2161355316638947, "z_loss": 15.631023645401001, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2524171769618988, "tokens_per_second": 1266717.7355036086, "eta_seconds": 22594.108245134354, "data_wait_seconds": 2.7507543563842773e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20045, "finemath": 0.08234}}
{"step": 17900, "total_steps": 72479, "tokens": 9384755200, "loss": 3.507380247116089, "l_mix": 2.1562949419021606, "l_a": 2.2831137478351593, "l_c": 2.2152516841888428, "z_loss": 15.756779313087463, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.24442963302135468, "tokens_per_second": 1343020.9305992266, "eta_seconds": 21306.529258060455, "data_wait_seconds": 1.84476375579834e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20765, "cosmopedia_v2": 0.20045, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.28083214, "gate_a2b/L1_absmax": 0.64307612, "gate_a2b/L1_rms": 0.30097625, "gate_a2b/L3_absmean": 0.32500324, "gate_a2b/L3_absmax": 0.49227786, "gate_a2b/L3_rms": 0.32964984, "gate_a2b/L5_absmean": 0.50666487, "gate_a2b/L5_absmax": 0.69096226, "gate_a2b/L5_rms": 0.51278168, "gate_c2b/L1_absmean": 0.01403805, "gate_c2b/L1_absmax": 0.46297881, "gate_c2b/L1_rms": 0.05419912, "gate_c2b/L3_absmean": 0.21003948, "gate_c2b/L3_absmax": 0.43285421, "gate_c2b/L3_rms": 0.21443482, "gate_c2b/L5_absmean": 0.5296647, "gate_c2b/L5_absmax": 0.73273599, "gate_c2b/L5_rms": 0.54160994, "gate_b2a/L7_absmean": 0.08297601, "gate_b2a/L7_absmax": 0.27403551, "gate_b2a/L7_rms": 0.0863988, "gate_b2a/L11_absmean": 0.12814161, "gate_b2a/L11_absmax": 0.40678731, "gate_b2a/L11_rms": 0.1308234, "gate_b2a/L14_absmean": 0.30872729, "gate_b2a/L14_absmax": 0.51352507, "gate_b2a/L14_rms": 0.31013495, "gate_b2c/L3_absmean": 0.08919284, "gate_b2c/L3_absmax": 0.59670377, "gate_b2c/L3_rms": 0.10768645, "gate_b2c/L5_absmean": 0.11998695, "gate_b2c/L5_absmax": 0.72336709, "gate_b2c/L5_rms": 0.12702505, "gate_b2c/L6_absmean": 0.17472355, "gate_b2c/L6_absmax": 0.48086846, "gate_b2c/L6_rms": 0.17840835}, "streams": {"cos_a2b/round0": 0.363607, "cos_c2b/round0": 0.248518, "cos_b2a/round0": 0.124369, "cos_b2c/round0": 0.094444, "rms_a/round0": 0.901398, "rms_b/round0": 1.658836, "rms_c/round0": 1.28381, "cos_a2b/round1": 0.40439, "cos_c2b/round1": 0.270527, "cos_b2a/round1": 0.218913, "cos_b2c/round1": 0.134636, "rms_a/round1": 1.606998, "rms_b/round1": 3.27776, "rms_c/round1": 2.167201, "cos_a2b/round2": 0.499307, "cos_c2b/round2": 0.474589, "cos_b2a/round2": 0.038462, "cos_b2c/round2": 0.110238, "rms_a/round2": 3.452882, "rms_b/round2": 7.261031, "rms_c/round2": 3.447181}, "alpha_mean": 0.42331904279605226, "alpha_hist": [0.05989, 0.14247, 0.1665, 0.14711, 0.11609, 0.10008, 0.10338, 0.09305, 0.0649, 0.00653], "alpha_by_type": {"special": {"share": 0.00108, "mean": 0.6583, "std": 0.08604}, "whitespace": {"share": 0.04396, "mean": 0.57098, "std": 0.18249}, "digit": {"share": 0.00981, "mean": 0.28374, "std": 0.1305}, "word_start": {"share": 0.54997, "mean": 0.45417, "std": 0.21684}, "word_cont": {"share": 0.25175, "mean": 0.23508, "std": 0.14789}, "punct": {"share": 0.14342, "mean": 0.59795, "std": 0.21076}}, "alpha_p05": 0.09774, "alpha_p50": 0.38491, "alpha_p95": 0.83224}
{"step": 17910, "total_steps": 72479, "tokens": 9389998080, "loss": 3.488344728946686, "l_mix": 2.144575983285904, "l_a": 2.2714053988456726, "l_c": 2.2026880383491516, "z_loss": 15.406743168830872, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22688648104667664, "tokens_per_second": 1463668.9496412685, "eta_seconds": 19546.682246017455, "data_wait_seconds": 3.400444984436035e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20765, "cosmopedia_v2": 0.20045, "finemath": 0.08236}}
{"step": 17920, "total_steps": 72479, "tokens": 9395240960, "loss": 3.030835658311844, "l_mix": 1.8714429289102554, "l_a": 1.952099785208702, "l_c": 1.9063676744699478, "z_loss": 18.523839235305786, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2204935997724533, "tokens_per_second": 1361139.185038447, "eta_seconds": 21015.212335681917, "data_wait_seconds": 3.3855438232421875e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20765, "cosmopedia_v2": 0.20045, "finemath": 0.08237}}
{"step": 17930, "total_steps": 72479, "tokens": 9400483840, "loss": 4.705975830554962, "l_mix": 2.913062632083893, "l_a": 3.0152591168880463, "l_c": 2.95554319024086, "z_loss": 16.724320888519287, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19116276502609253, "tokens_per_second": 1338724.8993574432, "eta_seconds": 21363.153942775727, "data_wait_seconds": 2.446770668029785e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20764, "cosmopedia_v2": 0.20045, "finemath": 0.08232}}
{"step": 17940, "total_steps": 72479, "tokens": 9405726720, "loss": 4.771013379096985, "l_mix": 2.9528696835041046, "l_a": 3.0536347031593323, "l_c": 3.0011276304721832, "z_loss": 17.1497220993042, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20054085552692413, "tokens_per_second": 1424176.2438982562, "eta_seconds": 20077.67181520462, "data_wait_seconds": 1.570582389831543e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20764, "cosmopedia_v2": 0.20045, "finemath": 0.08233}}
{"step": 17950, "total_steps": 72479, "tokens": 9410969600, "loss": 4.763717412948608, "l_mix": 2.94979128241539, "l_a": 3.0465138852596283, "l_c": 2.9942318499088287, "z_loss": 17.02391505241394, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20517246425151825, "tokens_per_second": 1422425.096948662, "eta_seconds": 20098.703554463384, "data_wait_seconds": 2.086162567138672e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20763, "cosmopedia_v2": 0.20045, "finemath": 0.08234}}
{"step": 17960, "total_steps": 72479, "tokens": 9416212480, "loss": 4.789822220802307, "l_mix": 2.964907318353653, "l_a": 3.0606749057769775, "l_c": 3.016838848590851, "z_loss": 16.607162952423096, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18225719034671783, "tokens_per_second": 1453037.0734305896, "eta_seconds": 19671.664264225958, "data_wait_seconds": 1.8030405044555664e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20763, "cosmopedia_v2": 0.20045, "finemath": 0.08235}}
{"step": 17970, "total_steps": 72479, "tokens": 9421455360, "loss": 4.732005715370178, "l_mix": 2.927415758371353, "l_a": 3.0306054949760437, "l_c": 2.978906124830246, "z_loss": 17.362743616104126, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.26609715819358826, "tokens_per_second": 1355331.0390650472, "eta_seconds": 21085.929391622543, "data_wait_seconds": 1.722574234008789e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20762, "cosmopedia_v2": 0.20045, "finemath": 0.08236}}
{"step": 17980, "total_steps": 72479, "tokens": 9426698240, "loss": 4.868409097194672, "l_mix": 3.015663594007492, "l_a": 3.1032877564430237, "l_c": 3.0670543015003204, "z_loss": 16.429158449172974, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20238225162029266, "tokens_per_second": 1413747.65080378, "eta_seconds": 20210.941956830025, "data_wait_seconds": 2.16066837310791e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20768, "cosmopedia_v2": 0.20044, "finemath": 0.08231}}
{"step": 17990, "total_steps": 72479, "tokens": 9431941120, "loss": 3.4886896908283234, "l_mix": 2.145134836435318, "l_a": 2.269650787115097, "l_c": 2.2037293314933777, "z_loss": 15.407175660133362, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.23175203800201416, "tokens_per_second": 1462938.5538057322, "eta_seconds": 19527.770840191843, "data_wait_seconds": 1.519918441772461e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08232}}
{"step": 18000, "total_steps": 72479, "tokens": 9437184000, "loss": 3.5351685285568237, "l_mix": 2.173212707042694, "l_a": 2.3002597093582153, "l_c": 2.234484702348709, "z_loss": 15.324782490730286, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21056905388832092, "tokens_per_second": 1448302.379387457, "eta_seconds": 19721.49349370003, "data_wait_seconds": 1.615285873413086e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.28200218, "gate_a2b/L1_absmax": 0.64434123, "gate_a2b/L1_rms": 0.30183089, "gate_a2b/L3_absmean": 0.32527065, "gate_a2b/L3_absmax": 0.49315408, "gate_a2b/L3_rms": 0.32998094, "gate_a2b/L5_absmean": 0.50748873, "gate_a2b/L5_absmax": 0.68515885, "gate_a2b/L5_rms": 0.51364082, "gate_c2b/L1_absmean": 0.01491509, "gate_c2b/L1_absmax": 0.46059033, "gate_c2b/L1_rms": 0.05431902, "gate_c2b/L3_absmean": 0.20968056, "gate_c2b/L3_absmax": 0.43339437, "gate_c2b/L3_rms": 0.21403587, "gate_c2b/L5_absmean": 0.53139704, "gate_c2b/L5_absmax": 0.73080152, "gate_c2b/L5_rms": 0.54332441, "gate_b2a/L7_absmean": 0.08276532, "gate_b2a/L7_absmax": 0.27004761, "gate_b2a/L7_rms": 0.08615357, "gate_b2a/L11_absmean": 0.12906863, "gate_b2a/L11_absmax": 0.40176892, "gate_b2a/L11_rms": 0.13162036, "gate_b2a/L14_absmean": 0.3113758, "gate_b2a/L14_absmax": 0.51718163, "gate_b2a/L14_rms": 0.31272116, "gate_b2c/L3_absmean": 0.08956461, "gate_b2c/L3_absmax": 0.59420699, "gate_b2c/L3_rms": 0.10800885, "gate_b2c/L5_absmean": 0.11990919, "gate_b2c/L5_absmax": 0.72460192, "gate_b2c/L5_rms": 0.12691709, "gate_b2c/L6_absmean": 0.17519626, "gate_b2c/L6_absmax": 0.4841969, "gate_b2c/L6_rms": 0.17890079}, "streams": {"cos_a2b/round0": 0.372013, "cos_c2b/round0": 0.248651, "cos_b2a/round0": 0.124299, "cos_b2c/round0": 0.099351, "rms_a/round0": 0.893123, "rms_b/round0": 1.646158, "rms_c/round0": 1.276656, "cos_a2b/round1": 0.407353, "cos_c2b/round1": 0.274571, "cos_b2a/round1": 0.219357, "cos_b2c/round1": 0.135823, "rms_a/round1": 1.596844, "rms_b/round1": 3.264297, "rms_c/round1": 2.151616, "cos_a2b/round2": 0.507747, "cos_c2b/round2": 0.468082, "cos_b2a/round2": 0.039096, "cos_b2c/round2": 0.116518, "rms_a/round2": 3.455395, "rms_b/round2": 7.198874, "rms_c/round2": 3.449223}, "alpha_mean": 0.4339289729546678, "alpha_hist": [0.04836, 0.13426, 0.16112, 0.14011, 0.12074, 0.11589, 0.11693, 0.10753, 0.04871, 0.00636], "alpha_by_type": {"special": {"share": 0.00116, "mean": 0.63689, "std": 0.11707}, "whitespace": {"share": 0.03134, "mean": 0.67127, "std": 0.1663}, "digit": {"share": 0.01125, "mean": 0.54261, "std": 0.18033}, "word_start": {"share": 0.57954, "mean": 0.44801, "std": 0.21084}, "word_cont": {"share": 0.25839, "mean": 0.27247, "std": 0.1621}, "punct": {"share": 0.11832, "mean": 0.64237, "std": 0.17392}}, "alpha_p05": 0.09204, "alpha_p50": 0.40993, "alpha_p95": 0.79944}
{"step": 18010, "total_steps": 72479, "tokens": 9442426880, "loss": 3.5248871445655823, "l_mix": 2.1679516434669495, "l_a": 2.2907042801380157, "l_c": 2.227121978998184, "z_loss": 15.875008583068848, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2390293926000595, "tokens_per_second": 1392815.7912300783, "eta_seconds": 20503.388353157043, "data_wait_seconds": 2.3752450942993164e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 18020, "total_steps": 72479, "tokens": 9447669760, "loss": 3.4955630898475647, "l_mix": 2.149073004722595, "l_a": 2.2748856842517853, "l_c": 2.208239644765854, "z_loss": 15.524073004722595, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21565596759319305, "tokens_per_second": 1435482.2226330396, "eta_seconds": 19890.319602584837, "data_wait_seconds": 3.170967102050781e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 18030, "total_steps": 72479, "tokens": 9452912640, "loss": 3.170542538166046, "l_mix": 1.9567686021327972, "l_a": 2.0199558287858963, "l_c": 2.0198919475078583, "z_loss": 18.195505380630493, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.661500871181488, "tokens_per_second": 1460003.5982181637, "eta_seconds": 19552.662299489977, "data_wait_seconds": 2.238154411315918e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08236}}
{"step": 18040, "total_steps": 72479, "tokens": 9458155520, "loss": 4.764859735965729, "l_mix": 2.949883759021759, "l_a": 3.048041135072708, "l_c": 2.996159106492996, "z_loss": 17.15902090072632, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20449988543987274, "tokens_per_second": 1444912.5558235308, "eta_seconds": 19753.246877789497, "data_wait_seconds": 1.7821788787841797e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08232}}
{"step": 18050, "total_steps": 72479, "tokens": 9463398400, "loss": 4.822885155677795, "l_mix": 2.984595000743866, "l_a": 3.0894678235054016, "l_c": 3.0326414108276367, "z_loss": 16.57241702079773, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20634374022483826, "tokens_per_second": 1285153.5336363318, "eta_seconds": 22204.71780617237, "data_wait_seconds": 2.9981136322021484e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 18060, "total_steps": 72479, "tokens": 9468641280, "loss": 4.72234445810318, "l_mix": 2.9217678010463715, "l_a": 3.0277366042137146, "l_c": 2.9684995114803314, "z_loss": 17.057594776153564, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22555430233478546, "tokens_per_second": 1479023.2170910053, "eta_seconds": 19290.588776636123, "data_wait_seconds": 1.5676021575927734e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 18070, "total_steps": 72479, "tokens": 9473884160, "loss": 4.817209422588348, "l_mix": 2.9818203151226044, "l_a": 3.0772807896137238, "l_c": 3.035136044025421, "z_loss": 16.639447689056396, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22863498330116272, "tokens_per_second": 1432724.8056326404, "eta_seconds": 19910.303555750845, "data_wait_seconds": 1.806020736694336e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 18080, "total_steps": 72479, "tokens": 9479127040, "loss": 4.746786832809448, "l_mix": 2.937399983406067, "l_a": 3.0365509390830994, "l_c": 2.9888959527015686, "z_loss": 17.52766513824463, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.24491578340530396, "tokens_per_second": 1441572.3943916806, "eta_seconds": 19784.46800379753, "data_wait_seconds": 2.0235776901245117e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20763, "cosmopedia_v2": 0.20044, "finemath": 0.08236}}
{"step": 18090, "total_steps": 72479, "tokens": 9484369920, "loss": 4.909071266651154, "l_mix": 3.0408146679401398, "l_a": 3.1333243250846863, "l_c": 3.088856339454651, "z_loss": 16.023646235466003, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17610926926136017, "tokens_per_second": 1459836.8900478946, "eta_seconds": 19533.34665427208, "data_wait_seconds": 2.047419548034668e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.20044, "finemath": 0.08231}}
{"step": 18100, "total_steps": 72479, "tokens": 9489612800, "loss": 4.842511236667633, "l_mix": 2.9991398751735687, "l_a": 3.0945909321308136, "l_c": 3.044482111930847, "z_loss": 16.493945598602295, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18947845697402954, "tokens_per_second": 1454422.8806885174, "eta_seconds": 19602.453681492807, "data_wait_seconds": 2.321600914001465e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.20044, "finemath": 0.08232}, "gates": {"gate_a2b/L1_absmean": 0.28316584, "gate_a2b/L1_absmax": 0.64998299, "gate_a2b/L1_rms": 0.30293906, "gate_a2b/L3_absmean": 0.32566291, "gate_a2b/L3_absmax": 0.49519089, "gate_a2b/L3_rms": 0.33040488, "gate_a2b/L5_absmean": 0.50829214, "gate_a2b/L5_absmax": 0.68848294, "gate_a2b/L5_rms": 0.51435548, "gate_c2b/L1_absmean": 0.01409048, "gate_c2b/L1_absmax": 0.45783195, "gate_c2b/L1_rms": 0.05383706, "gate_c2b/L3_absmean": 0.20961288, "gate_c2b/L3_absmax": 0.43208525, "gate_c2b/L3_rms": 0.213939, "gate_c2b/L5_absmean": 0.53160304, "gate_c2b/L5_absmax": 0.71825874, "gate_c2b/L5_rms": 0.5435816, "gate_b2a/L7_absmean": 0.08219354, "gate_b2a/L7_absmax": 0.2677229, "gate_b2a/L7_rms": 0.08557365, "gate_b2a/L11_absmean": 0.12856719, "gate_b2a/L11_absmax": 0.40212631, "gate_b2a/L11_rms": 0.13107435, "gate_b2a/L14_absmean": 0.3125425, "gate_b2a/L14_absmax": 0.51486027, "gate_b2a/L14_rms": 0.31384358, "gate_b2c/L3_absmean": 0.09026805, "gate_b2c/L3_absmax": 0.5937711, "gate_b2c/L3_rms": 0.10838441, "gate_b2c/L5_absmean": 0.11978691, "gate_b2c/L5_absmax": 0.72849619, "gate_b2c/L5_rms": 0.12691963, "gate_b2c/L6_absmean": 0.17576176, "gate_b2c/L6_absmax": 0.48875329, "gate_b2c/L6_rms": 0.17943092}, "streams": {"cos_a2b/round0": 0.381942, "cos_c2b/round0": 0.285076, "cos_b2a/round0": 0.137842, "cos_b2c/round0": 0.139749, "rms_a/round0": 0.846464, "rms_b/round0": 1.633254, "rms_c/round0": 1.251826, "cos_a2b/round1": 0.409174, "cos_c2b/round1": 0.272046, "cos_b2a/round1": 0.232686, "cos_b2c/round1": 0.134153, "rms_a/round1": 1.51288, "rms_b/round1": 3.122801, "rms_c/round1": 2.067247, "cos_a2b/round2": 0.534712, "cos_c2b/round2": 0.48708, "cos_b2a/round2": 0.077007, "cos_b2c/round2": 0.139038, "rms_a/round2": 3.365034, "rms_b/round2": 6.93279, "rms_c/round2": 3.312883}, "alpha_mean": 0.48108563948682104, "alpha_hist": [0.00856, 0.04796, 0.11987, 0.17088, 0.19028, 0.18423, 0.15056, 0.09816, 0.02892, 0.00058], "alpha_by_type": {"special": {"share": 0.00081, "mean": 0.63288, "std": 0.13374}, "whitespace": {"share": 0.04605, "mean": 0.60557, "std": 0.12868}, "digit": {"share": 0.03296, "mean": 0.50297, "std": 0.11903}, "word_start": {"share": 0.56447, "mean": 0.50068, "std": 0.16858}, "word_cont": {"share": 0.25882, "mean": 0.3572, "std": 0.13924}, "punct": {"share": 0.09689, "mean": 0.62999, "std": 0.15995}}, "alpha_p05": 0.19807, "alpha_p50": 0.4767, "alpha_p95": 0.75528}
{"step": 18110, "total_steps": 72479, "tokens": 9494855680, "loss": 4.85232138633728, "l_mix": 3.005831390619278, "l_a": 3.098779410123825, "l_c": 3.050645649433136, "z_loss": 16.625304698944092, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1695537567138672, "tokens_per_second": 1400759.7117688688, "eta_seconds": 20349.68169951439, "data_wait_seconds": 3.600120544433594e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20768, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 18120, "total_steps": 72479, "tokens": 9500098560, "loss": 4.875596761703491, "l_mix": 3.020212024450302, "l_a": 3.1140900552272797, "l_c": 3.0650798976421356, "z_loss": 16.335769176483154, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1909039467573166, "tokens_per_second": 1371888.2394873546, "eta_seconds": 20774.120348644257, "data_wait_seconds": 2.059340476989746e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 18130, "total_steps": 72479, "tokens": 9505341440, "loss": 3.464002877473831, "l_mix": 2.129599303007126, "l_a": 2.2522696554660797, "l_c": 2.1905592381954193, "z_loss": 15.548146367073059, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20780763030052185, "tokens_per_second": 1473589.3600548527, "eta_seconds": 19336.817490959165, "data_wait_seconds": 2.816319465637207e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 18140, "total_steps": 72479, "tokens": 9510584320, "loss": 3.5020756125450134, "l_mix": 2.152640610933304, "l_a": 2.2831632792949677, "l_c": 2.2097704112529755, "z_loss": 15.54831600189209, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19437135756015778, "tokens_per_second": 1460125.8427602528, "eta_seconds": 19511.52756679058, "data_wait_seconds": 2.4437904357910156e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08236}}
{"step": 18150, "total_steps": 72479, "tokens": 9515827200, "loss": 4.728833079338074, "l_mix": 2.9273931682109833, "l_a": 3.0257027745246887, "l_c": 2.973614424467087, "z_loss": 16.446284770965576, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20393106341362, "tokens_per_second": 1174671.8301567493, "eta_seconds": 24248.510963439945, "data_wait_seconds": 2.1696090698242188e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08231}}
{"step": 18160, "total_steps": 72479, "tokens": 9521070080, "loss": 4.819393634796143, "l_mix": 2.9834953248500824, "l_a": 3.0795487761497498, "l_c": 3.034718841314316, "z_loss": 16.179945468902588, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2292332649230957, "tokens_per_second": 1433360.0266723384, "eta_seconds": 19868.560125899316, "data_wait_seconds": 1.7195940017700195e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08232}}
{"step": 18170, "total_steps": 72479, "tokens": 9526312960, "loss": 4.764106333255768, "l_mix": 2.9479748010635376, "l_a": 3.0517975091934204, "l_c": 2.996522843837738, "z_loss": 16.35324490070343, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18142807483673096, "tokens_per_second": 1374222.137285902, "eta_seconds": 20719.762998604772, "data_wait_seconds": 2.3365020751953125e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 18180, "total_steps": 72479, "tokens": 9531555840, "loss": 4.77630341053009, "l_mix": 2.9568261802196503, "l_a": 3.055557817220688, "l_c": 3.0036825239658356, "z_loss": 17.050379753112793, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.23230329155921936, "tokens_per_second": 1412319.4931888257, "eta_seconds": 20157.13459262848, "data_wait_seconds": 1.627206802368164e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 18190, "total_steps": 72479, "tokens": 9536798720, "loss": 4.844863712787628, "l_mix": 2.999849498271942, "l_a": 3.093819409608841, "l_c": 3.0505918860435486, "z_loss": 16.907522916793823, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2849794924259186, "tokens_per_second": 1428941.334977086, "eta_seconds": 19918.992148447036, "data_wait_seconds": 2.5987625122070312e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 18200, "total_steps": 72479, "tokens": 9542041600, "loss": 3.3028534054756165, "l_mix": 2.041349619626999, "l_a": 2.113154038786888, "l_c": 2.085361495614052, "z_loss": 19.49067258834839, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.3339654207229614, "tokens_per_second": 1444502.7173324632, "eta_seconds": 19700.778690505027, "data_wait_seconds": 2.956390380859375e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08236}, "gates": {"gate_a2b/L1_absmean": 0.28309262, "gate_a2b/L1_absmax": 0.63617027, "gate_a2b/L1_rms": 0.30251092, "gate_a2b/L3_absmean": 0.32592702, "gate_a2b/L3_absmax": 0.49889353, "gate_a2b/L3_rms": 0.33078644, "gate_a2b/L5_absmean": 0.50776803, "gate_a2b/L5_absmax": 0.68703765, "gate_a2b/L5_rms": 0.51400566, "gate_c2b/L1_absmean": 0.01374129, "gate_c2b/L1_absmax": 0.4615505, "gate_c2b/L1_rms": 0.0542795, "gate_c2b/L3_absmean": 0.20894699, "gate_c2b/L3_absmax": 0.43761352, "gate_c2b/L3_rms": 0.21339771, "gate_c2b/L5_absmean": 0.53095382, "gate_c2b/L5_absmax": 0.73019636, "gate_c2b/L5_rms": 0.54285926, "gate_b2a/L7_absmean": 0.08358198, "gate_b2a/L7_absmax": 0.26613316, "gate_b2a/L7_rms": 0.08685491, "gate_b2a/L11_absmean": 0.12850745, "gate_b2a/L11_absmax": 0.40227342, "gate_b2a/L11_rms": 0.13108839, "gate_b2a/L14_absmean": 0.31335738, "gate_b2a/L14_absmax": 0.51295954, "gate_b2a/L14_rms": 0.31471765, "gate_b2c/L3_absmean": 0.09071984, "gate_b2c/L3_absmax": 0.59434956, "gate_b2c/L3_rms": 0.10865173, "gate_b2c/L5_absmean": 0.11913576, "gate_b2c/L5_absmax": 0.72448123, "gate_b2c/L5_rms": 0.12632571, "gate_b2c/L6_absmean": 0.1754086, "gate_b2c/L6_absmax": 0.49992868, "gate_b2c/L6_rms": 0.17913616}, "streams": {"cos_a2b/round0": 0.41596, "cos_c2b/round0": 0.298881, "cos_b2a/round0": 0.140095, "cos_b2c/round0": 0.15934, "rms_a/round0": 0.887822, "rms_b/round0": 1.60022, "rms_c/round0": 1.205378, "cos_a2b/round1": 0.375382, "cos_c2b/round1": 0.237653, "cos_b2a/round1": 0.222312, "cos_b2c/round1": 0.14281, "rms_a/round1": 1.548337, "rms_b/round1": 3.145518, "rms_c/round1": 2.123284, "cos_a2b/round2": 0.493362, "cos_c2b/round2": 0.534091, "cos_b2a/round2": 0.082895, "cos_b2c/round2": 0.130427, "rms_a/round2": 3.355517, "rms_b/round2": 7.228816, "rms_c/round2": 3.481844}, "alpha_mean": 0.4725131368488178, "alpha_hist": [0.01588, 0.03423, 0.09563, 0.17598, 0.23338, 0.21904, 0.15091, 0.06332, 0.01024, 0.00139], "alpha_by_type": {"special": {"share": 0.00056, "mean": 0.60512, "std": 0.11447}, "whitespace": {"share": 0.10297, "mean": 0.55313, "std": 0.13059}, "digit": {"share": 0.13432, "mean": 0.47885, "std": 0.12411}, "word_start": {"share": 0.29988, "mean": 0.45749, "std": 0.16517}, "word_cont": {"share": 0.22987, "mean": 0.36227, "std": 0.13178}, "punct": {"share": 0.23239, "mean": 0.56124, "std": 0.13756}}, "alpha_p05": 0.23829, "alpha_p50": 0.49008, "alpha_p95": 0.73754}
{"step": 18210, "total_steps": 72479, "tokens": 9547284480, "loss": 4.809226155281067, "l_mix": 2.978622704744339, "l_a": 3.0740397572517395, "l_c": 3.0225410163402557, "z_loss": 16.292174577713013, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17148803174495697, "tokens_per_second": 1412231.241751716, "eta_seconds": 20147.256788277624, "data_wait_seconds": 4.32431697845459e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20769, "cosmopedia_v2": 0.20044, "finemath": 0.08232}}
{"step": 18220, "total_steps": 72479, "tokens": 9552527360, "loss": 4.884494364261627, "l_mix": 3.025018811225891, "l_a": 3.120685577392578, "l_c": 3.072033107280731, "z_loss": 16.59846782684326, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19271846115589142, "tokens_per_second": 1269536.6757138239, "eta_seconds": 22407.657168316844, "data_wait_seconds": 2.193450927734375e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20768, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 18230, "total_steps": 72479, "tokens": 9557770240, "loss": 4.895039081573486, "l_mix": 3.0313377678394318, "l_a": 3.1272778809070587, "l_c": 3.079411596059799, "z_loss": 16.94284152984619, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21565251052379608, "tokens_per_second": 1450942.1493590255, "eta_seconds": 19602.504293203354, "data_wait_seconds": 2.0742416381835938e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20768, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 18240, "total_steps": 72479, "tokens": 9563013120, "loss": 4.9217734932899475, "l_mix": 3.0496246218681335, "l_a": 3.144580692052841, "l_c": 3.090468853712082, "z_loss": 16.33905851840973, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.24047879874706268, "tokens_per_second": 1405631.271054363, "eta_seconds": 20230.66604847908, "data_wait_seconds": 1.811981201171875e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20768, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 18250, "total_steps": 72479, "tokens": 9568256000, "loss": 4.829718291759491, "l_mix": 2.990332752466202, "l_a": 3.0898952782154083, "l_c": 3.035935342311859, "z_loss": 16.364182233810425, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.23098817467689514, "tokens_per_second": 1427767.9791531197, "eta_seconds": 19913.329313397408, "data_wait_seconds": 2.6971101760864258e-05, "mix": {"fineweb_hq": 0.50953, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08236}}
{"step": 18260, "total_steps": 72479, "tokens": 9573498880, "loss": 3.483930081129074, "l_mix": 2.1405284702777863, "l_a": 2.270498126745224, "l_c": 2.202523350715637, "z_loss": 14.951558351516724, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2760344445705414, "tokens_per_second": 1371927.2682552987, "eta_seconds": 20720.027752017977, "data_wait_seconds": 3.7997961044311523e-05, "mix": {"fineweb_hq": 0.50953, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08237}}
{"step": 18270, "total_steps": 72479, "tokens": 9578741760, "loss": 4.691150486469269, "l_mix": 2.904498130083084, "l_a": 3.001954525709152, "l_c": 2.947848081588745, "z_loss": 17.114786386489868, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.24406985938549042, "tokens_per_second": 1380955.460074356, "eta_seconds": 20580.77107748985, "data_wait_seconds": 2.5331974029541016e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08232}}
{"step": 18280, "total_steps": 72479, "tokens": 9583984640, "loss": 4.642314970493317, "l_mix": 2.871749997138977, "l_a": 2.974588692188263, "l_c": 2.921738922595978, "z_loss": 16.66607093811035, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18976645171642303, "tokens_per_second": 1271784.1367062689, "eta_seconds": 22343.32422606945, "data_wait_seconds": 2.682209014892578e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 18290, "total_steps": 72479, "tokens": 9589227520, "loss": 4.660414338111877, "l_mix": 2.884303092956543, "l_a": 2.98284050822258, "l_c": 2.9318789541721344, "z_loss": 16.954221963882446, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.23464709520339966, "tokens_per_second": 1381616.511639289, "eta_seconds": 20563.33446557522, "data_wait_seconds": 3.552436828613281e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 18300, "total_steps": 72479, "tokens": 9594470400, "loss": 4.722765743732452, "l_mix": 2.922856390476227, "l_a": 3.0256878435611725, "l_c": 2.968510240316391, "z_loss": 16.49902105331421, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21532675623893738, "tokens_per_second": 1465265.5807770917, "eta_seconds": 19385.83689172268, "data_wait_seconds": 1.8417835235595703e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.28325972, "gate_a2b/L1_absmax": 0.64249688, "gate_a2b/L1_rms": 0.30277634, "gate_a2b/L3_absmean": 0.32490689, "gate_a2b/L3_absmax": 0.50493848, "gate_a2b/L3_rms": 0.32976115, "gate_a2b/L5_absmean": 0.50756848, "gate_a2b/L5_absmax": 0.6838268, "gate_a2b/L5_rms": 0.51375967, "gate_c2b/L1_absmean": 0.01544932, "gate_c2b/L1_absmax": 0.46005267, "gate_c2b/L1_rms": 0.05450599, "gate_c2b/L3_absmean": 0.20851827, "gate_c2b/L3_absmax": 0.43276131, "gate_c2b/L3_rms": 0.2129847, "gate_c2b/L5_absmean": 0.5313639, "gate_c2b/L5_absmax": 0.72714299, "gate_c2b/L5_rms": 0.54336542, "gate_b2a/L7_absmean": 0.08319064, "gate_b2a/L7_absmax": 0.2614575, "gate_b2a/L7_rms": 0.08658335, "gate_b2a/L11_absmean": 0.12896141, "gate_b2a/L11_absmax": 0.40010381, "gate_b2a/L11_rms": 0.131506, "gate_b2a/L14_absmean": 0.31336427, "gate_b2a/L14_absmax": 0.51877737, "gate_b2a/L14_rms": 0.31471282, "gate_b2c/L3_absmean": 0.09013646, "gate_b2c/L3_absmax": 0.58678663, "gate_b2c/L3_rms": 0.10811769, "gate_b2c/L5_absmean": 0.11955236, "gate_b2c/L5_absmax": 0.7169041, "gate_b2c/L5_rms": 0.126623, "gate_b2c/L6_absmean": 0.17573059, "gate_b2c/L6_absmax": 0.49559459, "gate_b2c/L6_rms": 0.17939073}, "streams": {"cos_a2b/round0": 0.380728, "cos_c2b/round0": 0.283724, "cos_b2a/round0": 0.140131, "cos_b2c/round0": 0.137905, "rms_a/round0": 0.860562, "rms_b/round0": 1.633327, "rms_c/round0": 1.27272, "cos_a2b/round1": 0.404763, "cos_c2b/round1": 0.272455, "cos_b2a/round1": 0.231545, "cos_b2c/round1": 0.134668, "rms_a/round1": 1.520181, "rms_b/round1": 3.16238, "rms_c/round1": 2.060525, "cos_a2b/round2": 0.537453, "cos_c2b/round2": 0.480773, "cos_b2a/round2": 0.08211, "cos_b2c/round2": 0.146221, "rms_a/round2": 3.406475, "rms_b/round2": 7.003464, "rms_c/round2": 3.254889}, "alpha_mean": 0.4248468426640599, "alpha_hist": [0.01675, 0.07951, 0.16328, 0.19632, 0.19449, 0.17804, 0.12032, 0.04729, 0.004, 0.0], "alpha_by_type": {"special": {"share": 0.00117, "mean": 0.54166, "std": 0.09419}, "whitespace": {"share": 0.02626, "mean": 0.50991, "std": 0.10116}, "digit": {"share": 0.03841, "mean": 0.52763, "std": 0.11804}, "word_start": {"share": 0.57492, "mean": 0.44883, "std": 0.15412}, "word_cont": {"share": 0.25641, "mean": 0.29455, "std": 0.12994}, "punct": {"share": 0.10283, "mean": 0.55419, "std": 0.15563}}, "alpha_p05": 0.15304, "alpha_p50": 0.42513, "alpha_p95": 0.69306}
{"step": 18310, "total_steps": 72479, "tokens": 9599713280, "loss": 3.5420433282852173, "l_mix": 2.177791327238083, "l_a": 2.304071456193924, "l_c": 2.238205999135971, "z_loss": 15.68704617023468, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1995358020067215, "tokens_per_second": 1436118.0158818997, "eta_seconds": 19775.642640733717, "data_wait_seconds": 3.3974647521972656e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08236}}
{"step": 18320, "total_steps": 72479, "tokens": 9604956160, "loss": 3.2484565377235413, "l_mix": 2.008098930120468, "l_a": 2.0873310267925262, "l_c": 2.0409381687641144, "z_loss": 18.768243551254272, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.23728886246681213, "tokens_per_second": 1459320.6271043874, "eta_seconds": 19457.625188469887, "data_wait_seconds": 2.5272369384765625e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08237}}
{"step": 18330, "total_steps": 72479, "tokens": 9610199040, "loss": 4.610389709472656, "l_mix": 2.8525324761867523, "l_a": 2.9539214372634888, "l_c": 2.899974673986435, "z_loss": 16.88397240638733, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19253118336200714, "tokens_per_second": 1345402.2545326087, "eta_seconds": 21101.251180720326, "data_wait_seconds": 1.6063451766967773e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08232}}
{"step": 18340, "total_steps": 72479, "tokens": 9615441920, "loss": 4.74001157283783, "l_mix": 2.934614449739456, "l_a": 3.0351566672325134, "l_c": 2.977318823337555, "z_loss": 16.544591188430786, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22697722911834717, "tokens_per_second": 1437229.145334159, "eta_seconds": 19749.41026220322, "data_wait_seconds": 2.5600194931030273e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20763, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 18350, "total_steps": 72479, "tokens": 9620684800, "loss": 4.688634991645813, "l_mix": 2.902447074651718, "l_a": 2.99490487575531, "l_c": 2.9533839523792267, "z_loss": 17.011310338974, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.23656439781188965, "tokens_per_second": 1354312.195007069, "eta_seconds": 20954.684788799284, "data_wait_seconds": 2.619624137878418e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20763, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 18360, "total_steps": 72479, "tokens": 9625927680, "loss": 4.693533658981323, "l_mix": 2.904362291097641, "l_a": 3.009300321340561, "l_c": 2.949067234992981, "z_loss": 16.611766576766968, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22744840383529663, "tokens_per_second": 1444047.0237888307, "eta_seconds": 19648.904644083977, "data_wait_seconds": 2.0951032638549805e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20763, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 18370, "total_steps": 72479, "tokens": 9631170560, "loss": 4.7996731996536255, "l_mix": 2.9709226489067078, "l_a": 3.0699344873428345, "l_c": 3.020297735929489, "z_loss": 16.808831691741943, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2073247730731964, "tokens_per_second": 1452829.0461468857, "eta_seconds": 19526.522729730605, "data_wait_seconds": 1.9758939743041992e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20762, "cosmopedia_v2": 0.20044, "finemath": 0.08236}}
{"step": 18380, "total_steps": 72479, "tokens": 9636413440, "loss": 4.694826900959015, "l_mix": 2.9061141908168793, "l_a": 3.0076833069324493, "l_c": 2.949150651693344, "z_loss": 16.625168800354004, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18259486556053162, "tokens_per_second": 1406117.7018384202, "eta_seconds": 20171.466780424118, "data_wait_seconds": 1.9878149032592773e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08232}}
{"step": 18390, "total_steps": 72479, "tokens": 9641656320, "loss": 4.690835118293762, "l_mix": 2.9034029245376587, "l_a": 3.003848969936371, "l_c": 2.948707401752472, "z_loss": 16.653096675872803, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1620951145887375, "tokens_per_second": 1406340.8067834154, "eta_seconds": 20164.53870584965, "data_wait_seconds": 1.946091651916504e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 18400, "total_steps": 72479, "tokens": 9646899200, "loss": 4.6454824805259705, "l_mix": 2.8747269213199615, "l_a": 2.9751171469688416, "l_c": 2.9216936826705933, "z_loss": 17.12313175201416, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21207883954048157, "tokens_per_second": 1399324.1797364857, "eta_seconds": 20261.902969002724, "data_wait_seconds": 3.394484519958496e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.28524604, "gate_a2b/L1_absmax": 0.64729834, "gate_a2b/L1_rms": 0.30483127, "gate_a2b/L3_absmean": 0.32662946, "gate_a2b/L3_absmax": 0.5076952, "gate_a2b/L3_rms": 0.33141997, "gate_a2b/L5_absmean": 0.50875521, "gate_a2b/L5_absmax": 0.68479645, "gate_a2b/L5_rms": 0.51489973, "gate_c2b/L1_absmean": 0.01397252, "gate_c2b/L1_absmax": 0.45608747, "gate_c2b/L1_rms": 0.05405284, "gate_c2b/L3_absmean": 0.20914388, "gate_c2b/L3_absmax": 0.44238624, "gate_c2b/L3_rms": 0.21357042, "gate_c2b/L5_absmean": 0.53189147, "gate_c2b/L5_absmax": 0.72011715, "gate_c2b/L5_rms": 0.54394269, "gate_b2a/L7_absmean": 0.08293091, "gate_b2a/L7_absmax": 0.26529285, "gate_b2a/L7_rms": 0.08631068, "gate_b2a/L11_absmean": 0.12929434, "gate_b2a/L11_absmax": 0.40101546, "gate_b2a/L11_rms": 0.13179085, "gate_b2a/L14_absmean": 0.31410065, "gate_b2a/L14_absmax": 0.52361304, "gate_b2a/L14_rms": 0.31539598, "gate_b2c/L3_absmean": 0.09127533, "gate_b2c/L3_absmax": 0.58527291, "gate_b2c/L3_rms": 0.10934165, "gate_b2c/L5_absmean": 0.12025426, "gate_b2c/L5_absmax": 0.7083652, "gate_b2c/L5_rms": 0.12713861, "gate_b2c/L6_absmean": 0.17663638, "gate_b2c/L6_absmax": 0.48989424, "gate_b2c/L6_rms": 0.18032224}, "streams": {"cos_a2b/round0": 0.379521, "cos_c2b/round0": 0.286274, "cos_b2a/round0": 0.135935, "cos_b2c/round0": 0.137593, "rms_a/round0": 0.853405, "rms_b/round0": 1.629105, "rms_c/round0": 1.265827, "cos_a2b/round1": 0.408502, "cos_c2b/round1": 0.266354, "cos_b2a/round1": 0.229838, "cos_b2c/round1": 0.130347, "rms_a/round1": 1.535037, "rms_b/round1": 3.171524, "rms_c/round1": 2.097644, "cos_a2b/round2": 0.523425, "cos_c2b/round2": 0.487923, "cos_b2a/round2": 0.075839, "cos_b2c/round2": 0.135257, "rms_a/round2": 3.387527, "rms_b/round2": 7.078281, "rms_c/round2": 3.359925}, "alpha_mean": 0.4376525789111554, "alpha_hist": [0.01663, 0.07181, 0.15147, 0.19485, 0.19336, 0.16862, 0.13147, 0.06441, 0.00722, 0.00015], "alpha_by_type": {"special": {"share": 0.00122, "mean": 0.56567, "std": 0.1137}, "whitespace": {"share": 0.02766, "mean": 0.51969, "std": 0.13059}, "digit": {"share": 0.02731, "mean": 0.49458, "std": 0.12208}, "word_start": {"share": 0.56412, "mean": 0.45711, "std": 0.16537}, "word_cont": {"share": 0.26881, "mean": 0.32906, "std": 0.14317}, "punct": {"share": 0.11087, "mean": 0.56604, "std": 0.15819}}, "alpha_p05": 0.15279, "alpha_p50": 0.42728, "alpha_p95": 0.71622}
{"step": 18410, "total_steps": 72479, "tokens": 9652142080, "loss": 4.696372628211975, "l_mix": 2.906813621520996, "l_a": 3.0099065005779266, "l_c": 2.9496873319149017, "z_loss": 16.807395458221436, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18651345372200012, "tokens_per_second": 1412377.8195647649, "eta_seconds": 20070.923997330665, "data_wait_seconds": 1.9043684005737305e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 18420, "total_steps": 72479, "tokens": 9657384960, "loss": 3.4829788506031036, "l_mix": 2.140940696001053, "l_a": 2.267289161682129, "l_c": 2.200932264328003, "z_loss": 15.717001676559448, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19333627820014954, "tokens_per_second": 1471899.47089957, "eta_seconds": 19255.720619750024, "data_wait_seconds": 1.958012580871582e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08236}}
{"step": 18430, "total_steps": 72479, "tokens": 9662627840, "loss": 3.2299974858760834, "l_mix": 1.994164153933525, "l_a": 2.081085830926895, "l_c": 2.032004803419113, "z_loss": 19.060853481292725, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20820438861846924, "tokens_per_second": 1296446.2158862564, "eta_seconds": 21857.63031644821, "data_wait_seconds": 4.735589027404785e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08237}}
{"step": 18440, "total_steps": 72479, "tokens": 9667870720, "loss": 4.684510350227356, "l_mix": 2.900285094976425, "l_a": 2.992917060852051, "l_c": 2.9488686621189117, "z_loss": 16.894025325775146, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17727598547935486, "tokens_per_second": 1337476.7642795423, "eta_seconds": 21183.171168780325, "data_wait_seconds": 1.6510486602783203e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08232}}
{"step": 18450, "total_steps": 72479, "tokens": 9673113600, "loss": 4.747357189655304, "l_mix": 2.937649130821228, "l_a": 3.0410018861293793, "l_c": 2.985696256160736, "z_loss": 16.986664533615112, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2067536860704422, "tokens_per_second": 1469230.8664782587, "eta_seconds": 19279.98995821476, "data_wait_seconds": 3.0040740966796875e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20764, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 18460, "total_steps": 72479, "tokens": 9678356480, "loss": 4.669982194900513, "l_mix": 2.8895949721336365, "l_a": 2.9916591942310333, "l_c": 2.9375323951244354, "z_loss": 16.29627013206482, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2141898274421692, "tokens_per_second": 1345549.1190001704, "eta_seconds": 21048.29401772022, "data_wait_seconds": 1.7821788787841797e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20764, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 18470, "total_steps": 72479, "tokens": 9683599360, "loss": 4.641293287277222, "l_mix": 2.8715415596961975, "l_a": 2.9762275218963623, "l_c": 2.9173751175403595, "z_loss": 16.710468530654907, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.46455636620521545, "tokens_per_second": 1429132.6391023973, "eta_seconds": 19813.605691480636, "data_wait_seconds": 1.8030405044555664e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20763, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 18480, "total_steps": 72479, "tokens": 9688842240, "loss": 4.696138858795166, "l_mix": 2.906585603952408, "l_a": 3.005666583776474, "l_c": 2.953885465860367, "z_loss": 16.874250411987305, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19319091737270355, "tokens_per_second": 1441698.1881356246, "eta_seconds": 19637.27772219181, "data_wait_seconds": 1.704692840576172e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20763, "cosmopedia_v2": 0.20043, "finemath": 0.08236}}
{"step": 18490, "total_steps": 72479, "tokens": 9694085120, "loss": 4.857769668102264, "l_mix": 3.0092605352401733, "l_a": 3.096963584423065, "l_c": 3.0592649281024933, "z_loss": 16.405983686447144, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1794043928384781, "tokens_per_second": 1411176.7970540023, "eta_seconds": 20058.28390254974, "data_wait_seconds": 1.8358230590820312e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20768, "cosmopedia_v2": 0.20043, "finemath": 0.08231}}
{"step": 18500, "total_steps": 72479, "tokens": 9699328000, "loss": 4.899901270866394, "l_mix": 3.03550386428833, "l_a": 3.130387544631958, "l_c": 3.078797549009323, "z_loss": 16.418736457824707, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.173703134059906, "tokens_per_second": 1293038.8063666935, "eta_seconds": 21886.84656071663, "data_wait_seconds": 2.3096799850463867e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20768, "cosmopedia_v2": 0.20043, "finemath": 0.08232}, "gates": {"gate_a2b/L1_absmean": 0.28497443, "gate_a2b/L1_absmax": 0.64699644, "gate_a2b/L1_rms": 0.30427411, "gate_a2b/L3_absmean": 0.32744431, "gate_a2b/L3_absmax": 0.50808823, "gate_a2b/L3_rms": 0.33215123, "gate_a2b/L5_absmean": 0.50935471, "gate_a2b/L5_absmax": 0.68978351, "gate_a2b/L5_rms": 0.51548618, "gate_c2b/L1_absmean": 0.01569557, "gate_c2b/L1_absmax": 0.45836541, "gate_c2b/L1_rms": 0.05405103, "gate_c2b/L3_absmean": 0.20910458, "gate_c2b/L3_absmax": 0.43883634, "gate_c2b/L3_rms": 0.21352075, "gate_c2b/L5_absmean": 0.53203982, "gate_c2b/L5_absmax": 0.71917135, "gate_c2b/L5_rms": 0.54392171, "gate_b2a/L7_absmean": 0.08345208, "gate_b2a/L7_absmax": 0.26801103, "gate_b2a/L7_rms": 0.08680306, "gate_b2a/L11_absmean": 0.12993118, "gate_b2a/L11_absmax": 0.39332795, "gate_b2a/L11_rms": 0.13233787, "gate_b2a/L14_absmean": 0.31479865, "gate_b2a/L14_absmax": 0.52393121, "gate_b2a/L14_rms": 0.31613171, "gate_b2c/L3_absmean": 0.09153651, "gate_b2c/L3_absmax": 0.5917657, "gate_b2c/L3_rms": 0.10944653, "gate_b2c/L5_absmean": 0.12003746, "gate_b2c/L5_absmax": 0.7186988, "gate_b2c/L5_rms": 0.12709513, "gate_b2c/L6_absmean": 0.17666161, "gate_b2c/L6_absmax": 0.49182281, "gate_b2c/L6_rms": 0.18043382}, "streams": {"cos_a2b/round0": 0.385093, "cos_c2b/round0": 0.291795, "cos_b2a/round0": 0.140579, "cos_b2c/round0": 0.146062, "rms_a/round0": 0.848874, "rms_b/round0": 1.625928, "rms_c/round0": 1.258401, "cos_a2b/round1": 0.416769, "cos_c2b/round1": 0.281385, "cos_b2a/round1": 0.239802, "cos_b2c/round1": 0.142839, "rms_a/round1": 1.505928, "rms_b/round1": 3.117795, "rms_c/round1": 2.068195, "cos_a2b/round2": 0.5451, "cos_c2b/round2": 0.489119, "cos_b2a/round2": 0.087769, "cos_b2c/round2": 0.145508, "rms_a/round2": 3.364408, "rms_b/round2": 6.84074, "rms_c/round2": 3.302277}, "alpha_mean": 0.4449149820003626, "alpha_hist": [0.01262, 0.06691, 0.14302, 0.18126, 0.2095, 0.19171, 0.12187, 0.0648, 0.00821, 9e-05], "alpha_by_type": {"special": {"share": 0.00063, "mean": 0.55279, "std": 0.10745}, "whitespace": {"share": 0.05054, "mean": 0.53013, "std": 0.11468}, "digit": {"share": 0.03352, "mean": 0.44829, "std": 0.12675}, "word_start": {"share": 0.59651, "mean": 0.46545, "std": 0.15814}, "word_cont": {"share": 0.2269, "mean": 0.31856, "std": 0.13717}, "punct": {"share": 0.0919, "mean": 0.57477, "std": 0.16302}}, "alpha_p05": 0.1764, "alpha_p50": 0.44794, "alpha_p95": 0.7176}
{"step": 18510, "total_steps": 72479, "tokens": 9704570880, "loss": 4.879797279834747, "l_mix": 3.023164689540863, "l_a": 3.1126109957695007, "l_c": 3.0707459449768066, "z_loss": 16.25560235977173, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1670771986246109, "tokens_per_second": 1422502.3884733373, "eta_seconds": 19891.213752102853, "data_wait_seconds": 2.2202730178833008e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 18520, "total_steps": 72479, "tokens": 9709813760, "loss": 4.920318126678467, "l_mix": 3.0468578934669495, "l_a": 3.1441647708415985, "l_c": 3.095342665910721, "z_loss": 16.078179359436035, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19535793364048004, "tokens_per_second": 1365211.8129814935, "eta_seconds": 20722.100353217127, "data_wait_seconds": 2.22623348236084e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 18530, "total_steps": 72479, "tokens": 9715056640, "loss": 4.888786554336548, "l_mix": 3.0286911725997925, "l_a": 3.1181486546993256, "l_c": 3.076592892408371, "z_loss": 16.72939133644104, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18725654482841492, "tokens_per_second": 1425223.701078207, "eta_seconds": 19845.876328468323, "data_wait_seconds": 4.717707633972168e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 18540, "total_steps": 72479, "tokens": 9720299520, "loss": 3.299141377210617, "l_mix": 2.0385847836732864, "l_a": 2.1267571449279785, "l_c": 2.0687236189842224, "z_loss": 19.12337589263916, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.25876152515411377, "tokens_per_second": 1470532.587668628, "eta_seconds": 19230.83559598923, "data_wait_seconds": 1.996755599975586e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08236}}
{"step": 18550, "total_steps": 72479, "tokens": 9725542400, "loss": 4.60022497177124, "l_mix": 2.8462473452091217, "l_a": 2.9479875564575195, "l_c": 2.8930784463882446, "z_loss": 16.577985286712646, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16700749099254608, "tokens_per_second": 1433410.6668691714, "eta_seconds": 19725.21078956127, "data_wait_seconds": 1.5467405319213867e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08232}}
{"step": 18560, "total_steps": 72479, "tokens": 9730785280, "loss": 4.718750178813934, "l_mix": 2.9208795726299286, "l_a": 3.0219689309597015, "l_c": 2.9654391705989838, "z_loss": 16.480421781539917, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2042437493801117, "tokens_per_second": 1470063.7652982115, "eta_seconds": 19229.8356978178, "data_wait_seconds": 2.5093555450439453e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 18570, "total_steps": 72479, "tokens": 9736028160, "loss": 4.632859945297241, "l_mix": 2.8670055866241455, "l_a": 2.9661322236061096, "l_c": 2.9144051671028137, "z_loss": 16.93048095703125, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1753484308719635, "tokens_per_second": 1397862.0853923876, "eta_seconds": 20219.33500261307, "data_wait_seconds": 1.7076730728149414e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 18580, "total_steps": 72479, "tokens": 9741271040, "loss": 4.6087236404418945, "l_mix": 2.85175222158432, "l_a": 2.9540037512779236, "l_c": 2.897058665752411, "z_loss": 16.526050209999084, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19363707304000854, "tokens_per_second": 1390064.5981611644, "eta_seconds": 20328.982515907286, "data_wait_seconds": 1.6450881958007812e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 18590, "total_steps": 72479, "tokens": 9746513920, "loss": 4.68902188539505, "l_mix": 2.901504397392273, "l_a": 3.004764139652252, "l_c": 2.9480001032352448, "z_loss": 16.881333827972412, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18326643109321594, "tokens_per_second": 1407480.2756763226, "eta_seconds": 20073.71365714073, "data_wait_seconds": 1.671910285949707e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20043, "finemath": 0.08236}}
{"step": 18600, "total_steps": 72479, "tokens": 9751756800, "loss": 3.1460896134376526, "l_mix": 1.943455845117569, "l_a": 2.017536237835884, "l_c": 1.9849717319011688, "z_loss": 18.813051462173462, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2342754751443863, "tokens_per_second": 1430067.2362197668, "eta_seconds": 19752.996528100968, "data_wait_seconds": 2.682209014892578e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20763, "cosmopedia_v2": 0.20043, "finemath": 0.08237}, "gates": {"gate_a2b/L1_absmean": 0.28562543, "gate_a2b/L1_absmax": 0.65015334, "gate_a2b/L1_rms": 0.30498144, "gate_a2b/L3_absmean": 0.32730493, "gate_a2b/L3_absmax": 0.5082705, "gate_a2b/L3_rms": 0.33204493, "gate_a2b/L5_absmean": 0.51064742, "gate_a2b/L5_absmax": 0.6903829, "gate_a2b/L5_rms": 0.5167914, "gate_c2b/L1_absmean": 0.01408585, "gate_c2b/L1_absmax": 0.45606622, "gate_c2b/L1_rms": 0.05384198, "gate_c2b/L3_absmean": 0.20894694, "gate_c2b/L3_absmax": 0.44259241, "gate_c2b/L3_rms": 0.21335705, "gate_c2b/L5_absmean": 0.5322606, "gate_c2b/L5_absmax": 0.72053611, "gate_c2b/L5_rms": 0.54418194, "gate_b2a/L7_absmean": 0.08332089, "gate_b2a/L7_absmax": 0.26712587, "gate_b2a/L7_rms": 0.08658042, "gate_b2a/L11_absmean": 0.12959674, "gate_b2a/L11_absmax": 0.40017489, "gate_b2a/L11_rms": 0.13207397, "gate_b2a/L14_absmean": 0.31501728, "gate_b2a/L14_absmax": 0.51894712, "gate_b2a/L14_rms": 0.3163259, "gate_b2c/L3_absmean": 0.09177019, "gate_b2c/L3_absmax": 0.58927733, "gate_b2c/L3_rms": 0.10968245, "gate_b2c/L5_absmean": 0.12045173, "gate_b2c/L5_absmax": 0.72287363, "gate_b2c/L5_rms": 0.12754153, "gate_b2c/L6_absmean": 0.17738889, "gate_b2c/L6_absmax": 0.48840988, "gate_b2c/L6_rms": 0.18106}, "streams": {"cos_a2b/round0": 0.42395, "cos_c2b/round0": 0.32712, "cos_b2a/round0": 0.137688, "cos_b2c/round0": 0.169489, "rms_a/round0": 0.877339, "rms_b/round0": 1.573763, "rms_c/round0": 1.179677, "cos_a2b/round1": 0.384095, "cos_c2b/round1": 0.23248, "cos_b2a/round1": 0.214575, "cos_b2c/round1": 0.117097, "rms_a/round1": 1.523128, "rms_b/round1": 3.102137, "rms_c/round1": 2.079259, "cos_a2b/round2": 0.489512, "cos_c2b/round2": 0.515813, "cos_b2a/round2": 0.073383, "cos_b2c/round2": 0.114495, "rms_a/round2": 3.249968, "rms_b/round2": 7.111629, "rms_c/round2": 3.312677}, "alpha_mean": 0.4126460247609529, "alpha_hist": [0.03174, 0.06999, 0.1555, 0.21118, 0.22813, 0.16971, 0.09183, 0.03474, 0.00673, 0.00044], "alpha_by_type": {"special": {"share": 0.00073, "mean": 0.55715, "std": 0.11938}, "whitespace": {"share": 0.10709, "mean": 0.48667, "std": 0.14266}, "digit": {"share": 0.16817, "mean": 0.43149, "std": 0.12778}, "word_start": {"share": 0.33875, "mean": 0.40447, "std": 0.17074}, "word_cont": {"share": 0.19455, "mean": 0.30151, "std": 0.14081}, "punct": {"share": 0.19072, "mean": 0.4818, "std": 0.1544}}, "alpha_p05": 0.13117, "alpha_p50": 0.40545, "alpha_p95": 0.68972}
{"step": 18610, "total_steps": 72479, "tokens": 9756999680, "loss": 4.769083857536316, "l_mix": 2.9540560245513916, "l_a": 3.0461659133434296, "l_c": 2.998429447412491, "z_loss": 16.49135160446167, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17323517799377441, "tokens_per_second": 1428389.9971094732, "eta_seconds": 19772.52034049034, "data_wait_seconds": 3.707408905029297e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.20043, "finemath": 0.08232}}
{"step": 18620, "total_steps": 72479, "tokens": 9762242560, "loss": 4.873689711093903, "l_mix": 3.0194292664527893, "l_a": 3.110121339559555, "l_c": 3.065285950899124, "z_loss": 16.38211750984192, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1690947711467743, "tokens_per_second": 1248268.680181279, "eta_seconds": 22621.433863019945, "data_wait_seconds": 2.0384788513183594e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 18630, "total_steps": 72479, "tokens": 9767485440, "loss": 4.86328250169754, "l_mix": 3.011953502893448, "l_a": 3.1072336435317993, "l_c": 3.0584426820278168, "z_loss": 16.26135540008545, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2069108933210373, "tokens_per_second": 1422426.845116496, "eta_seconds": 19848.03971390724, "data_wait_seconds": 2.104043960571289e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20768, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 18640, "total_steps": 72479, "tokens": 9772728320, "loss": 4.696809560060501, "l_mix": 2.908646672964096, "l_a": 2.9984912276268005, "l_c": 2.9564461410045624, "z_loss": 16.815551280975342, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18936066329479218, "tokens_per_second": 1359388.5125603327, "eta_seconds": 20764.58743853569, "data_wait_seconds": 3.0249357223510742e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 18650, "total_steps": 72479, "tokens": 9777971200, "loss": 4.887212097644806, "l_mix": 3.0273610651493073, "l_a": 3.118688851594925, "l_c": 3.0753707885742188, "z_loss": 16.33137345314026, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18758969008922577, "tokens_per_second": 1386086.2837163184, "eta_seconds": 20360.85277197361, "data_wait_seconds": 2.0235776901245117e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08236}}
{"step": 18660, "total_steps": 72479, "tokens": 9783214080, "loss": 4.629386246204376, "l_mix": 2.8652264177799225, "l_a": 2.9627596735954285, "l_c": 2.912313997745514, "z_loss": 16.375694513320923, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1896524876356125, "tokens_per_second": 1477005.8054325029, "eta_seconds": 19103.95732245445, "data_wait_seconds": 1.6033649444580078e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08232}}
{"step": 18670, "total_steps": 72479, "tokens": 9788456960, "loss": 4.660520553588867, "l_mix": 2.8841700851917267, "l_a": 2.987052232027054, "l_c": 2.928493231534958, "z_loss": 16.868184804916382, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17297515273094177, "tokens_per_second": 1438783.5515031302, "eta_seconds": 19607.822846269606, "data_wait_seconds": 2.256035804748535e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08232}}
{"step": 18680, "total_steps": 72479, "tokens": 9793699840, "loss": 4.610413551330566, "l_mix": 2.8537012338638306, "l_a": 2.950432062149048, "l_c": 2.899695634841919, "z_loss": 16.739273071289062, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20138032734394073, "tokens_per_second": 1360974.662652979, "eta_seconds": 20724.97812488079, "data_wait_seconds": 1.850724220275879e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 18690, "total_steps": 72479, "tokens": 9798942720, "loss": 4.684077084064484, "l_mix": 2.8985509276390076, "l_a": 2.9970214664936066, "l_c": 2.9493099749088287, "z_loss": 16.266324043273926, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2253134697675705, "tokens_per_second": 1317096.8112530985, "eta_seconds": 21411.430800724032, "data_wait_seconds": 1.5616416931152344e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 18700, "total_steps": 72479, "tokens": 9804185600, "loss": 4.644298791885376, "l_mix": 2.8753446340560913, "l_a": 2.974829524755478, "l_c": 2.9161515831947327, "z_loss": 16.596802473068237, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2751404047012329, "tokens_per_second": 1460847.706245566, "eta_seconds": 19300.906063961982, "data_wait_seconds": 1.6897916793823242e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.28659531, "gate_a2b/L1_absmax": 0.64781821, "gate_a2b/L1_rms": 0.30612648, "gate_a2b/L3_absmean": 0.32738942, "gate_a2b/L3_absmax": 0.51082897, "gate_a2b/L3_rms": 0.3320969, "gate_a2b/L5_absmean": 0.51100409, "gate_a2b/L5_absmax": 0.68711692, "gate_a2b/L5_rms": 0.51714259, "gate_c2b/L1_absmean": 0.01487425, "gate_c2b/L1_absmax": 0.45489222, "gate_c2b/L1_rms": 0.05384437, "gate_c2b/L3_absmean": 0.20884342, "gate_c2b/L3_absmax": 0.44415748, "gate_c2b/L3_rms": 0.21325095, "gate_c2b/L5_absmean": 0.5317325, "gate_c2b/L5_absmax": 0.71678877, "gate_c2b/L5_rms": 0.54362833, "gate_b2a/L7_absmean": 0.0834693, "gate_b2a/L7_absmax": 0.26528758, "gate_b2a/L7_rms": 0.08668035, "gate_b2a/L11_absmean": 0.129563, "gate_b2a/L11_absmax": 0.4015528, "gate_b2a/L11_rms": 0.13201396, "gate_b2a/L14_absmean": 0.31492674, "gate_b2a/L14_absmax": 0.52157539, "gate_b2a/L14_rms": 0.31626704, "gate_b2c/L3_absmean": 0.09178303, "gate_b2c/L3_absmax": 0.59186757, "gate_b2c/L3_rms": 0.10983741, "gate_b2c/L5_absmean": 0.1205114, "gate_b2c/L5_absmax": 0.72540593, "gate_b2c/L5_rms": 0.12764531, "gate_b2c/L6_absmean": 0.17673884, "gate_b2c/L6_absmax": 0.48703104, "gate_b2c/L6_rms": 0.18043758}, "streams": {"cos_a2b/round0": 0.385729, "cos_c2b/round0": 0.295607, "cos_b2a/round0": 0.140396, "cos_b2c/round0": 0.146838, "rms_a/round0": 0.863028, "rms_b/round0": 1.639961, "rms_c/round0": 1.281151, "cos_a2b/round1": 0.410333, "cos_c2b/round1": 0.267441, "cos_b2a/round1": 0.232044, "cos_b2c/round1": 0.130479, "rms_a/round1": 1.533763, "rms_b/round1": 3.158991, "rms_c/round1": 2.101043, "cos_a2b/round2": 0.5406, "cos_c2b/round2": 0.482233, "cos_b2a/round2": 0.076048, "cos_b2c/round2": 0.140102, "rms_a/round2": 3.45827, "rms_b/round2": 7.024753, "rms_c/round2": 3.373869}, "alpha_mean": 0.4497620982760182, "alpha_hist": [0.01456, 0.06708, 0.1432, 0.18782, 0.19017, 0.17651, 0.1261, 0.07822, 0.01624, 0.00011], "alpha_by_type": {"special": {"share": 0.00133, "mean": 0.51868, "std": 0.11157}, "whitespace": {"share": 0.02773, "mean": 0.51865, "std": 0.1174}, "digit": {"share": 0.0287, "mean": 0.52034, "std": 0.12387}, "word_start": {"share": 0.57941, "mean": 0.47094, "std": 0.16757}, "word_cont": {"share": 0.25014, "mean": 0.32497, "std": 0.14073}, "punct": {"share": 0.1127, "mean": 0.58212, "std": 0.16617}}, "alpha_p05": 0.15766, "alpha_p50": 0.44903, "alpha_p95": 0.75707}
{"step": 18710, "total_steps": 72479, "tokens": 9809428480, "loss": 3.357645720243454, "l_mix": 2.075353056192398, "l_a": 2.1535859405994415, "l_c": 2.1145254969596863, "z_loss": 18.592710494995117, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.3017815947532654, "tokens_per_second": 1315289.8619689962, "eta_seconds": 21432.873685956, "data_wait_seconds": 4.178285598754883e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20764, "cosmopedia_v2": 0.20043, "finemath": 0.08236}}
{"step": 18720, "total_steps": 72479, "tokens": 9814671360, "loss": 3.3014305531978607, "l_mix": 2.0408632457256317, "l_a": 2.119496136903763, "l_c": 2.076241448521614, "z_loss": 18.460010528564453, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.28736406564712524, "tokens_per_second": 1224543.5897836834, "eta_seconds": 23016.90101287365, "data_wait_seconds": 2.09808349609375e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20764, "cosmopedia_v2": 0.20043, "finemath": 0.08237}}
{"step": 18730, "total_steps": 72479, "tokens": 9819914240, "loss": 4.922490298748016, "l_mix": 3.0515891313552856, "l_a": 3.1334574222564697, "l_c": 3.0975260138511658, "z_loss": 16.059855222702026, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20775958895683289, "tokens_per_second": 1437784.5090145324, "eta_seconds": 19599.56831870079, "data_wait_seconds": 1.9997358322143555e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20769, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 18740, "total_steps": 72479, "tokens": 9825157120, "loss": 4.889487981796265, "l_mix": 3.0278230011463165, "l_a": 3.125376909971237, "l_c": 3.0746218860149384, "z_loss": 16.65303945541382, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19742068648338318, "tokens_per_second": 1368674.378227266, "eta_seconds": 20585.402401185038, "data_wait_seconds": 1.621246337890625e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20768, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 18750, "total_steps": 72479, "tokens": 9830400000, "loss": 4.835543096065521, "l_mix": 2.995821177959442, "l_a": 3.0833046436309814, "l_c": 3.0436485409736633, "z_loss": 16.358309864997864, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20653487741947174, "tokens_per_second": 1431415.891342167, "eta_seconds": 19679.444752836225, "data_wait_seconds": 2.002716064453125e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20768, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 18760, "total_steps": 72479, "tokens": 9835642880, "loss": 4.7660271525383, "l_mix": 2.9520886838436127, "l_a": 3.045025944709778, "l_c": 2.9959717392921448, "z_loss": 16.390381336212158, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1942342072725296, "tokens_per_second": 1469030.3457121602, "eta_seconds": 19171.984536743163, "data_wait_seconds": 1.5795230865478516e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20768, "cosmopedia_v2": 0.20043, "finemath": 0.08236}}
{"step": 18770, "total_steps": 72479, "tokens": 9840885760, "loss": 4.60557895898819, "l_mix": 2.8495996296405792, "l_a": 2.9513238966464996, "l_c": 2.8963360488414764, "z_loss": 16.811914443969727, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15158887207508087, "tokens_per_second": 1425939.3822991892, "eta_seconds": 19747.672686195372, "data_wait_seconds": 1.6450881958007812e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08231}}
{"step": 18780, "total_steps": 72479, "tokens": 9846128640, "loss": 4.695827126502991, "l_mix": 2.9064548909664154, "l_a": 3.0034835636615753, "l_c": 2.955599397420883, "z_loss": 16.4715678691864, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17064140737056732, "tokens_per_second": 1453718.2106141064, "eta_seconds": 19366.711585807803, "data_wait_seconds": 1.6391277313232422e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08232}}
{"step": 18790, "total_steps": 72479, "tokens": 9851371520, "loss": 4.676785886287689, "l_mix": 2.8942968249320984, "l_a": 2.993753105401993, "l_c": 2.942355513572693, "z_loss": 16.56392526626587, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16651850938796997, "tokens_per_second": 1452386.502414212, "eta_seconds": 19380.859285879134, "data_wait_seconds": 1.767277717590332e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 18800, "total_steps": 72479, "tokens": 9856614400, "loss": 4.670710384845734, "l_mix": 2.8907832205295563, "l_a": 2.993078827857971, "l_c": 2.9344322085380554, "z_loss": 16.738314390182495, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16388723254203796, "tokens_per_second": 1396075.0299238756, "eta_seconds": 20158.841716074945, "data_wait_seconds": 1.7583370208740234e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.2873016, "gate_a2b/L1_absmax": 0.65469199, "gate_a2b/L1_rms": 0.30669895, "gate_a2b/L3_absmean": 0.32810566, "gate_a2b/L3_absmax": 0.50840384, "gate_a2b/L3_rms": 0.33271763, "gate_a2b/L5_absmean": 0.51058656, "gate_a2b/L5_absmax": 0.68400663, "gate_a2b/L5_rms": 0.51669735, "gate_c2b/L1_absmean": 0.01450695, "gate_c2b/L1_absmax": 0.45142794, "gate_c2b/L1_rms": 0.05371648, "gate_c2b/L3_absmean": 0.20914464, "gate_c2b/L3_absmax": 0.43913972, "gate_c2b/L3_rms": 0.21347716, "gate_c2b/L5_absmean": 0.53222859, "gate_c2b/L5_absmax": 0.72012436, "gate_c2b/L5_rms": 0.54408705, "gate_b2a/L7_absmean": 0.08303548, "gate_b2a/L7_absmax": 0.26574263, "gate_b2a/L7_rms": 0.08638475, "gate_b2a/L11_absmean": 0.12963557, "gate_b2a/L11_absmax": 0.40920058, "gate_b2a/L11_rms": 0.13216169, "gate_b2a/L14_absmean": 0.31470603, "gate_b2a/L14_absmax": 0.517892, "gate_b2a/L14_rms": 0.3160086, "gate_b2c/L3_absmean": 0.09251051, "gate_b2c/L3_absmax": 0.59024918, "gate_b2c/L3_rms": 0.11033041, "gate_b2c/L5_absmean": 0.12108378, "gate_b2c/L5_absmax": 0.7277528, "gate_b2c/L5_rms": 0.12814461, "gate_b2c/L6_absmean": 0.17715609, "gate_b2c/L6_absmax": 0.4903309, "gate_b2c/L6_rms": 0.18092021}, "streams": {"cos_a2b/round0": 0.384839, "cos_c2b/round0": 0.294803, "cos_b2a/round0": 0.138098, "cos_b2c/round0": 0.146545, "rms_a/round0": 0.846187, "rms_b/round0": 1.619607, "rms_c/round0": 1.260953, "cos_a2b/round1": 0.41052, "cos_c2b/round1": 0.270662, "cos_b2a/round1": 0.226938, "cos_b2c/round1": 0.132468, "rms_a/round1": 1.519809, "rms_b/round1": 3.121825, "rms_c/round1": 2.058442, "cos_a2b/round2": 0.543683, "cos_c2b/round2": 0.474854, "cos_b2a/round2": 0.076455, "cos_b2c/round2": 0.143374, "rms_a/round2": 3.399017, "rms_b/round2": 6.914047, "rms_c/round2": 3.289546}, "alpha_mean": 0.46549205998771015, "alpha_hist": [0.01225, 0.05733, 0.13177, 0.17583, 0.19205, 0.17732, 0.14462, 0.08855, 0.01971, 0.00056], "alpha_by_type": {"special": {"share": 0.00111, "mean": 0.54786, "std": 0.09149}, "whitespace": {"share": 0.02457, "mean": 0.56847, "std": 0.11591}, "digit": {"share": 0.03706, "mean": 0.52144, "std": 0.12746}, "word_start": {"share": 0.58682, "mean": 0.48693, "std": 0.1615}, "word_cont": {"share": 0.24261, "mean": 0.32675, "std": 0.14434}, "punct": {"share": 0.10782, "mean": 0.61746, "std": 0.15926}}, "alpha_p05": 0.1781, "alpha_p50": 0.45689, "alpha_p95": 0.74982}
{"step": 18810, "total_steps": 72479, "tokens": 9861857280, "loss": 4.622989237308502, "l_mix": 2.861554265022278, "l_a": 2.958782523870468, "l_c": 2.90723317861557, "z_loss": 16.30098295211792, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1610315591096878, "tokens_per_second": 1401359.6649058834, "eta_seconds": 20079.0798940897, "data_wait_seconds": 1.683831214904785e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 18820, "total_steps": 72479, "tokens": 9867100160, "loss": 3.4045186936855316, "l_mix": 2.104314297437668, "l_a": 2.1851015388965607, "l_c": 2.1429018676280975, "z_loss": 18.0322048664093, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22594062983989716, "tokens_per_second": 1443601.0944678234, "eta_seconds": 19487.911099410056, "data_wait_seconds": 2.0235776901245117e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08236}}
{"step": 18830, "total_steps": 72479, "tokens": 9872343040, "loss": 3.286965936422348, "l_mix": 2.0315676629543304, "l_a": 2.1102333068847656, "l_c": 2.0681352764368057, "z_loss": 18.877259016036987, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.24494704604148865, "tokens_per_second": 1435759.3637596623, "eta_seconds": 19590.697175288202, "data_wait_seconds": 2.5480985641479492e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08237}}
{"step": 18840, "total_steps": 72479, "tokens": 9877585920, "loss": 4.889014661312103, "l_mix": 3.02825865149498, "l_a": 3.1219877302646637, "l_c": 3.0750518441200256, "z_loss": 16.441112995147705, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17973417043685913, "tokens_per_second": 1459391.0359111256, "eta_seconds": 19269.875818061828, "data_wait_seconds": 1.9609928131103516e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.2077, "cosmopedia_v2": 0.20042, "finemath": 0.08232}}
{"step": 18850, "total_steps": 72479, "tokens": 9882828800, "loss": 4.612946808338165, "l_mix": 2.8547213673591614, "l_a": 2.954989790916443, "l_c": 2.900217980146408, "z_loss": 16.63061785697937, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17492687702178955, "tokens_per_second": 1380951.6443148286, "eta_seconds": 20360.62686753273, "data_wait_seconds": 2.092123031616211e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20764, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 18860, "total_steps": 72479, "tokens": 9888071680, "loss": 4.567873775959015, "l_mix": 2.8262414932250977, "l_a": 2.925126016139984, "l_c": 2.87471941113472, "z_loss": 16.78707456588745, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16226638853549957, "tokens_per_second": 1346107.7272389762, "eta_seconds": 20883.7656178236, "data_wait_seconds": 1.806020736694336e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20764, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 18870, "total_steps": 72479, "tokens": 9893314560, "loss": 4.631634294986725, "l_mix": 2.867001861333847, "l_a": 2.964097708463669, "l_c": 2.9123654067516327, "z_loss": 16.933743000030518, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16662080585956573, "tokens_per_second": 1457173.6653348147, "eta_seconds": 19288.404711556435, "data_wait_seconds": 2.8848648071289062e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20763, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 18880, "total_steps": 72479, "tokens": 9898557440, "loss": 4.614086627960205, "l_mix": 2.8550427556037903, "l_a": 2.959746330976486, "l_c": 2.8982784748077393, "z_loss": 16.363598108291626, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20259898900985718, "tokens_per_second": 1426201.4732148706, "eta_seconds": 19703.606425714494, "data_wait_seconds": 1.570582389831543e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20763, "cosmopedia_v2": 0.20042, "finemath": 0.08236}}
{"step": 18890, "total_steps": 72479, "tokens": 9903800320, "loss": 4.59437096118927, "l_mix": 2.844178706407547, "l_a": 2.941723942756653, "l_c": 2.886714667081833, "z_loss": 16.605865240097046, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17325882613658905, "tokens_per_second": 1399507.1009450168, "eta_seconds": 20075.689228749277, "data_wait_seconds": 1.6629695892333984e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20768, "cosmopedia_v2": 0.20042, "finemath": 0.08232}}
{"step": 18900, "total_steps": 72479, "tokens": 9909043200, "loss": 4.592497885227203, "l_mix": 2.8426328897476196, "l_a": 2.9400033950805664, "l_c": 2.8873468339443207, "z_loss": 16.59981083869934, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15828931331634521, "tokens_per_second": 1464219.2970822926, "eta_seconds": 19184.849433398245, "data_wait_seconds": 1.4662742614746094e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.28960457, "gate_a2b/L1_absmax": 0.64988619, "gate_a2b/L1_rms": 0.30881613, "gate_a2b/L3_absmean": 0.32901174, "gate_a2b/L3_absmax": 0.51081181, "gate_a2b/L3_rms": 0.33363768, "gate_a2b/L5_absmean": 0.5116266, "gate_a2b/L5_absmax": 0.68729049, "gate_a2b/L5_rms": 0.51774263, "gate_c2b/L1_absmean": 0.0145802, "gate_c2b/L1_absmax": 0.45010367, "gate_c2b/L1_rms": 0.05326856, "gate_c2b/L3_absmean": 0.20917125, "gate_c2b/L3_absmax": 0.44133207, "gate_c2b/L3_rms": 0.21347371, "gate_c2b/L5_absmean": 0.53251153, "gate_c2b/L5_absmax": 0.71967423, "gate_c2b/L5_rms": 0.54436982, "gate_b2a/L7_absmean": 0.08402034, "gate_b2a/L7_absmax": 0.26622927, "gate_b2a/L7_rms": 0.08719277, "gate_b2a/L11_absmean": 0.13033527, "gate_b2a/L11_absmax": 0.41101047, "gate_b2a/L11_rms": 0.13287838, "gate_b2a/L14_absmean": 0.3150174, "gate_b2a/L14_absmax": 0.51848465, "gate_b2a/L14_rms": 0.31629324, "gate_b2c/L3_absmean": 0.09285051, "gate_b2c/L3_absmax": 0.59199762, "gate_b2c/L3_rms": 0.11064117, "gate_b2c/L5_absmean": 0.12160036, "gate_b2c/L5_absmax": 0.73545367, "gate_b2c/L5_rms": 0.12873159, "gate_b2c/L6_absmean": 0.17785779, "gate_b2c/L6_absmax": 0.49221537, "gate_b2c/L6_rms": 0.18161842}, "streams": {"cos_a2b/round0": 0.382638, "cos_c2b/round0": 0.29004, "cos_b2a/round0": 0.135421, "cos_b2c/round0": 0.144817, "rms_a/round0": 0.846738, "rms_b/round0": 1.628104, "rms_c/round0": 1.255926, "cos_a2b/round1": 0.407854, "cos_c2b/round1": 0.273275, "cos_b2a/round1": 0.223493, "cos_b2c/round1": 0.13898, "rms_a/round1": 1.509733, "rms_b/round1": 3.119972, "rms_c/round1": 2.063188, "cos_a2b/round2": 0.540128, "cos_c2b/round2": 0.485281, "cos_b2a/round2": 0.078637, "cos_b2c/round2": 0.13907, "rms_a/round2": 3.369436, "rms_b/round2": 6.882753, "rms_c/round2": 3.307554}, "alpha_mean": 0.4602340406501497, "alpha_hist": [0.02055, 0.05879, 0.13216, 0.17053, 0.17688, 0.188, 0.16197, 0.08055, 0.01038, 0.00018], "alpha_by_type": {"special": {"share": 0.00095, "mean": 0.59246, "std": 0.09871}, "whitespace": {"share": 0.0266, "mean": 0.56373, "std": 0.11081}, "digit": {"share": 0.04247, "mean": 0.51176, "std": 0.13163}, "word_start": {"share": 0.58347, "mean": 0.48569, "std": 0.16254}, "word_cont": {"share": 0.24069, "mean": 0.31787, "std": 0.14868}, "punct": {"share": 0.10583, "mean": 0.59579, "std": 0.15263}}, "alpha_p05": 0.1733, "alpha_p50": 0.47366, "alpha_p95": 0.73412}
{"step": 18910, "total_steps": 72479, "tokens": 9914286080, "loss": 4.6602336168289185, "l_mix": 2.8843016922473907, "l_a": 2.9838628470897675, "l_c": 2.930256277322769, "z_loss": 16.961456060409546, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16635558009147644, "tokens_per_second": 1458019.62686629, "eta_seconds": 19262.82976887226, "data_wait_seconds": 1.8596649169921875e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 18920, "total_steps": 72479, "tokens": 9919528960, "loss": 4.6693554520606995, "l_mix": 2.888636827468872, "l_a": 2.9920412600040436, "l_c": 2.9381615817546844, "z_loss": 16.577943801879883, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1585502326488495, "tokens_per_second": 1328545.6587749633, "eta_seconds": 21136.150501513483, "data_wait_seconds": 1.9788742065429688e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 18930, "total_steps": 72479, "tokens": 9924771840, "loss": 4.554923593997955, "l_mix": 2.8180156350135803, "l_a": 2.9194375872612, "l_c": 2.8648279905319214, "z_loss": 16.282857179641724, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16866320371627808, "tokens_per_second": 1435783.0808379767, "eta_seconds": 19553.857742643355, "data_wait_seconds": 2.0056962966918945e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08236}}
{"step": 18940, "total_steps": 72479, "tokens": 9930014720, "loss": 3.2992146611213684, "l_mix": 2.0393591076135635, "l_a": 2.120937004685402, "l_c": 2.0721806585788727, "z_loss": 19.202486515045166, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2291783094406128, "tokens_per_second": 1446847.9514106356, "eta_seconds": 19400.694595885274, "data_wait_seconds": 2.0056962966918945e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08237}}
{"step": 18950, "total_steps": 72479, "tokens": 9935257600, "loss": 4.65191388130188, "l_mix": 2.8800239861011505, "l_a": 2.9758791029453278, "l_c": 2.924803465604782, "z_loss": 16.85015618801117, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17462202906608582, "tokens_per_second": 1286921.87843662, "eta_seconds": 21807.54933321476, "data_wait_seconds": 1.5974044799804688e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08232}}
{"step": 18960, "total_steps": 72479, "tokens": 9940500480, "loss": 4.562104880809784, "l_mix": 2.8224459886550903, "l_a": 2.925272047519684, "l_c": 2.868102729320526, "z_loss": 16.463505744934082, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19418776035308838, "tokens_per_second": 1353202.7771390285, "eta_seconds": 20735.524598407745, "data_wait_seconds": 1.9252300262451172e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 18970, "total_steps": 72479, "tokens": 9945743360, "loss": 4.672954320907593, "l_mix": 2.892250567674637, "l_a": 2.9931012094020844, "l_c": 2.9371039867401123, "z_loss": 16.42068862915039, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16710492968559265, "tokens_per_second": 1426708.9122572269, "eta_seconds": 19663.5251598835, "data_wait_seconds": 1.8030405044555664e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20764, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 18980, "total_steps": 72479, "tokens": 9950986240, "loss": 4.694509387016296, "l_mix": 2.9060764014720917, "l_a": 3.0033620297908783, "l_c": 2.952605217695236, "z_loss": 16.4283230304718, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17335748672485352, "tokens_per_second": 1459568.9767494248, "eta_seconds": 19217.23752615452, "data_wait_seconds": 2.104043960571289e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20764, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 18990, "total_steps": 72479, "tokens": 9956229120, "loss": 4.568731069564819, "l_mix": 2.8280182480812073, "l_a": 2.9254015386104584, "l_c": 2.8714873790740967, "z_loss": 16.46017074584961, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1876448094844818, "tokens_per_second": 1381782.93704754, "eta_seconds": 20295.25772833824, "data_wait_seconds": 2.8580427169799805e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20042, "finemath": 0.08236}}
{"step": 19000, "total_steps": 72479, "tokens": 9961472000, "loss": 4.462666571140289, "l_mix": 2.7601606249809265, "l_a": 2.862359404563904, "l_c": 2.8070212602615356, "z_loss": 16.916630029678345, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1862991899251938, "tokens_per_second": 1428371.9976262904, "eta_seconds": 19629.618893814088, "data_wait_seconds": 2.327561378479004e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20763, "cosmopedia_v2": 0.20042, "finemath": 0.08237}, "gates": {"gate_a2b/L1_absmean": 0.29044095, "gate_a2b/L1_absmax": 0.6588316, "gate_a2b/L1_rms": 0.30957836, "gate_a2b/L3_absmean": 0.32954925, "gate_a2b/L3_absmax": 0.51058924, "gate_a2b/L3_rms": 0.33418322, "gate_a2b/L5_absmean": 0.51230073, "gate_a2b/L5_absmax": 0.68789715, "gate_a2b/L5_rms": 0.51832932, "gate_c2b/L1_absmean": 0.01433136, "gate_c2b/L1_absmax": 0.4466354, "gate_c2b/L1_rms": 0.05322134, "gate_c2b/L3_absmean": 0.20889167, "gate_c2b/L3_absmax": 0.43728513, "gate_c2b/L3_rms": 0.21325393, "gate_c2b/L5_absmean": 0.53206205, "gate_c2b/L5_absmax": 0.71731097, "gate_c2b/L5_rms": 0.54392326, "gate_b2a/L7_absmean": 0.08340626, "gate_b2a/L7_absmax": 0.26409429, "gate_b2a/L7_rms": 0.08656646, "gate_b2a/L11_absmean": 0.12995122, "gate_b2a/L11_absmax": 0.41357198, "gate_b2a/L11_rms": 0.13250342, "gate_b2a/L14_absmean": 0.31503752, "gate_b2a/L14_absmax": 0.51778716, "gate_b2a/L14_rms": 0.31632474, "gate_b2c/L3_absmean": 0.0932796, "gate_b2c/L3_absmax": 0.59195673, "gate_b2c/L3_rms": 0.11097282, "gate_b2c/L5_absmean": 0.12190791, "gate_b2c/L5_absmax": 0.74383134, "gate_b2c/L5_rms": 0.12917194, "gate_b2c/L6_absmean": 0.17829186, "gate_b2c/L6_absmax": 0.49302897, "gate_b2c/L6_rms": 0.18206237}, "streams": {"cos_a2b/round0": 0.383335, "cos_c2b/round0": 0.289829, "cos_b2a/round0": 0.136771, "cos_b2c/round0": 0.138574, "rms_a/round0": 0.848196, "rms_b/round0": 1.620793, "rms_c/round0": 1.264642, "cos_a2b/round1": 0.404388, "cos_c2b/round1": 0.259553, "cos_b2a/round1": 0.226431, "cos_b2c/round1": 0.125812, "rms_a/round1": 1.516873, "rms_b/round1": 3.140427, "rms_c/round1": 2.052083, "cos_a2b/round2": 0.524138, "cos_c2b/round2": 0.490098, "cos_b2a/round2": 0.072854, "cos_b2c/round2": 0.129479, "rms_a/round2": 3.348688, "rms_b/round2": 6.993867, "rms_c/round2": 3.301527}, "alpha_mean": 0.44689010532701445, "alpha_hist": [0.02325, 0.0733, 0.15193, 0.18304, 0.17909, 0.15385, 0.11966, 0.08789, 0.0274, 0.00056], "alpha_by_type": {"special": {"share": 0.00084, "mean": 0.58929, "std": 0.1168}, "whitespace": {"share": 0.0247, "mean": 0.55613, "std": 0.12214}, "digit": {"share": 0.04918, "mean": 0.47681, "std": 0.13461}, "word_start": {"share": 0.54115, "mean": 0.4653, "std": 0.18178}, "word_cont": {"share": 0.23914, "mean": 0.32009, "std": 0.15364}, "punct": {"share": 0.14499, "mean": 0.55772, "std": 0.18754}}, "alpha_p05": 0.15178, "alpha_p50": 0.45048, "alpha_p95": 0.76419}
{"step": 19010, "total_steps": 72479, "tokens": 9966714880, "loss": 4.768347084522247, "l_mix": 2.9533236622810364, "l_a": 3.0442830324172974, "l_c": 3.0003313422203064, "z_loss": 16.389968872070312, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.13170531392097473, "tokens_per_second": 1462525.9168636966, "eta_seconds": 19167.629611730576, "data_wait_seconds": 2.3752450942993164e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20768, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 19020, "total_steps": 72479, "tokens": 9971957760, "loss": 4.912972390651703, "l_mix": 3.0437368154525757, "l_a": 3.1362685561180115, "l_c": 3.089037150144577, "z_loss": 16.43890881538391, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15855024755001068, "tokens_per_second": 1392046.778070589, "eta_seconds": 20134.317778348923, "data_wait_seconds": 2.1964311599731445e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20768, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 19030, "total_steps": 72479, "tokens": 9977200640, "loss": 4.916571915149689, "l_mix": 3.0443649888038635, "l_a": 3.1387886703014374, "l_c": 3.096322625875473, "z_loss": 16.73346734046936, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21700842678546906, "tokens_per_second": 1423324.037845391, "eta_seconds": 19688.186644005775, "data_wait_seconds": 2.60770320892334e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 19040, "total_steps": 72479, "tokens": 9982443520, "loss": 4.861722946166992, "l_mix": 3.012435108423233, "l_a": 3.1012655198574066, "l_c": 3.057528793811798, "z_loss": 16.495508193969727, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1637219786643982, "tokens_per_second": 1388604.5931523133, "eta_seconds": 20176.677054190633, "data_wait_seconds": 3.376603126525879e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 19050, "total_steps": 72479, "tokens": 9987686400, "loss": 3.1235590279102325, "l_mix": 1.9303960800170898, "l_a": 2.004706382751465, "l_c": 1.9662549942731857, "z_loss": 18.745173454284668, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2643517851829529, "tokens_per_second": 1434120.9408020831, "eta_seconds": 19532.650807213784, "data_wait_seconds": 2.1576881408691406e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08236}}
{"step": 19060, "total_steps": 72479, "tokens": 9992929280, "loss": 4.554210662841797, "l_mix": 2.817139983177185, "l_a": 2.920538991689682, "l_c": 2.8640348613262177, "z_loss": 16.985759973526, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1424533575773239, "tokens_per_second": 1306295.9982868042, "eta_seconds": 21439.965144753456, "data_wait_seconds": 1.9103288650512695e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08232}}
{"step": 19070, "total_steps": 72479, "tokens": 9998172160, "loss": 4.589378893375397, "l_mix": 2.839716851711273, "l_a": 2.9389752745628357, "l_c": 2.887621760368347, "z_loss": 16.82852840423584, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2001233994960785, "tokens_per_second": 1417809.0507691484, "eta_seconds": 19749.97816300392, "data_wait_seconds": 3.88026237487793e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 19080, "total_steps": 72479, "tokens": 10003415040, "loss": 4.5370659828186035, "l_mix": 2.806410253047943, "l_a": 2.9106640815734863, "l_c": 2.852592259645462, "z_loss": 16.787280321121216, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15019577741622925, "tokens_per_second": 1304025.7711383663, "eta_seconds": 21469.24971241951, "data_wait_seconds": 2.4110078811645508e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 19090, "total_steps": 72479, "tokens": 10008657920, "loss": 4.525768160820007, "l_mix": 2.7996811866760254, "l_a": 2.9013043642044067, "l_c": 2.846796900033951, "z_loss": 16.566089987754822, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.14361566305160522, "tokens_per_second": 1444821.6081739112, "eta_seconds": 19373.47273437977, "data_wait_seconds": 2.8818845748901367e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 19100, "total_steps": 72479, "tokens": 10013900800, "loss": 4.449732184410095, "l_mix": 2.7524166107177734, "l_a": 2.854257583618164, "l_c": 2.797895848751068, "z_loss": 16.695719480514526, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.14649732410907745, "tokens_per_second": 1453715.3275716838, "eta_seconds": 19251.34076886177, "data_wait_seconds": 3.713369369506836e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20042, "finemath": 0.08236}, "gates": {"gate_a2b/L1_absmean": 0.29096776, "gate_a2b/L1_absmax": 0.65328252, "gate_a2b/L1_rms": 0.31017223, "gate_a2b/L3_absmean": 0.33006051, "gate_a2b/L3_absmax": 0.51063186, "gate_a2b/L3_rms": 0.33469108, "gate_a2b/L5_absmean": 0.51269758, "gate_a2b/L5_absmax": 0.68357652, "gate_a2b/L5_rms": 0.51872045, "gate_c2b/L1_absmean": 0.01438478, "gate_c2b/L1_absmax": 0.44560447, "gate_c2b/L1_rms": 0.0530469, "gate_c2b/L3_absmean": 0.20911112, "gate_c2b/L3_absmax": 0.43673939, "gate_c2b/L3_rms": 0.21340421, "gate_c2b/L5_absmean": 0.53251755, "gate_c2b/L5_absmax": 0.71352512, "gate_c2b/L5_rms": 0.54438186, "gate_b2a/L7_absmean": 0.08362924, "gate_b2a/L7_absmax": 0.26454994, "gate_b2a/L7_rms": 0.08683859, "gate_b2a/L11_absmean": 0.13036481, "gate_b2a/L11_absmax": 0.41566253, "gate_b2a/L11_rms": 0.13292034, "gate_b2a/L14_absmean": 0.31527686, "gate_b2a/L14_absmax": 0.52063954, "gate_b2a/L14_rms": 0.31657183, "gate_b2c/L3_absmean": 0.09367603, "gate_b2c/L3_absmax": 0.59067839, "gate_b2c/L3_rms": 0.11146834, "gate_b2c/L5_absmean": 0.1223709, "gate_b2c/L5_absmax": 0.74809974, "gate_b2c/L5_rms": 0.12970287, "gate_b2c/L6_absmean": 0.17880934, "gate_b2c/L6_absmax": 0.48917499, "gate_b2c/L6_rms": 0.18258829}, "streams": {"cos_a2b/round0": 0.38674, "cos_c2b/round0": 0.290985, "cos_b2a/round0": 0.139058, "cos_b2c/round0": 0.144668, "rms_a/round0": 0.83879, "rms_b/round0": 1.60399, "rms_c/round0": 1.25558, "cos_a2b/round1": 0.410994, "cos_c2b/round1": 0.270794, "cos_b2a/round1": 0.232088, "cos_b2c/round1": 0.134665, "rms_a/round1": 1.50275, "rms_b/round1": 3.117056, "rms_c/round1": 2.047967, "cos_a2b/round2": 0.534013, "cos_c2b/round2": 0.486809, "cos_b2a/round2": 0.071646, "cos_b2c/round2": 0.137239, "rms_a/round2": 3.360777, "rms_b/round2": 6.961826, "rms_c/round2": 3.312696}, "alpha_mean": 0.43840664942186436, "alpha_hist": [0.01781, 0.07866, 0.15381, 0.18463, 0.19131, 0.16589, 0.12186, 0.07127, 0.0146, 0.00015], "alpha_by_type": {"special": {"share": 0.00127, "mean": 0.54876, "std": 0.12794}, "whitespace": {"share": 0.02567, "mean": 0.52796, "std": 0.11527}, "digit": {"share": 0.03207, "mean": 0.48043, "std": 0.12044}, "word_start": {"share": 0.59065, "mean": 0.45975, "std": 0.16807}, "word_cont": {"share": 0.24828, "mean": 0.31142, "std": 0.14265}, "punct": {"share": 0.10207, "mean": 0.58669, "std": 0.16706}}, "alpha_p05": 0.14416, "alpha_p50": 0.41395, "alpha_p95": 0.73564}
{"step": 19110, "total_steps": 72479, "tokens": 10019143680, "loss": 4.502947747707367, "l_mix": 2.7853834331035614, "l_a": 2.8878890573978424, "l_c": 2.831772267818451, "z_loss": 16.658509731292725, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17036233842372894, "tokens_per_second": 1350289.7323630017, "eta_seconds": 20722.016617155074, "data_wait_seconds": 2.0235776901245117e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20042, "finemath": 0.08237}}
{"step": 19120, "total_steps": 72479, "tokens": 10024386560, "loss": 4.965867817401886, "l_mix": 3.077066719532013, "l_a": 3.17031392455101, "l_c": 3.1201232969760895, "z_loss": 16.699198246002197, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19702154397964478, "tokens_per_second": 1414874.9432042132, "eta_seconds": 19772.407113695146, "data_wait_seconds": 2.053380012512207e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20769, "cosmopedia_v2": 0.20042, "finemath": 0.08232}}
{"step": 19130, "total_steps": 72479, "tokens": 10029629440, "loss": 4.927917957305908, "l_mix": 3.0533230304718018, "l_a": 3.1437701880931854, "l_c": 3.0993854105472565, "z_loss": 16.48084807395935, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16971881687641144, "tokens_per_second": 1436080.8765238198, "eta_seconds": 19476.786418676376, "data_wait_seconds": 3.069639205932617e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20768, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 19140, "total_steps": 72479, "tokens": 10034872320, "loss": 4.907591223716736, "l_mix": 3.0399003624916077, "l_a": 3.13274022936821, "l_c": 3.087289720773697, "z_loss": 16.817312955856323, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17168182134628296, "tokens_per_second": 1409611.3164835603, "eta_seconds": 19838.800458669662, "data_wait_seconds": 1.5407800674438477e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 19150, "total_steps": 72479, "tokens": 10040115200, "loss": 4.871539354324341, "l_mix": 3.017609864473343, "l_a": 3.110318958759308, "l_c": 3.0639489889144897, "z_loss": 16.490925312042236, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15151123702526093, "tokens_per_second": 1306968.3472445637, "eta_seconds": 21392.832359671593, "data_wait_seconds": 2.491474151611328e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 19160, "total_steps": 72479, "tokens": 10045358080, "loss": 4.831311583518982, "l_mix": 2.99351167678833, "l_a": 3.079793483018875, "l_c": 3.0409062206745148, "z_loss": 15.89786684513092, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19516244530677795, "tokens_per_second": 1470811.8210497766, "eta_seconds": 19006.17840564251, "data_wait_seconds": 2.0235776901245117e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08236}}
{"step": 19170, "total_steps": 72479, "tokens": 10050600960, "loss": 3.4975413978099823, "l_mix": 2.16293728351593, "l_a": 2.236064374446869, "l_c": 2.2065391689538956, "z_loss": 18.229349851608276, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.35317692160606384, "tokens_per_second": 1353694.0069245119, "eta_seconds": 20646.666712737082, "data_wait_seconds": 2.276897430419922e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08237}}
{"step": 19180, "total_steps": 72479, "tokens": 10055843840, "loss": 4.197849750518799, "l_mix": 2.593585819005966, "l_a": 2.699499934911728, "l_c": 2.6425438821315765, "z_loss": 16.506840229034424, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17987501621246338, "tokens_per_second": 1421061.731620693, "eta_seconds": 19664.18874719143, "data_wait_seconds": 5.21242618560791e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 19190, "total_steps": 72479, "tokens": 10061086720, "loss": 4.181147068738937, "l_mix": 2.5836348235607147, "l_a": 2.6913808286190033, "l_c": 2.6280675530433655, "z_loss": 16.777230501174927, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1694972664117813, "tokens_per_second": 1424316.6400214105, "eta_seconds": 19615.57033524513, "data_wait_seconds": 2.0384788513183594e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 19200, "total_steps": 72479, "tokens": 10066329600, "loss": 4.287619411945343, "l_mix": 2.6502727270126343, "l_a": 2.7573744356632233, "l_c": 2.6949115693569183, "z_loss": 16.608197450637817, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1673399806022644, "tokens_per_second": 1320294.9069048832, "eta_seconds": 21157.04620680809, "data_wait_seconds": 1.9311904907226562e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.29234323, "gate_a2b/L1_absmax": 0.65966856, "gate_a2b/L1_rms": 0.31137225, "gate_a2b/L3_absmean": 0.33152577, "gate_a2b/L3_absmax": 0.50911695, "gate_a2b/L3_rms": 0.33615109, "gate_a2b/L5_absmean": 0.51397175, "gate_a2b/L5_absmax": 0.68538582, "gate_a2b/L5_rms": 0.52000916, "gate_c2b/L1_absmean": 0.01447594, "gate_c2b/L1_absmax": 0.4455204, "gate_c2b/L1_rms": 0.05318165, "gate_c2b/L3_absmean": 0.2087882, "gate_c2b/L3_absmax": 0.43191287, "gate_c2b/L3_rms": 0.21302426, "gate_c2b/L5_absmean": 0.53216571, "gate_c2b/L5_absmax": 0.71390581, "gate_c2b/L5_rms": 0.5438906, "gate_b2a/L7_absmean": 0.08411978, "gate_b2a/L7_absmax": 0.26482269, "gate_b2a/L7_rms": 0.08729693, "gate_b2a/L11_absmean": 0.13081634, "gate_b2a/L11_absmax": 0.41912425, "gate_b2a/L11_rms": 0.13339825, "gate_b2a/L14_absmean": 0.31594592, "gate_b2a/L14_absmax": 0.51876044, "gate_b2a/L14_rms": 0.31724107, "gate_b2c/L3_absmean": 0.0938943, "gate_b2c/L3_absmax": 0.59046358, "gate_b2c/L3_rms": 0.11163121, "gate_b2c/L5_absmean": 0.12321141, "gate_b2c/L5_absmax": 0.74747562, "gate_b2c/L5_rms": 0.13048425, "gate_b2c/L6_absmean": 0.17857508, "gate_b2c/L6_absmax": 0.49392056, "gate_b2c/L6_rms": 0.18238825}, "streams": {"cos_a2b/round0": 0.380763, "cos_c2b/round0": 0.298471, "cos_b2a/round0": 0.138287, "cos_b2c/round0": 0.14165, "rms_a/round0": 0.838169, "rms_b/round0": 1.608967, "rms_c/round0": 1.263058, "cos_a2b/round1": 0.41818, "cos_c2b/round1": 0.273984, "cos_b2a/round1": 0.231001, "cos_b2c/round1": 0.132765, "rms_a/round1": 1.528829, "rms_b/round1": 3.164196, "rms_c/round1": 2.063013, "cos_a2b/round2": 0.517837, "cos_c2b/round2": 0.50075, "cos_b2a/round2": 0.068737, "cos_b2c/round2": 0.136531, "rms_a/round2": 3.391632, "rms_b/round2": 7.199296, "rms_c/round2": 3.424383}, "alpha_mean": 0.4195594717989408, "alpha_hist": [0.02509, 0.09947, 0.18436, 0.19044, 0.16783, 0.13179, 0.10612, 0.07158, 0.02255, 0.00076], "alpha_by_type": {"special": {"share": 0.00104, "mean": 0.55136, "std": 0.09703}, "whitespace": {"share": 0.02512, "mean": 0.51113, "std": 0.11495}, "digit": {"share": 0.02748, "mean": 0.36041, "std": 0.10194}, "word_start": {"share": 0.60007, "mean": 0.44798, "std": 0.17992}, "word_cont": {"share": 0.24474, "mean": 0.28096, "std": 0.13844}, "punct": {"share": 0.10156, "mean": 0.57766, "std": 0.18595}}, "alpha_p05": 0.11241, "alpha_p50": 0.38584, "alpha_p95": 0.75419}
{"step": 19210, "total_steps": 72479, "tokens": 10071572480, "loss": 4.122172296047211, "l_mix": 2.545707553625107, "l_a": 2.6562054455280304, "l_c": 2.593013435602188, "z_loss": 16.98935580253601, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18947833776474, "tokens_per_second": 1352928.6199283479, "eta_seconds": 20642.84623787403, "data_wait_seconds": 1.8358230590820312e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 19220, "total_steps": 72479, "tokens": 10076815360, "loss": 4.1429687440395355, "l_mix": 2.559246838092804, "l_a": 2.6684156954288483, "l_c": 2.6051509976387024, "z_loss": 16.51837944984436, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21406766772270203, "tokens_per_second": 1434858.9758986696, "eta_seconds": 19460.487100839615, "data_wait_seconds": 2.2739171981811523e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08236}}
{"step": 19230, "total_steps": 72479, "tokens": 10082058240, "loss": 3.5131328999996185, "l_mix": 2.17100390791893, "l_a": 2.2583247423171997, "l_c": 2.2091658115386963, "z_loss": 18.817914962768555, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2950901687145233, "tokens_per_second": 1399262.3856172366, "eta_seconds": 19951.80603649616, "data_wait_seconds": 3.743171691894531e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20764, "cosmopedia_v2": 0.20042, "finemath": 0.08237}}
{"step": 19240, "total_steps": 72479, "tokens": 10087301120, "loss": 4.95927894115448, "l_mix": 3.07268163561821, "l_a": 3.165389806032181, "l_c": 3.1177805066108704, "z_loss": 16.46116280555725, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16672374308109283, "tokens_per_second": 1332659.1459702668, "eta_seconds": 20945.01727347374, "data_wait_seconds": 3.0517578125e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20769, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 19250, "total_steps": 72479, "tokens": 10092544000, "loss": 4.780947387218475, "l_mix": 2.9603682458400726, "l_a": 3.0558106005191803, "l_c": 3.0073418617248535, "z_loss": 16.333545207977295, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16681048274040222, "tokens_per_second": 1343737.291414795, "eta_seconds": 20768.43898751736, "data_wait_seconds": 1.8090009689331055e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20769, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 19260, "total_steps": 72479, "tokens": 10097786880, "loss": 4.84948593378067, "l_mix": 3.0039324164390564, "l_a": 3.095942884683609, "l_c": 3.0505227744579315, "z_loss": 16.137187242507935, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15752339363098145, "tokens_per_second": 1438721.235400307, "eta_seconds": 19393.66875629425, "data_wait_seconds": 1.9043684005737305e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20768, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 19270, "total_steps": 72479, "tokens": 10103029760, "loss": 4.9422120451927185, "l_mix": 3.062136262655258, "l_a": 3.1527422964572906, "l_c": 3.108816474676132, "z_loss": 16.080420970916748, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1767747402191162, "tokens_per_second": 1365781.7804103252, "eta_seconds": 20425.54717900753, "data_wait_seconds": 2.199411392211914e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20768, "cosmopedia_v2": 0.20042, "finemath": 0.08236}}
{"step": 19280, "total_steps": 72479, "tokens": 10108272640, "loss": 3.3299255073070526, "l_mix": 2.057143956422806, "l_a": 2.1468828171491623, "l_c": 2.089467242360115, "z_loss": 18.765137434005737, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.3435492515563965, "tokens_per_second": 1386195.414516584, "eta_seconds": 20120.97069425583, "data_wait_seconds": 2.467632293701172e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20768, "cosmopedia_v2": 0.20041, "finemath": 0.08237}}
{"step": 19290, "total_steps": 72479, "tokens": 10113515520, "loss": 4.0616267919540405, "l_mix": 2.5082776844501495, "l_a": 2.618955373764038, "l_c": 2.553356498479843, "z_loss": 16.55394697189331, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1463368684053421, "tokens_per_second": 1316636.7451120003, "eta_seconds": 21179.991015386582, "data_wait_seconds": 1.8149614334106445e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08232}}
{"step": 19300, "total_steps": 72479, "tokens": 10118758400, "loss": 4.036219298839569, "l_mix": 2.492446392774582, "l_a": 2.6030041873455048, "l_c": 2.537301778793335, "z_loss": 16.80941081047058, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15915389358997345, "tokens_per_second": 1421880.5218860595, "eta_seconds": 19608.6176882267, "data_wait_seconds": 1.677870750427246e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.29214445, "gate_a2b/L1_absmax": 0.65416795, "gate_a2b/L1_rms": 0.31114709, "gate_a2b/L3_absmean": 0.3322745, "gate_a2b/L3_absmax": 0.51038599, "gate_a2b/L3_rms": 0.33685786, "gate_a2b/L5_absmean": 0.51436251, "gate_a2b/L5_absmax": 0.68744159, "gate_a2b/L5_rms": 0.52038628, "gate_c2b/L1_absmean": 0.01514811, "gate_c2b/L1_absmax": 0.44388056, "gate_c2b/L1_rms": 0.05321763, "gate_c2b/L3_absmean": 0.20939253, "gate_c2b/L3_absmax": 0.43194145, "gate_c2b/L3_rms": 0.21369384, "gate_c2b/L5_absmean": 0.53182852, "gate_c2b/L5_absmax": 0.7158736, "gate_c2b/L5_rms": 0.54341751, "gate_b2a/L7_absmean": 0.08383271, "gate_b2a/L7_absmax": 0.26638561, "gate_b2a/L7_rms": 0.08703691, "gate_b2a/L11_absmean": 0.13118389, "gate_b2a/L11_absmax": 0.42099547, "gate_b2a/L11_rms": 0.13378343, "gate_b2a/L14_absmean": 0.31645709, "gate_b2a/L14_absmax": 0.51527429, "gate_b2a/L14_rms": 0.31772768, "gate_b2c/L3_absmean": 0.0947059, "gate_b2c/L3_absmax": 0.59292799, "gate_b2c/L3_rms": 0.11223634, "gate_b2c/L5_absmean": 0.12353759, "gate_b2c/L5_absmax": 0.74866951, "gate_b2c/L5_rms": 0.13088587, "gate_b2c/L6_absmean": 0.17916918, "gate_b2c/L6_absmax": 0.49399793, "gate_b2c/L6_rms": 0.18298084}, "streams": {"cos_a2b/round0": 0.379724, "cos_c2b/round0": 0.297516, "cos_b2a/round0": 0.137648, "cos_b2c/round0": 0.14583, "rms_a/round0": 0.82988, "rms_b/round0": 1.593681, "rms_c/round0": 1.252128, "cos_a2b/round1": 0.418377, "cos_c2b/round1": 0.262225, "cos_b2a/round1": 0.229841, "cos_b2c/round1": 0.13588, "rms_a/round1": 1.513906, "rms_b/round1": 3.168277, "rms_c/round1": 2.049135, "cos_a2b/round2": 0.518139, "cos_c2b/round2": 0.493496, "cos_b2a/round2": 0.066709, "cos_b2c/round2": 0.129729, "rms_a/round2": 3.347138, "rms_b/round2": 7.073182, "rms_c/round2": 3.371495}, "alpha_mean": 0.4269888552184966, "alpha_hist": [0.02971, 0.10539, 0.17276, 0.17207, 0.15034, 0.14252, 0.12405, 0.08072, 0.022, 0.00043], "alpha_by_type": {"special": {"share": 0.00089, "mean": 0.62849, "std": 0.1091}, "whitespace": {"share": 0.02396, "mean": 0.58628, "std": 0.13077}, "digit": {"share": 0.02361, "mean": 0.51353, "std": 0.13725}, "word_start": {"share": 0.6004, "mean": 0.44373, "std": 0.18345}, "word_cont": {"share": 0.25879, "mean": 0.29742, "std": 0.15604}, "punct": {"share": 0.09236, "mean": 0.61585, "std": 0.17219}}, "alpha_p05": 0.1373, "alpha_p50": 0.45084, "alpha_p95": 0.75528}
{"step": 19310, "total_steps": 72479, "tokens": 10124001280, "loss": 4.014354407787323, "l_mix": 2.4788215458393097, "l_a": 2.5890853106975555, "l_c": 2.523789882659912, "z_loss": 16.702885627746582, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15677358210086823, "tokens_per_second": 1443343.8434993373, "eta_seconds": 19313.394239044188, "data_wait_seconds": 1.6242265701293945e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20041, "finemath": 0.08234}}
{"step": 19320, "total_steps": 72479, "tokens": 10129244160, "loss": 4.084596961736679, "l_mix": 2.5227317810058594, "l_a": 2.6328847408294678, "l_c": 2.5677742660045624, "z_loss": 16.674007415771484, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17243580520153046, "tokens_per_second": 1448975.3602678685, "eta_seconds": 19234.713409376145, "data_wait_seconds": 1.5556812286376953e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 19330, "total_steps": 72479, "tokens": 10134487040, "loss": 4.049070566892624, "l_mix": 2.5005816221237183, "l_a": 2.610625982284546, "l_c": 2.5454623997211456, "z_loss": 16.624775171279907, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.206029012799263, "tokens_per_second": 1435581.088911014, "eta_seconds": 19410.525206303595, "data_wait_seconds": 2.014636993408203e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20041, "finemath": 0.08236}}
{"step": 19340, "total_steps": 72479, "tokens": 10139729920, "loss": 3.2874269783496857, "l_mix": 2.032164141535759, "l_a": 2.108127638697624, "l_c": 2.0699362456798553, "z_loss": 18.435948610305786, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.31324103474617004, "tokens_per_second": 1368905.8689223307, "eta_seconds": 20352.12257065773, "data_wait_seconds": 2.8342008590698242e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20041, "finemath": 0.08237}}
{"step": 19350, "total_steps": 72479, "tokens": 10144972800, "loss": 3.455799847841263, "l_mix": 2.1246350407600403, "l_a": 2.2468777298927307, "l_c": 2.1851737797260284, "z_loss": 15.49364161491394, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20713068544864655, "tokens_per_second": 1382570.8110839133, "eta_seconds": 20147.17577478886, "data_wait_seconds": 3.2007694244384766e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20047, "finemath": 0.08233}}
{"step": 19360, "total_steps": 72479, "tokens": 10150215680, "loss": 3.425153523683548, "l_mix": 2.105615884065628, "l_a": 2.2313205897808075, "l_c": 2.161934345960617, "z_loss": 15.611960411071777, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1801898032426834, "tokens_per_second": 1350372.733660371, "eta_seconds": 20623.679357409477, "data_wait_seconds": 2.3186206817626953e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20764, "cosmopedia_v2": 0.20046, "finemath": 0.08233}}
{"step": 19370, "total_steps": 72479, "tokens": 10155458560, "loss": 4.8329479694366455, "l_mix": 2.993239104747772, "l_a": 3.0875016152858734, "l_c": 3.0394075512886047, "z_loss": 16.361366748809814, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1577063947916031, "tokens_per_second": 1351052.5506895855, "eta_seconds": 20609.42143056393, "data_wait_seconds": 2.7120113372802734e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20769, "cosmopedia_v2": 0.20041, "finemath": 0.08234}}
{"step": 19380, "total_steps": 72479, "tokens": 10160701440, "loss": 4.043937653303146, "l_mix": 2.497383624315262, "l_a": 2.6049320101737976, "l_c": 2.544601947069168, "z_loss": 16.93786382675171, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17442384362220764, "tokens_per_second": 1228156.1999070877, "eta_seconds": 22667.44939618111, "data_wait_seconds": 2.345442771911621e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20764, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 19390, "total_steps": 72479, "tokens": 10165944320, "loss": 4.100113540887833, "l_mix": 2.5325211584568024, "l_a": 2.639898657798767, "l_c": 2.579835385084152, "z_loss": 16.721363306045532, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20140069723129272, "tokens_per_second": 1416592.0245565788, "eta_seconds": 19648.51216828823, "data_wait_seconds": 2.1696090698242188e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20763, "cosmopedia_v2": 0.20041, "finemath": 0.08236}}
{"step": 19400, "total_steps": 72479, "tokens": 10171187200, "loss": 4.169617384672165, "l_mix": 2.5779029428958893, "l_a": 2.678927183151245, "l_c": 2.621259331703186, "z_loss": 16.584258556365967, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2659198045730591, "tokens_per_second": 1439332.7690074025, "eta_seconds": 19334.43283667564, "data_wait_seconds": 1.5914440155029297e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20768, "cosmopedia_v2": 0.20041, "finemath": 0.08232}, "gates": {"gate_a2b/L1_absmean": 0.29298335, "gate_a2b/L1_absmax": 0.65076852, "gate_a2b/L1_rms": 0.31177542, "gate_a2b/L3_absmean": 0.33345118, "gate_a2b/L3_absmax": 0.5086422, "gate_a2b/L3_rms": 0.33793193, "gate_a2b/L5_absmean": 0.51525849, "gate_a2b/L5_absmax": 0.68738389, "gate_a2b/L5_rms": 0.52121806, "gate_c2b/L1_absmean": 0.01423252, "gate_c2b/L1_absmax": 0.44400236, "gate_c2b/L1_rms": 0.05328471, "gate_c2b/L3_absmean": 0.20936094, "gate_c2b/L3_absmax": 0.43758929, "gate_c2b/L3_rms": 0.21370468, "gate_c2b/L5_absmean": 0.53181297, "gate_c2b/L5_absmax": 0.72171342, "gate_c2b/L5_rms": 0.54345101, "gate_b2a/L7_absmean": 0.08373628, "gate_b2a/L7_absmax": 0.26380861, "gate_b2a/L7_rms": 0.08697621, "gate_b2a/L11_absmean": 0.13066697, "gate_b2a/L11_absmax": 0.42153883, "gate_b2a/L11_rms": 0.13325314, "gate_b2a/L14_absmean": 0.31666601, "gate_b2a/L14_absmax": 0.52207738, "gate_b2a/L14_rms": 0.31795031, "gate_b2c/L3_absmean": 0.09493005, "gate_b2c/L3_absmax": 0.59392732, "gate_b2c/L3_rms": 0.11233272, "gate_b2c/L5_absmean": 0.12361859, "gate_b2c/L5_absmax": 0.74419785, "gate_b2c/L5_rms": 0.13084671, "gate_b2c/L6_absmean": 0.17948286, "gate_b2c/L6_absmax": 0.50536412, "gate_b2c/L6_rms": 0.18335733}, "streams": {"cos_a2b/round0": 0.381173, "cos_c2b/round0": 0.305865, "cos_b2a/round0": 0.141116, "cos_b2c/round0": 0.149945, "rms_a/round0": 0.84105, "rms_b/round0": 1.595349, "rms_c/round0": 1.263122, "cos_a2b/round1": 0.426873, "cos_c2b/round1": 0.278916, "cos_b2a/round1": 0.236002, "cos_b2c/round1": 0.139269, "rms_a/round1": 1.529295, "rms_b/round1": 3.135015, "rms_c/round1": 2.075412, "cos_a2b/round2": 0.539877, "cos_c2b/round2": 0.489909, "cos_b2a/round2": 0.068524, "cos_b2c/round2": 0.141318, "rms_a/round2": 3.419744, "rms_b/round2": 7.044444, "rms_c/round2": 3.38414}, "alpha_mean": 0.4460708831217701, "alpha_hist": [0.0199, 0.07787, 0.15268, 0.17746, 0.17021, 0.16998, 0.13068, 0.08026, 0.02036, 0.00061], "alpha_by_type": {"special": {"share": 0.00079, "mean": 0.57899, "std": 0.09229}, "whitespace": {"share": 0.02118, "mean": 0.56395, "std": 0.11719}, "digit": {"share": 0.0143, "mean": 0.50139, "std": 0.14109}, "word_start": {"share": 0.62938, "mean": 0.46688, "std": 0.17331}, "word_cont": {"share": 0.23898, "mean": 0.31068, "std": 0.14492}, "punct": {"share": 0.09537, "mean": 0.61242, "std": 0.16586}}, "alpha_p05": 0.15053, "alpha_p50": 0.44517, "alpha_p95": 0.73048}
{"step": 19410, "total_steps": 72479, "tokens": 10176430080, "loss": 4.0368911921978, "l_mix": 2.492818146944046, "l_a": 2.6025748252868652, "l_c": 2.5387409329414368, "z_loss": 16.78303360939026, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16474035382270813, "tokens_per_second": 1391114.6452660377, "eta_seconds": 20000.824494719505, "data_wait_seconds": 1.9043684005737305e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20768, "cosmopedia_v2": 0.20041, "finemath": 0.08233}}
{"step": 19420, "total_steps": 72479, "tokens": 10181672960, "loss": 3.892509400844574, "l_mix": 2.4027320444583893, "l_a": 2.5115110278129578, "l_c": 2.4487912952899933, "z_loss": 16.86608362197876, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15836864709854126, "tokens_per_second": 1416984.5325769288, "eta_seconds": 19631.9694057703, "data_wait_seconds": 1.683831214904785e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08234}}
{"step": 19430, "total_steps": 72479, "tokens": 10186915840, "loss": 4.124445855617523, "l_mix": 2.547520726919174, "l_a": 2.6581554114818573, "l_c": 2.592666655778885, "z_loss": 16.78426456451416, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1704251617193222, "tokens_per_second": 1347550.291345381, "eta_seconds": 20639.640902924537, "data_wait_seconds": 1.8030405044555664e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 19440, "total_steps": 72479, "tokens": 10192158720, "loss": 4.047842442989349, "l_mix": 2.4999496936798096, "l_a": 2.6068533062934875, "l_c": 2.547210603952408, "z_loss": 16.735198497772217, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16950954496860504, "tokens_per_second": 1485739.6885600942, "eta_seconds": 18716.408699393272, "data_wait_seconds": 1.6182661056518555e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20041, "finemath": 0.08236}}
{"step": 19450, "total_steps": 72479, "tokens": 10197401600, "loss": 3.3791019916534424, "l_mix": 2.0881479680538177, "l_a": 2.167157679796219, "l_c": 2.130045548081398, "z_loss": 17.93031358718872, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2840353846549988, "tokens_per_second": 1438001.3206401686, "eta_seconds": 19334.104880809784, "data_wait_seconds": 2.0891427993774414e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20041, "finemath": 0.08237}}
{"step": 19460, "total_steps": 72479, "tokens": 10202644480, "loss": 3.4246917963027954, "l_mix": 2.10592058300972, "l_a": 2.229524999856949, "l_c": 2.16109237074852, "z_loss": 15.859343647956848, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19020502269268036, "tokens_per_second": 1286070.5675961834, "eta_seconds": 21614.074820137026, "data_wait_seconds": 2.0116567611694336e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20046, "finemath": 0.08232}}
{"step": 19470, "total_steps": 72479, "tokens": 10207887360, "loss": 3.421843647956848, "l_mix": 2.103404402732849, "l_a": 2.2256554067134857, "l_c": 2.1639020144939423, "z_loss": 15.7194105386734, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1842992901802063, "tokens_per_second": 1451768.9082661422, "eta_seconds": 19143.530649924276, "data_wait_seconds": 3.2067298889160156e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20765, "cosmopedia_v2": 0.20046, "finemath": 0.08233}}
{"step": 19480, "total_steps": 72479, "tokens": 10213130240, "loss": 3.9157865047454834, "l_mix": 2.418014407157898, "l_a": 2.524423748254776, "l_c": 2.462512493133545, "z_loss": 16.911922216415405, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1660880595445633, "tokens_per_second": 1442876.0996905952, "eta_seconds": 19257.88341629505, "data_wait_seconds": 2.2083520889282227e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20765, "cosmopedia_v2": 0.20041, "finemath": 0.08234}}
{"step": 19490, "total_steps": 72479, "tokens": 10218373120, "loss": 3.982018142938614, "l_mix": 2.458661198616028, "l_a": 2.568765014410019, "l_c": 2.503492832183838, "z_loss": 16.795103788375854, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.13639485836029053, "tokens_per_second": 1402062.8376061793, "eta_seconds": 19814.730186724664, "data_wait_seconds": 1.862645149230957e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20764, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 19500, "total_steps": 72479, "tokens": 10223616000, "loss": 4.180068343877792, "l_mix": 2.5831688344478607, "l_a": 2.69039323925972, "l_c": 2.627172112464905, "z_loss": 16.298670887947083, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21226905286312103, "tokens_per_second": 1396282.3695988287, "eta_seconds": 19893.006283521652, "data_wait_seconds": 1.6748905181884766e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20764, "cosmopedia_v2": 0.20041, "finemath": 0.08236}, "gates": {"gate_a2b/L1_absmean": 0.29444596, "gate_a2b/L1_absmax": 0.6605525, "gate_a2b/L1_rms": 0.31322378, "gate_a2b/L3_absmean": 0.33423954, "gate_a2b/L3_absmax": 0.50816941, "gate_a2b/L3_rms": 0.3387436, "gate_a2b/L5_absmean": 0.51567113, "gate_a2b/L5_absmax": 0.68523151, "gate_a2b/L5_rms": 0.52166367, "gate_c2b/L1_absmean": 0.01458834, "gate_c2b/L1_absmax": 0.4419196, "gate_c2b/L1_rms": 0.05307056, "gate_c2b/L3_absmean": 0.20948349, "gate_c2b/L3_absmax": 0.441286, "gate_c2b/L3_rms": 0.2138259, "gate_c2b/L5_absmean": 0.53104937, "gate_c2b/L5_absmax": 0.71837789, "gate_c2b/L5_rms": 0.542741, "gate_b2a/L7_absmean": 0.08432856, "gate_b2a/L7_absmax": 0.26271281, "gate_b2a/L7_rms": 0.08744591, "gate_b2a/L11_absmean": 0.13110025, "gate_b2a/L11_absmax": 0.4223915, "gate_b2a/L11_rms": 0.1336858, "gate_b2a/L14_absmean": 0.31649661, "gate_b2a/L14_absmax": 0.52086765, "gate_b2a/L14_rms": 0.31777591, "gate_b2c/L3_absmean": 0.09523673, "gate_b2c/L3_absmax": 0.59360468, "gate_b2c/L3_rms": 0.11284091, "gate_b2c/L5_absmean": 0.12402663, "gate_b2c/L5_absmax": 0.7477712, "gate_b2c/L5_rms": 0.13142286, "gate_b2c/L6_absmean": 0.17998192, "gate_b2c/L6_absmax": 0.49644771, "gate_b2c/L6_rms": 0.18382965}, "streams": {"cos_a2b/round0": 0.385419, "cos_c2b/round0": 0.29911, "cos_b2a/round0": 0.138351, "cos_b2c/round0": 0.143492, "rms_a/round0": 0.825517, "rms_b/round0": 1.581061, "rms_c/round0": 1.264722, "cos_a2b/round1": 0.41746, "cos_c2b/round1": 0.26566, "cos_b2a/round1": 0.23487, "cos_b2c/round1": 0.136829, "rms_a/round1": 1.488553, "rms_b/round1": 3.147373, "rms_c/round1": 2.058845, "cos_a2b/round2": 0.524901, "cos_c2b/round2": 0.491311, "cos_b2a/round2": 0.064531, "cos_b2c/round2": 0.134742, "rms_a/round2": 3.335134, "rms_b/round2": 7.011572, "rms_c/round2": 3.355192}, "alpha_mean": 0.4169800437637221, "alpha_hist": [0.02328, 0.10175, 0.186, 0.19974, 0.16458, 0.12689, 0.10217, 0.07561, 0.01941, 0.00056], "alpha_by_type": {"special": {"share": 0.00089, "mean": 0.55695, "std": 0.08956}, "whitespace": {"share": 0.02336, "mean": 0.53069, "std": 0.11229}, "digit": {"share": 0.0278, "mean": 0.44333, "std": 0.11969}, "word_start": {"share": 0.58595, "mean": 0.43187, "std": 0.1846}, "word_cont": {"share": 0.26071, "mean": 0.30279, "std": 0.13708}, "punct": {"share": 0.10129, "mean": 0.5901, "std": 0.17818}}, "alpha_p05": 0.12507, "alpha_p50": 0.38607, "alpha_p95": 0.75238}
{"step": 19510, "total_steps": 72479, "tokens": 10228858880, "loss": 4.019399911165237, "l_mix": 2.4822146892547607, "l_a": 2.5906961262226105, "l_c": 2.5276151299476624, "z_loss": 16.917580127716064, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17825089395046234, "tokens_per_second": 1443555.3226728875, "eta_seconds": 19237.92641391754, "data_wait_seconds": 1.677870750427246e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20041, "finemath": 0.08237}}
{"step": 19520, "total_steps": 72479, "tokens": 10234101760, "loss": 4.1414134204387665, "l_mix": 2.5583623349666595, "l_a": 2.666803330183029, "l_c": 2.6045685410499573, "z_loss": 16.395370721817017, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1729336827993393, "tokens_per_second": 1467554.185346196, "eta_seconds": 18919.756741690635, "data_wait_seconds": 2.893805503845215e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20768, "cosmopedia_v2": 0.20041, "finemath": 0.08233}}
{"step": 19530, "total_steps": 72479, "tokens": 10239344640, "loss": 4.863672256469727, "l_mix": 3.0123676657676697, "l_a": 3.106074571609497, "l_c": 3.059460788965225, "z_loss": 16.43850588798523, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16129644215106964, "tokens_per_second": 1419459.7780101993, "eta_seconds": 19557.10597937107, "data_wait_seconds": 2.3365020751953125e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20768, "cosmopedia_v2": 0.20041, "finemath": 0.08233}}
{"step": 19540, "total_steps": 72479, "tokens": 10244587520, "loss": 4.964644014835358, "l_mix": 3.075103670358658, "l_a": 3.167728364467621, "l_c": 3.1253076791763306, "z_loss": 16.29425024986267, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1671948879957199, "tokens_per_second": 1437474.166628099, "eta_seconds": 19308.369552898406, "data_wait_seconds": 6.565451622009277e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20768, "cosmopedia_v2": 0.20041, "finemath": 0.08234}}
{"step": 19550, "total_steps": 72479, "tokens": 10249830400, "loss": 4.7810617089271545, "l_mix": 2.960658311843872, "l_a": 3.0547992289066315, "l_c": 3.00762277841568, "z_loss": 16.766652822494507, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17827464640140533, "tokens_per_second": 1407161.8955287728, "eta_seconds": 19720.573475003242, "data_wait_seconds": 2.0295381546020508e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 19560, "total_steps": 72479, "tokens": 10255073280, "loss": 3.2908434569835663, "l_mix": 2.034898981451988, "l_a": 2.111679494380951, "l_c": 2.068375378847122, "z_loss": 19.279497623443604, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.39829009771347046, "tokens_per_second": 1401089.3055022215, "eta_seconds": 19802.30422360897, "data_wait_seconds": 3.120303153991699e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08236}}
{"step": 19570, "total_steps": 72479, "tokens": 10260316160, "loss": 3.4309646785259247, "l_mix": 2.108657121658325, "l_a": 2.23168882727623, "l_c": 2.1708783507347107, "z_loss": 15.373843312263489, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19030945003032684, "tokens_per_second": 1386832.1056952917, "eta_seconds": 20002.099517369268, "data_wait_seconds": 2.726912498474121e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20046, "finemath": 0.08232}}
{"step": 19580, "total_steps": 72479, "tokens": 10265559040, "loss": 3.389957547187805, "l_mix": 2.084017515182495, "l_a": 2.2067136764526367, "l_c": 2.141080915927887, "z_loss": 16.016239643096924, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18778377771377563, "tokens_per_second": 1356931.2571653111, "eta_seconds": 20438.994802093504, "data_wait_seconds": 2.771615982055664e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20766, "cosmopedia_v2": 0.20046, "finemath": 0.08233}}
{"step": 19590, "total_steps": 72479, "tokens": 10270801920, "loss": 3.895025283098221, "l_mix": 2.4049198627471924, "l_a": 2.5130997598171234, "l_c": 2.4483372569084167, "z_loss": 16.74227809906006, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18378350138664246, "tokens_per_second": 1395286.6557578696, "eta_seconds": 19873.384381318094, "data_wait_seconds": 2.0772218704223633e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20766, "cosmopedia_v2": 0.20041, "finemath": 0.08234}}
{"step": 19600, "total_steps": 72479, "tokens": 10276044800, "loss": 4.079409599304199, "l_mix": 2.5200438499450684, "l_a": 2.627771884202957, "l_c": 2.5645922124385834, "z_loss": 16.564117193222046, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16900143027305603, "tokens_per_second": 1403499.7869574954, "eta_seconds": 19753.351877665522, "data_wait_seconds": 1.615285873413086e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20765, "cosmopedia_v2": 0.20041, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.29535183, "gate_a2b/L1_absmax": 0.660514, "gate_a2b/L1_rms": 0.31390315, "gate_a2b/L3_absmean": 0.33423471, "gate_a2b/L3_absmax": 0.50931352, "gate_a2b/L3_rms": 0.33878347, "gate_a2b/L5_absmean": 0.51610178, "gate_a2b/L5_absmax": 0.68754667, "gate_a2b/L5_rms": 0.52207243, "gate_c2b/L1_absmean": 0.01442918, "gate_c2b/L1_absmax": 0.441677, "gate_c2b/L1_rms": 0.05316686, "gate_c2b/L3_absmean": 0.20958395, "gate_c2b/L3_absmax": 0.43841809, "gate_c2b/L3_rms": 0.21387435, "gate_c2b/L5_absmean": 0.53042763, "gate_c2b/L5_absmax": 0.71766591, "gate_c2b/L5_rms": 0.54203153, "gate_b2a/L7_absmean": 0.08402596, "gate_b2a/L7_absmax": 0.26253328, "gate_b2a/L7_rms": 0.08717161, "gate_b2a/L11_absmean": 0.1311253, "gate_b2a/L11_absmax": 0.42474666, "gate_b2a/L11_rms": 0.13374293, "gate_b2a/L14_absmean": 0.31669849, "gate_b2a/L14_absmax": 0.52177185, "gate_b2a/L14_rms": 0.31797257, "gate_b2c/L3_absmean": 0.09587246, "gate_b2c/L3_absmax": 0.59457874, "gate_b2c/L3_rms": 0.11320893, "gate_b2c/L5_absmean": 0.12410303, "gate_b2c/L5_absmax": 0.7563132, "gate_b2c/L5_rms": 0.1315529, "gate_b2c/L6_absmean": 0.18001905, "gate_b2c/L6_absmax": 0.50467378, "gate_b2c/L6_rms": 0.18392114}, "streams": {"cos_a2b/round0": 0.385734, "cos_c2b/round0": 0.307121, "cos_b2a/round0": 0.14093, "cos_b2c/round0": 0.147414, "rms_a/round0": 0.822018, "rms_b/round0": 1.574709, "rms_c/round0": 1.265974, "cos_a2b/round1": 0.421881, "cos_c2b/round1": 0.271685, "cos_b2a/round1": 0.232827, "cos_b2c/round1": 0.130099, "rms_a/round1": 1.479635, "rms_b/round1": 3.106114, "rms_c/round1": 2.050053, "cos_a2b/round2": 0.524668, "cos_c2b/round2": 0.495654, "cos_b2a/round2": 0.066957, "cos_b2c/round2": 0.129777, "rms_a/round2": 3.29759, "rms_b/round2": 6.969676, "rms_c/round2": 3.349135}, "alpha_mean": 0.4253301059707013, "alpha_hist": [0.02527, 0.09679, 0.17438, 0.18463, 0.1666, 0.14185, 0.11472, 0.07716, 0.01808, 0.00053], "alpha_by_type": {"special": {"share": 0.00108, "mean": 0.57436, "std": 0.09213}, "whitespace": {"share": 0.02449, "mean": 0.55223, "std": 0.12}, "digit": {"share": 0.02122, "mean": 0.45431, "std": 0.12905}, "word_start": {"share": 0.61047, "mean": 0.44147, "std": 0.18153}, "word_cont": {"share": 0.23912, "mean": 0.29907, "std": 0.14368}, "punct": {"share": 0.10361, "mean": 0.58415, "std": 0.17746}}, "alpha_p05": 0.14247, "alpha_p50": 0.40286, "alpha_p95": 0.71661}
{"step": 19610, "total_steps": 72479, "tokens": 10281287680, "loss": 4.179924637079239, "l_mix": 2.5834215581417084, "l_a": 2.6883909702301025, "l_c": 2.6277194917201996, "z_loss": 16.698075771331787, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2375490516424179, "tokens_per_second": 1426883.508847882, "eta_seconds": 19425.96021337509, "data_wait_seconds": 2.384185791015625e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20765, "cosmopedia_v2": 0.20041, "finemath": 0.08236}}
{"step": 19620, "total_steps": 72479, "tokens": 10286530560, "loss": 4.039247035980225, "l_mix": 2.4940205812454224, "l_a": 2.6043799221515656, "l_c": 2.5407059490680695, "z_loss": 17.006343126296997, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2089586704969406, "tokens_per_second": 1426326.5413614106, "eta_seconds": 19429.870081186295, "data_wait_seconds": 1.6868114471435547e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20041, "finemath": 0.08236}}
{"step": 19630, "total_steps": 72479, "tokens": 10291773440, "loss": 3.419550597667694, "l_mix": 2.1018151342868805, "l_a": 2.2273486256599426, "l_c": 2.1599285304546356, "z_loss": 15.5227609872818, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1830659657716751, "tokens_per_second": 1397822.3667850916, "eta_seconds": 19822.330197596548, "data_wait_seconds": 1.6629695892333984e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20046, "finemath": 0.08232}}
{"step": 19640, "total_steps": 72479, "tokens": 10297016320, "loss": 3.393287092447281, "l_mix": 2.0852855443954468, "l_a": 2.2086471617221832, "l_c": 2.1461190581321716, "z_loss": 15.715535402297974, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18871350586414337, "tokens_per_second": 1452376.8139918614, "eta_seconds": 19074.150292897222, "data_wait_seconds": 1.5527009963989258e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20764, "cosmopedia_v2": 0.20046, "finemath": 0.08233}}
{"step": 19650, "total_steps": 72479, "tokens": 10302259200, "loss": 4.819627225399017, "l_mix": 2.985850602388382, "l_a": 3.0758916437625885, "l_c": 3.0312028527259827, "z_loss": 16.48221218585968, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1524987518787384, "tokens_per_second": 1285915.8706198698, "eta_seconds": 21539.20904533863, "data_wait_seconds": 3.069639205932617e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20768, "cosmopedia_v2": 0.20041, "finemath": 0.08234}}
{"step": 19660, "total_steps": 72479, "tokens": 10307502080, "loss": 4.95522141456604, "l_mix": 3.07075497508049, "l_a": 3.159412831068039, "l_c": 3.1167191565036774, "z_loss": 16.267393469810486, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1818593293428421, "tokens_per_second": 1314834.5161947454, "eta_seconds": 21061.48532831669, "data_wait_seconds": 2.0295381546020508e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 19670, "total_steps": 72479, "tokens": 10312744960, "loss": 4.822461426258087, "l_mix": 2.9872153401374817, "l_a": 3.077199697494507, "l_c": 3.0349196195602417, "z_loss": 16.102229475975037, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1691744029521942, "tokens_per_second": 1397961.525093144, "eta_seconds": 19805.355508732795, "data_wait_seconds": 2.631545066833496e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.20041, "finemath": 0.08236}}
{"step": 19680, "total_steps": 72479, "tokens": 10317987840, "loss": 3.3706648647785187, "l_mix": 2.0718539357185364, "l_a": 2.19533109664917, "l_c": 2.128894567489624, "z_loss": 15.43163776397705, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17992490530014038, "tokens_per_second": 1368744.4902560615, "eta_seconds": 20224.287519741058, "data_wait_seconds": 5.009770393371582e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20046, "finemath": 0.08232}}
{"step": 19690, "total_steps": 72479, "tokens": 10323230720, "loss": 3.434381276369095, "l_mix": 2.111843705177307, "l_a": 2.2309440970420837, "l_c": 2.172370105981827, "z_loss": 15.433436512947083, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.23640970885753632, "tokens_per_second": 1346818.0645357277, "eta_seconds": 20549.649548649788, "data_wait_seconds": 2.3871660232543945e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20046, "finemath": 0.08233}}
{"step": 19700, "total_steps": 72479, "tokens": 10328473600, "loss": 4.0056789219379425, "l_mix": 2.4726352393627167, "l_a": 2.5821000933647156, "l_c": 2.522466391324997, "z_loss": 16.73696804046631, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2147679626941681, "tokens_per_second": 1408029.6511337687, "eta_seconds": 19652.5664993763, "data_wait_seconds": 1.6480684280395508e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20766, "cosmopedia_v2": 0.20041, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.29634827, "gate_a2b/L1_absmax": 0.65680581, "gate_a2b/L1_rms": 0.3147313, "gate_a2b/L3_absmean": 0.33502063, "gate_a2b/L3_absmax": 0.51198667, "gate_a2b/L3_rms": 0.33947727, "gate_a2b/L5_absmean": 0.5170387, "gate_a2b/L5_absmax": 0.68786281, "gate_a2b/L5_rms": 0.52294016, "gate_c2b/L1_absmean": 0.01510266, "gate_c2b/L1_absmax": 0.43846318, "gate_c2b/L1_rms": 0.05301953, "gate_c2b/L3_absmean": 0.20957637, "gate_c2b/L3_absmax": 0.43477896, "gate_c2b/L3_rms": 0.21379332, "gate_c2b/L5_absmean": 0.53095716, "gate_c2b/L5_absmax": 0.71634066, "gate_c2b/L5_rms": 0.54258782, "gate_b2a/L7_absmean": 0.08446227, "gate_b2a/L7_absmax": 0.26139566, "gate_b2a/L7_rms": 0.08760511, "gate_b2a/L11_absmean": 0.13120675, "gate_b2a/L11_absmax": 0.42285731, "gate_b2a/L11_rms": 0.13380438, "gate_b2a/L14_absmean": 0.3168619, "gate_b2a/L14_absmax": 0.52509987, "gate_b2a/L14_rms": 0.31815198, "gate_b2c/L3_absmean": 0.09629127, "gate_b2c/L3_absmax": 0.59541905, "gate_b2c/L3_rms": 0.1135795, "gate_b2c/L5_absmean": 0.12459135, "gate_b2c/L5_absmax": 0.75868851, "gate_b2c/L5_rms": 0.13205676, "gate_b2c/L6_absmean": 0.18019116, "gate_b2c/L6_absmax": 0.5020926, "gate_b2c/L6_rms": 0.18412396}, "streams": {"cos_a2b/round0": 0.379109, "cos_c2b/round0": 0.296024, "cos_b2a/round0": 0.140002, "cos_b2c/round0": 0.140575, "rms_a/round0": 0.826371, "rms_b/round0": 1.587067, "rms_c/round0": 1.261685, "cos_a2b/round1": 0.416042, "cos_c2b/round1": 0.266495, "cos_b2a/round1": 0.229093, "cos_b2c/round1": 0.130504, "rms_a/round1": 1.500618, "rms_b/round1": 3.14962, "rms_c/round1": 2.055551, "cos_a2b/round2": 0.515404, "cos_c2b/round2": 0.494987, "cos_b2a/round2": 0.055872, "cos_b2c/round2": 0.124559, "rms_a/round2": 3.311918, "rms_b/round2": 7.056187, "rms_c/round2": 3.380713}, "alpha_mean": 0.4052464746535449, "alpha_hist": [0.03754, 0.12201, 0.18063, 0.17854, 0.16393, 0.12637, 0.10634, 0.06793, 0.01646, 0.00024], "alpha_by_type": {"special": {"share": 0.00111, "mean": 0.54313, "std": 0.08823}, "whitespace": {"share": 0.02335, "mean": 0.52517, "std": 0.1224}, "digit": {"share": 0.01585, "mean": 0.42155, "std": 0.12716}, "word_start": {"share": 0.59859, "mean": 0.42291, "std": 0.18777}, "word_cont": {"share": 0.24153, "mean": 0.2783, "std": 0.14606}, "punct": {"share": 0.11957, "mean": 0.54641, "std": 0.17655}}, "alpha_p05": 0.11086, "alpha_p50": 0.38931, "alpha_p95": 0.7235}
{"step": 19710, "total_steps": 72479, "tokens": 10333716480, "loss": 4.132779538631439, "l_mix": 2.5533301532268524, "l_a": 2.660693258047104, "l_c": 2.598527252674103, "z_loss": 16.831867933273315, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17236289381980896, "tokens_per_second": 1450981.2102572927, "eta_seconds": 19067.203128767014, "data_wait_seconds": 1.570582389831543e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20766, "cosmopedia_v2": 0.20041, "finemath": 0.08234}}
{"step": 19720, "total_steps": 72479, "tokens": 10338959360, "loss": 4.044254869222641, "l_mix": 2.497548609972, "l_a": 2.6076328456401825, "l_c": 2.5424596071243286, "z_loss": 16.784079551696777, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16491293907165527, "tokens_per_second": 1347044.6957763275, "eta_seconds": 20534.515802431106, "data_wait_seconds": 1.8030405044555664e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 19730, "total_steps": 72479, "tokens": 10344202240, "loss": 4.155284017324448, "l_mix": 2.567306101322174, "l_a": 2.6750487089157104, "l_c": 2.612738251686096, "z_loss": 16.41815996170044, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1905703991651535, "tokens_per_second": 1402373.3694166967, "eta_seconds": 19720.616716718672, "data_wait_seconds": 2.5272369384765625e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20041, "finemath": 0.08236}}
{"step": 19740, "total_steps": 72479, "tokens": 10349445120, "loss": 3.4377762377262115, "l_mix": 2.1141785085201263, "l_a": 2.2313928306102753, "l_c": 2.1753218472003937, "z_loss": 15.832696080207825, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1949242353439331, "tokens_per_second": 1447820.8357794816, "eta_seconds": 19097.96029224396, "data_wait_seconds": 1.582503318786621e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20046, "finemath": 0.08232}}
{"step": 19750, "total_steps": 72479, "tokens": 10354688000, "loss": 3.3717430233955383, "l_mix": 2.0726346373558044, "l_a": 2.1944278478622437, "l_c": 2.130628615617752, "z_loss": 15.914628028869629, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18900856375694275, "tokens_per_second": 1404150.3253469376, "eta_seconds": 19688.192533922196, "data_wait_seconds": 2.7835369110107422e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20046, "finemath": 0.08233}}
{"step": 19760, "total_steps": 72479, "tokens": 10359930880, "loss": 3.432845860719681, "l_mix": 2.1104851365089417, "l_a": 2.233745276927948, "l_c": 2.16895192861557, "z_loss": 15.515376329421997, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18529103696346283, "tokens_per_second": 1289836.405163655, "eta_seconds": 21429.026938104627, "data_wait_seconds": 2.205371856689453e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20764, "cosmopedia_v2": 0.20046, "finemath": 0.08234}}
{"step": 19770, "total_steps": 72479, "tokens": 10365173760, "loss": 3.4380849301815033, "l_mix": 2.1138909459114075, "l_a": 2.236277550458908, "l_c": 2.1725167632102966, "z_loss": 15.555282473564148, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18815523386001587, "tokens_per_second": 1288497.7889236768, "eta_seconds": 21447.220499372484, "data_wait_seconds": 2.282857894897461e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20764, "cosmopedia_v2": 0.20046, "finemath": 0.08235}}
{"step": 19780, "total_steps": 72479, "tokens": 10370416640, "loss": 4.904189646244049, "l_mix": 3.038139909505844, "l_a": 3.1295306980609894, "l_c": 3.085225611925125, "z_loss": 16.226941347122192, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1709308922290802, "tokens_per_second": 1364919.2980925532, "eta_seconds": 20242.55452363491, "data_wait_seconds": 2.771615982055664e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.2004, "finemath": 0.08236}}
{"step": 19790, "total_steps": 72479, "tokens": 10375659520, "loss": 4.750906944274902, "l_mix": 2.942170649766922, "l_a": 3.0324691832065582, "l_c": 2.991033613681793, "z_loss": 16.853217124938965, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17810401320457458, "tokens_per_second": 1368319.2429584728, "eta_seconds": 20188.425014233588, "data_wait_seconds": 1.6897916793823242e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20768, "cosmopedia_v2": 0.2004, "finemath": 0.08236}}
{"step": 19800, "total_steps": 72479, "tokens": 10380902400, "loss": 3.452299863100052, "l_mix": 2.1220338344573975, "l_a": 2.247888505458832, "l_c": 2.1811640858650208, "z_loss": 15.502052903175354, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.28049373626708984, "tokens_per_second": 1394237.109959002, "eta_seconds": 19809.376292395595, "data_wait_seconds": 1.5914440155029297e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20768, "cosmopedia_v2": 0.20045, "finemath": 0.08232}, "gates": {"gate_a2b/L1_absmean": 0.29517576, "gate_a2b/L1_absmax": 0.64996994, "gate_a2b/L1_rms": 0.31355885, "gate_a2b/L3_absmean": 0.33497998, "gate_a2b/L3_absmax": 0.51097071, "gate_a2b/L3_rms": 0.33949, "gate_a2b/L5_absmean": 0.51685584, "gate_a2b/L5_absmax": 0.69071573, "gate_a2b/L5_rms": 0.52280259, "gate_c2b/L1_absmean": 0.01515963, "gate_c2b/L1_absmax": 0.44390655, "gate_c2b/L1_rms": 0.05275746, "gate_c2b/L3_absmean": 0.2094311, "gate_c2b/L3_absmax": 0.43540257, "gate_c2b/L3_rms": 0.2136267, "gate_c2b/L5_absmean": 0.53024715, "gate_c2b/L5_absmax": 0.7112065, "gate_c2b/L5_rms": 0.54189664, "gate_b2a/L7_absmean": 0.08452135, "gate_b2a/L7_absmax": 0.25887391, "gate_b2a/L7_rms": 0.08767357, "gate_b2a/L11_absmean": 0.13079223, "gate_b2a/L11_absmax": 0.41911638, "gate_b2a/L11_rms": 0.13337697, "gate_b2a/L14_absmean": 0.31642798, "gate_b2a/L14_absmax": 0.52305603, "gate_b2a/L14_rms": 0.31772739, "gate_b2c/L3_absmean": 0.09676482, "gate_b2c/L3_absmax": 0.59687358, "gate_b2c/L3_rms": 0.11377411, "gate_b2c/L5_absmean": 0.12492299, "gate_b2c/L5_absmax": 0.75624722, "gate_b2c/L5_rms": 0.13234368, "gate_b2c/L6_absmean": 0.18071119, "gate_b2c/L6_absmax": 0.4986217, "gate_b2c/L6_rms": 0.18459404}, "streams": {"cos_a2b/round0": 0.372536, "cos_c2b/round0": 0.255901, "cos_b2a/round0": 0.117945, "cos_b2c/round0": 0.105271, "rms_a/round0": 0.859087, "rms_b/round0": 1.582796, "rms_c/round0": 1.264151, "cos_a2b/round1": 0.411527, "cos_c2b/round1": 0.268916, "cos_b2a/round1": 0.216317, "cos_b2c/round1": 0.134641, "rms_a/round1": 1.536822, "rms_b/round1": 3.199467, "rms_c/round1": 2.10546, "cos_a2b/round2": 0.505254, "cos_c2b/round2": 0.468095, "cos_b2a/round2": 0.032825, "cos_b2c/round2": 0.112904, "rms_a/round2": 3.343223, "rms_b/round2": 7.070507, "rms_c/round2": 3.391714}, "alpha_mean": 0.4298111072158406, "alpha_hist": [0.05109, 0.13545, 0.15945, 0.14433, 0.12346, 0.1147, 0.11476, 0.10394, 0.04874, 0.00407], "alpha_by_type": {"special": {"share": 0.00113, "mean": 0.59587, "std": 0.11489}, "whitespace": {"share": 0.0358, "mean": 0.63652, "std": 0.16757}, "digit": {"share": 0.00916, "mean": 0.41883, "std": 0.17245}, "word_start": {"share": 0.57167, "mean": 0.45119, "std": 0.20861}, "word_cont": {"share": 0.25958, "mean": 0.26074, "std": 0.15606}, "punct": {"share": 0.12267, "mean": 0.62692, "std": 0.18491}}, "alpha_p05": 0.10822, "alpha_p50": 0.40969, "alpha_p95": 0.79181}
{"step": 19810, "total_steps": 72479, "tokens": 10386145280, "loss": 3.981958359479904, "l_mix": 2.458574503660202, "l_a": 2.566637307405472, "l_c": 2.5057272613048553, "z_loss": 16.744415521621704, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18130312860012054, "tokens_per_second": 1415417.990623325, "eta_seconds": 19509.236744856833, "data_wait_seconds": 1.65402889251709e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20767, "cosmopedia_v2": 0.2004, "finemath": 0.08233}}
{"step": 19820, "total_steps": 72479, "tokens": 10391388160, "loss": 4.073615372180939, "l_mix": 2.5159641802310944, "l_a": 2.6230593025684357, "l_c": 2.563509464263916, "z_loss": 16.804821968078613, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18490822613239288, "tokens_per_second": 1413622.1434733518, "eta_seconds": 19530.312197971343, "data_wait_seconds": 1.8209218978881836e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20767, "cosmopedia_v2": 0.2004, "finemath": 0.08234}}
{"step": 19830, "total_steps": 72479, "tokens": 10396631040, "loss": 3.9680879712104797, "l_mix": 2.4505113661289215, "l_a": 2.559022903442383, "l_c": 2.493990957736969, "z_loss": 16.722959756851196, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18012681603431702, "tokens_per_second": 1444492.0900620455, "eta_seconds": 19109.304302811623, "data_wait_seconds": 1.8656253814697266e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.2004, "finemath": 0.08235}}
{"step": 19840, "total_steps": 72479, "tokens": 10401873920, "loss": 4.019428610801697, "l_mix": 2.4816736578941345, "l_a": 2.590309351682663, "l_c": 2.5298828184604645, "z_loss": 16.97212839126587, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16426105797290802, "tokens_per_second": 1365132.9097422652, "eta_seconds": 20216.343650531766, "data_wait_seconds": 2.518296241760254e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.2004, "finemath": 0.08236}}
{"step": 19850, "total_steps": 72479, "tokens": 10407116800, "loss": 3.429619699716568, "l_mix": 2.1082897782325745, "l_a": 2.2297964692115784, "l_c": 2.1694010198116302, "z_loss": 15.706512570381165, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19204078614711761, "tokens_per_second": 1457284.523511768, "eta_seconds": 18934.362306618692, "data_wait_seconds": 3.8951635360717773e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20045, "finemath": 0.08232}}
{"step": 19860, "total_steps": 72479, "tokens": 10412359680, "loss": 3.4384427070617676, "l_mix": 2.1142785847187042, "l_a": 2.2354148030281067, "l_c": 2.1732951998710632, "z_loss": 15.510801196098328, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19907210767269135, "tokens_per_second": 1388834.5423889104, "eta_seconds": 19863.78465540409, "data_wait_seconds": 1.9669532775878906e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20045, "finemath": 0.08233}}
{"step": 19870, "total_steps": 72479, "tokens": 10417602560, "loss": 3.388215720653534, "l_mix": 2.082575261592865, "l_a": 2.2055486142635345, "l_c": 2.1413062810897827, "z_loss": 15.839179515838623, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20440170168876648, "tokens_per_second": 1425640.4166741762, "eta_seconds": 19347.28215432167, "data_wait_seconds": 2.4199485778808594e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20045, "finemath": 0.08234}}
{"step": 19880, "total_steps": 72479, "tokens": 10422845440, "loss": 3.4542453289031982, "l_mix": 2.1244971454143524, "l_a": 2.241848737001419, "l_c": 2.1853654086589813, "z_loss": 15.83868670463562, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22132115066051483, "tokens_per_second": 1445435.9646115794, "eta_seconds": 19078.689881229402, "data_wait_seconds": 1.850724220275879e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20045, "finemath": 0.08234}}
{"step": 19890, "total_steps": 72479, "tokens": 10428088320, "loss": 3.413154810667038, "l_mix": 2.097543179988861, "l_a": 2.22555348277092, "l_c": 2.154525935649872, "z_loss": 15.877626657485962, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21805629134178162, "tokens_per_second": 1351050.4755180445, "eta_seconds": 20407.662135219576, "data_wait_seconds": 2.446770668029785e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20764, "cosmopedia_v2": 0.20045, "finemath": 0.08235}}
{"step": 19900, "total_steps": 72479, "tokens": 10433331200, "loss": 3.381992906332016, "l_mix": 2.0792291909456253, "l_a": 2.1993037164211273, "l_c": 2.1380669474601746, "z_loss": 15.524205327033997, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21147140860557556, "tokens_per_second": 1334469.5455011071, "eta_seconds": 20657.300756645203, "data_wait_seconds": 1.6629695892333984e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20764, "cosmopedia_v2": 0.20045, "finemath": 0.08236}, "gates": {"gate_a2b/L1_absmean": 0.29864493, "gate_a2b/L1_absmax": 0.65667486, "gate_a2b/L1_rms": 0.31675392, "gate_a2b/L3_absmean": 0.33567879, "gate_a2b/L3_absmax": 0.51003313, "gate_a2b/L3_rms": 0.34014297, "gate_a2b/L5_absmean": 0.51709753, "gate_a2b/L5_absmax": 0.69641471, "gate_a2b/L5_rms": 0.52304536, "gate_c2b/L1_absmean": 0.01430327, "gate_c2b/L1_absmax": 0.43956068, "gate_c2b/L1_rms": 0.05244929, "gate_c2b/L3_absmean": 0.20965949, "gate_c2b/L3_absmax": 0.44061139, "gate_c2b/L3_rms": 0.21388799, "gate_c2b/L5_absmean": 0.53008157, "gate_c2b/L5_absmax": 0.71958381, "gate_c2b/L5_rms": 0.54169244, "gate_b2a/L7_absmean": 0.08483191, "gate_b2a/L7_absmax": 0.26391909, "gate_b2a/L7_rms": 0.0879786, "gate_b2a/L11_absmean": 0.13172442, "gate_b2a/L11_absmax": 0.42131898, "gate_b2a/L11_rms": 0.13424769, "gate_b2a/L14_absmean": 0.31748924, "gate_b2a/L14_absmax": 0.52522326, "gate_b2a/L14_rms": 0.31874773, "gate_b2c/L3_absmean": 0.09737572, "gate_b2c/L3_absmax": 0.59272569, "gate_b2c/L3_rms": 0.11434897, "gate_b2c/L5_absmean": 0.1253475, "gate_b2c/L5_absmax": 0.76088774, "gate_b2c/L5_rms": 0.13276027, "gate_b2c/L6_absmean": 0.18087777, "gate_b2c/L6_absmax": 0.50435185, "gate_b2c/L6_rms": 0.18482067}, "streams": {"cos_a2b/round0": 0.376096, "cos_c2b/round0": 0.259982, "cos_b2a/round0": 0.122174, "cos_b2c/round0": 0.112378, "rms_a/round0": 0.859965, "rms_b/round0": 1.588246, "rms_c/round0": 1.264931, "cos_a2b/round1": 0.414101, "cos_c2b/round1": 0.276224, "cos_b2a/round1": 0.223251, "cos_b2c/round1": 0.134466, "rms_a/round1": 1.525822, "rms_b/round1": 3.175193, "rms_c/round1": 2.136046, "cos_a2b/round2": 0.513507, "cos_c2b/round2": 0.4701, "cos_b2a/round2": 0.037742, "cos_b2c/round2": 0.118155, "rms_a/round2": 3.343423, "rms_b/round2": 7.024976, "rms_c/round2": 3.460387}, "alpha_mean": 0.43942543521108846, "alpha_hist": [0.04372, 0.13304, 0.16066, 0.14647, 0.11833, 0.10959, 0.11479, 0.10313, 0.06401, 0.00626], "alpha_by_type": {"special": {"share": 0.00117, "mean": 0.57272, "std": 0.10234}, "whitespace": {"share": 0.03336, "mean": 0.63048, "std": 0.17169}, "digit": {"share": 0.01105, "mean": 0.44619, "std": 0.15762}, "word_start": {"share": 0.58562, "mean": 0.44839, "std": 0.21666}, "word_cont": {"share": 0.25035, "mean": 0.28364, "std": 0.16348}, "punct": {"share": 0.11845, "mean": 0.66861, "std": 0.17319}}, "alpha_p05": 0.11359, "alpha_p50": 0.41158, "alpha_p95": 0.82219}
{"step": 19910, "total_steps": 72479, "tokens": 10438574080, "loss": 4.0305430591106415, "l_mix": 2.4883656203746796, "l_a": 2.600533366203308, "l_c": 2.5342988073825836, "z_loss": 17.27742838859558, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.258798211812973, "tokens_per_second": 1385104.0067643255, "eta_seconds": 19898.35834522247, "data_wait_seconds": 1.65402889251709e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20763, "cosmopedia_v2": 0.20045, "finemath": 0.08232}}
{"step": 19920, "total_steps": 72479, "tokens": 10443816960, "loss": 3.952790290117264, "l_mix": 2.4410766065120697, "l_a": 2.547743409872055, "l_c": 2.485737055540085, "z_loss": 16.69581151008606, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18314386904239655, "tokens_per_second": 1301494.9962526928, "eta_seconds": 21172.61539332867, "data_wait_seconds": 2.0951032638549805e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20768, "cosmopedia_v2": 0.2004, "finemath": 0.08233}}
{"step": 19930, "total_steps": 72479, "tokens": 10449059840, "loss": 4.081521421670914, "l_mix": 2.5209566056728363, "l_a": 2.6247286796569824, "l_c": 2.5716329216957092, "z_loss": 16.56193208694458, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18354851007461548, "tokens_per_second": 1393252.6958913798, "eta_seconds": 19774.453114819527, "data_wait_seconds": 1.96993350982666e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20768, "cosmopedia_v2": 0.2004, "finemath": 0.08234}}
{"step": 19940, "total_steps": 72479, "tokens": 10454302720, "loss": 4.001787424087524, "l_mix": 2.4713632464408875, "l_a": 2.581140100955963, "l_c": 2.514678508043289, "z_loss": 16.784640312194824, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15530064702033997, "tokens_per_second": 1447560.363617525, "eta_seconds": 19028.959292006493, "data_wait_seconds": 1.71661376953125e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.2004, "finemath": 0.08235}}
{"step": 19950, "total_steps": 72479, "tokens": 10459545600, "loss": 4.068044513463974, "l_mix": 2.513180762529373, "l_a": 2.6221978664398193, "l_c": 2.5551423132419586, "z_loss": 16.616576671600342, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18651430308818817, "tokens_per_second": 1326789.710008106, "eta_seconds": 20757.113312125206, "data_wait_seconds": 1.665949821472168e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.2004, "finemath": 0.08236}}
{"step": 19960, "total_steps": 72479, "tokens": 10464788480, "loss": 3.438570499420166, "l_mix": 2.1146136224269867, "l_a": 2.2327625155448914, "l_c": 2.175132542848587, "z_loss": 15.882790565490723, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21443821489810944, "tokens_per_second": 1444890.5298723523, "eta_seconds": 19056.863411259652, "data_wait_seconds": 1.8268823623657227e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20045, "finemath": 0.08231}}
{"step": 19970, "total_steps": 72479, "tokens": 10470031360, "loss": 3.397869735956192, "l_mix": 2.089822679758072, "l_a": 2.209041327238083, "l_c": 2.145851284265518, "z_loss": 15.791837453842163, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22126497328281403, "tokens_per_second": 1420685.685533031, "eta_seconds": 19377.853153824806, "data_wait_seconds": 2.7686357498168945e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20045, "finemath": 0.08232}}
{"step": 19980, "total_steps": 72479, "tokens": 10475274240, "loss": 3.425109475851059, "l_mix": 2.1059171557426453, "l_a": 2.225112736225128, "l_c": 2.167004406452179, "z_loss": 15.57224416732788, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20617681741714478, "tokens_per_second": 1437587.0285338068, "eta_seconds": 19146.385690522195, "data_wait_seconds": 1.8894672393798828e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20045, "finemath": 0.08233}}
{"step": 19990, "total_steps": 72479, "tokens": 10480517120, "loss": 3.385131299495697, "l_mix": 2.081113874912262, "l_a": 2.1996712684631348, "l_c": 2.141852706670761, "z_loss": 15.602310180664062, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1917763650417328, "tokens_per_second": 1353510.3683718932, "eta_seconds": 20331.8374761343, "data_wait_seconds": 1.6808509826660156e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20765, "cosmopedia_v2": 0.20045, "finemath": 0.08234}}
{"step": 20000, "total_steps": 72479, "tokens": 10485760000, "loss": 3.4256883561611176, "l_mix": 2.1063603460788727, "l_a": 2.230439990758896, "l_c": 2.1621024310588837, "z_loss": 15.652390122413635, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18839529156684875, "tokens_per_second": 1390289.3178240475, "eta_seconds": 19790.204527401926, "data_wait_seconds": 2.193450927734375e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20765, "cosmopedia_v2": 0.20045, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.29911542, "gate_a2b/L1_absmax": 0.65545642, "gate_a2b/L1_rms": 0.31694919, "gate_a2b/L3_absmean": 0.33628789, "gate_a2b/L3_absmax": 0.51282912, "gate_a2b/L3_rms": 0.34070152, "gate_a2b/L5_absmean": 0.51692241, "gate_a2b/L5_absmax": 0.68416905, "gate_a2b/L5_rms": 0.52280426, "gate_c2b/L1_absmean": 0.01370314, "gate_c2b/L1_absmax": 0.44040537, "gate_c2b/L1_rms": 0.05237863, "gate_c2b/L3_absmean": 0.2095098, "gate_c2b/L3_absmax": 0.43730354, "gate_c2b/L3_rms": 0.21380565, "gate_c2b/L5_absmean": 0.5302276, "gate_c2b/L5_absmax": 0.72603089, "gate_c2b/L5_rms": 0.54174089, "gate_b2a/L7_absmean": 0.08499447, "gate_b2a/L7_absmax": 0.26157132, "gate_b2a/L7_rms": 0.08809327, "gate_b2a/L11_absmean": 0.13154379, "gate_b2a/L11_absmax": 0.42092702, "gate_b2a/L11_rms": 0.13404678, "gate_b2a/L14_absmean": 0.31717947, "gate_b2a/L14_absmax": 0.5239988, "gate_b2a/L14_rms": 0.31843251, "gate_b2c/L3_absmean": 0.09741362, "gate_b2c/L3_absmax": 0.59371269, "gate_b2c/L3_rms": 0.11441013, "gate_b2c/L5_absmean": 0.12551951, "gate_b2c/L5_absmax": 0.75906509, "gate_b2c/L5_rms": 0.13298622, "gate_b2c/L6_absmean": 0.18093476, "gate_b2c/L6_absmax": 0.50080061, "gate_b2c/L6_rms": 0.18487412}, "streams": {"cos_a2b/round0": 0.370908, "cos_c2b/round0": 0.254881, "cos_b2a/round0": 0.117769, "cos_b2c/round0": 0.107775, "rms_a/round0": 0.848231, "rms_b/round0": 1.57761, "rms_c/round0": 1.261375, "cos_a2b/round1": 0.408001, "cos_c2b/round1": 0.269284, "cos_b2a/round1": 0.222206, "cos_b2c/round1": 0.136931, "rms_a/round1": 1.509241, "rms_b/round1": 3.163174, "rms_c/round1": 2.120141, "cos_a2b/round2": 0.510403, "cos_c2b/round2": 0.46804, "cos_b2a/round2": 0.038273, "cos_b2c/round2": 0.117242, "rms_a/round2": 3.318399, "rms_b/round2": 6.974236, "rms_c/round2": 3.422875}, "alpha_mean": 0.4336210991558005, "alpha_hist": [0.04272, 0.12611, 0.16727, 0.153, 0.12769, 0.11272, 0.11156, 0.10434, 0.05064, 0.00395], "alpha_by_type": {"special": {"share": 0.00104, "mean": 0.57792, "std": 0.10305}, "whitespace": {"share": 0.03584, "mean": 0.61385, "std": 0.17744}, "digit": {"share": 0.01147, "mean": 0.42838, "std": 0.15851}, "word_start": {"share": 0.56935, "mean": 0.45898, "std": 0.20683}, "word_cont": {"share": 0.26747, "mean": 0.26847, "std": 0.15316}, "punct": {"share": 0.11482, "mean": 0.63553, "std": 0.1796}}, "alpha_p05": 0.10858, "alpha_p50": 0.40616, "alpha_p95": 0.81047}
{"step": 20010, "total_steps": 72479, "tokens": 10491002880, "loss": 3.4055725634098053, "l_mix": 2.0942400991916656, "l_a": 2.2141577899456024, "l_c": 2.1518313884735107, "z_loss": 15.357280731201172, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22880114614963531, "tokens_per_second": 1337246.1846939411, "eta_seconds": 20571.281030273436, "data_wait_seconds": 1.5974044799804688e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20765, "cosmopedia_v2": 0.20045, "finemath": 0.08236}}
{"step": 20020, "total_steps": 72479, "tokens": 10496245760, "loss": 3.939191371202469, "l_mix": 2.432192772626877, "l_a": 2.54305499792099, "l_c": 2.474396675825119, "z_loss": 17.630760431289673, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.32647472620010376, "tokens_per_second": 1443288.61519333, "eta_seconds": 19056.219180607797, "data_wait_seconds": 2.288818359375e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20764, "cosmopedia_v2": 0.20045, "finemath": 0.08232}}
{"step": 20030, "total_steps": 72479, "tokens": 10501488640, "loss": 4.0031441152095795, "l_mix": 2.471973568201065, "l_a": 2.581699937582016, "l_c": 2.5164803564548492, "z_loss": 17.164008378982544, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2888869643211365, "tokens_per_second": 1427092.1365147482, "eta_seconds": 19268.819866919515, "data_wait_seconds": 1.9371509552001953e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20764, "cosmopedia_v2": 0.20045, "finemath": 0.08233}}
{"step": 20040, "total_steps": 72479, "tokens": 10506731520, "loss": 4.128553003072739, "l_mix": 2.5503916144371033, "l_a": 2.658080518245697, "l_c": 2.5969863831996918, "z_loss": 16.413649797439575, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17747049033641815, "tokens_per_second": 1388796.6507757758, "eta_seconds": 19796.374376797678, "data_wait_seconds": 1.5556812286376953e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20763, "cosmopedia_v2": 0.20045, "finemath": 0.08234}}
{"step": 20050, "total_steps": 72479, "tokens": 10511974400, "loss": 3.9309966266155243, "l_mix": 2.4262846410274506, "l_a": 2.5386877357959747, "l_c": 2.471468925476074, "z_loss": 16.64905548095703, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17321287095546722, "tokens_per_second": 1365295.9808396178, "eta_seconds": 20133.286802101135, "data_wait_seconds": 1.5079975128173828e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20763, "cosmopedia_v2": 0.20045, "finemath": 0.08234}}
{"step": 20060, "total_steps": 72479, "tokens": 10517217280, "loss": 5.0105286836624146, "l_mix": 3.104245662689209, "l_a": 3.1967055201530457, "l_c": 3.1521416306495667, "z_loss": 16.28683853149414, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.177678182721138, "tokens_per_second": 1329056.1761457669, "eta_seconds": 20678.322831845286, "data_wait_seconds": 6.434321403503418e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20768, "cosmopedia_v2": 0.2004, "finemath": 0.08235}}
{"step": 20070, "total_steps": 72479, "tokens": 10522460160, "loss": 4.773126602172852, "l_mix": 2.9551737010478973, "l_a": 3.0510534048080444, "l_c": 3.0032993853092194, "z_loss": 16.469836473464966, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.31589391827583313, "tokens_per_second": 1391098.1009541922, "eta_seconds": 19752.31637017727, "data_wait_seconds": 2.9414892196655273e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.2004, "finemath": 0.08236}}
{"step": 20080, "total_steps": 72479, "tokens": 10527703040, "loss": 3.4473758935928345, "l_mix": 2.120784491300583, "l_a": 2.2382088601589203, "l_c": 2.178447186946869, "z_loss": 15.945914626121521, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22569477558135986, "tokens_per_second": 1311256.3131058533, "eta_seconds": 20951.027375364305, "data_wait_seconds": 2.6881694793701172e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20045, "finemath": 0.08232}}
{"step": 20090, "total_steps": 72479, "tokens": 10532945920, "loss": 3.4077972769737244, "l_mix": 2.094807356595993, "l_a": 2.2165111899375916, "l_c": 2.154824048280716, "z_loss": 15.892412304878235, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21233834326267242, "tokens_per_second": 1405204.977269021, "eta_seconds": 19546.560449409484, "data_wait_seconds": 2.777576446533203e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20045, "finemath": 0.08233}}
{"step": 20100, "total_steps": 72479, "tokens": 10538188800, "loss": 3.4216607809066772, "l_mix": 2.1042909622192383, "l_a": 2.224169671535492, "l_c": 2.161742687225342, "z_loss": 15.96077585220337, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18693560361862183, "tokens_per_second": 1441704.5209330136, "eta_seconds": 19048.06481027603, "data_wait_seconds": 3.30507755279541e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20766, "cosmopedia_v2": 0.20045, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.29833469, "gate_a2b/L1_absmax": 0.64669013, "gate_a2b/L1_rms": 0.3163496, "gate_a2b/L3_absmean": 0.33643699, "gate_a2b/L3_absmax": 0.50814086, "gate_a2b/L3_rms": 0.3408958, "gate_a2b/L5_absmean": 0.51768917, "gate_a2b/L5_absmax": 0.69319826, "gate_a2b/L5_rms": 0.52359265, "gate_c2b/L1_absmean": 0.01464389, "gate_c2b/L1_absmax": 0.43908906, "gate_c2b/L1_rms": 0.05265909, "gate_c2b/L3_absmean": 0.20944415, "gate_c2b/L3_absmax": 0.43546522, "gate_c2b/L3_rms": 0.21362451, "gate_c2b/L5_absmean": 0.52959239, "gate_c2b/L5_absmax": 0.71934384, "gate_c2b/L5_rms": 0.54106861, "gate_b2a/L7_absmean": 0.08510407, "gate_b2a/L7_absmax": 0.2616277, "gate_b2a/L7_rms": 0.08819097, "gate_b2a/L11_absmean": 0.13211977, "gate_b2a/L11_absmax": 0.42367208, "gate_b2a/L11_rms": 0.13468832, "gate_b2a/L14_absmean": 0.31746277, "gate_b2a/L14_absmax": 0.52961463, "gate_b2a/L14_rms": 0.31876728, "gate_b2c/L3_absmean": 0.09749528, "gate_b2c/L3_absmax": 0.59409678, "gate_b2c/L3_rms": 0.11455781, "gate_b2c/L5_absmean": 0.12567377, "gate_b2c/L5_absmax": 0.7593171, "gate_b2c/L5_rms": 0.13310686, "gate_b2c/L6_absmean": 0.18155536, "gate_b2c/L6_absmax": 0.50410181, "gate_b2c/L6_rms": 0.18551052}, "streams": {"cos_a2b/round0": 0.378225, "cos_c2b/round0": 0.266399, "cos_b2a/round0": 0.122698, "cos_b2c/round0": 0.115095, "rms_a/round0": 0.84843, "rms_b/round0": 1.572557, "rms_c/round0": 1.256305, "cos_a2b/round1": 0.413216, "cos_c2b/round1": 0.27298, "cos_b2a/round1": 0.221869, "cos_b2c/round1": 0.135449, "rms_a/round1": 1.518641, "rms_b/round1": 3.168015, "rms_c/round1": 2.127652, "cos_a2b/round2": 0.512422, "cos_c2b/round2": 0.475278, "cos_b2a/round2": 0.034709, "cos_b2c/round2": 0.117474, "rms_a/round2": 3.320269, "rms_b/round2": 6.976608, "rms_c/round2": 3.45812}, "alpha_mean": 0.4346566185435705, "alpha_hist": [0.04521, 0.13173, 0.16455, 0.14766, 0.12343, 0.11563, 0.10739, 0.09618, 0.06157, 0.00665], "alpha_by_type": {"special": {"share": 0.00116, "mean": 0.65771, "std": 0.08591}, "whitespace": {"share": 0.03502, "mean": 0.63171, "std": 0.1902}, "digit": {"share": 0.0085, "mean": 0.43903, "std": 0.17546}, "word_start": {"share": 0.58574, "mean": 0.44247, "std": 0.21043}, "word_cont": {"share": 0.24829, "mean": 0.2795, "std": 0.1625}, "punct": {"share": 0.12129, "mean": 0.65524, "std": 0.18699}}, "alpha_p05": 0.08602, "alpha_p50": 0.39187, "alpha_p95": 0.81726}
{"step": 20110, "total_steps": 72479, "tokens": 10543431680, "loss": 3.4299624264240265, "l_mix": 2.1097614467144012, "l_a": 2.2261177599430084, "l_c": 2.1693831980228424, "z_loss": 15.506494283676147, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1770026534795761, "tokens_per_second": 1429455.9294439976, "eta_seconds": 19207.61438422203, "data_wait_seconds": 2.014636993408203e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20766, "cosmopedia_v2": 0.20045, "finemath": 0.08235}}
{"step": 20120, "total_steps": 72479, "tokens": 10548674560, "loss": 3.452635258436203, "l_mix": 2.123322993516922, "l_a": 2.2439701557159424, "l_c": 2.181847184896469, "z_loss": 15.670804500579834, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1864493191242218, "tokens_per_second": 1455580.9292134452, "eta_seconds": 18859.271127462387, "data_wait_seconds": 2.0831823348999023e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20765, "cosmopedia_v2": 0.20045, "finemath": 0.08236}}
{"step": 20130, "total_steps": 72479, "tokens": 10553917440, "loss": 3.9615272879600525, "l_mix": 2.4464230239391327, "l_a": 2.5551493763923645, "l_c": 2.489661306142807, "z_loss": 16.60931158065796, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18047113716602325, "tokens_per_second": 1381875.4997711352, "eta_seconds": 19861.378623867036, "data_wait_seconds": 1.710653305053711e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20765, "cosmopedia_v2": 0.20045, "finemath": 0.08231}}
{"step": 20140, "total_steps": 72479, "tokens": 10559160320, "loss": 3.95618137717247, "l_mix": 2.442192882299423, "l_a": 2.5546563863754272, "l_c": 2.48626309633255, "z_loss": 17.12628698348999, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18513457477092743, "tokens_per_second": 1403846.1758057745, "eta_seconds": 19546.806555390358, "data_wait_seconds": 1.7970800399780273e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20045, "finemath": 0.08232}}
{"step": 20150, "total_steps": 72479, "tokens": 10564403200, "loss": 4.038365840911865, "l_mix": 2.493528813123703, "l_a": 2.6044922471046448, "l_c": 2.5393594801425934, "z_loss": 16.813719749450684, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19711025059223175, "tokens_per_second": 1322959.0867960874, "eta_seconds": 20737.955561757088, "data_wait_seconds": 2.7179718017578125e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20045, "finemath": 0.08233}}
{"step": 20160, "total_steps": 72479, "tokens": 10569646080, "loss": 3.9636527001857758, "l_mix": 2.44721457362175, "l_a": 2.5576135218143463, "l_c": 2.4917314052581787, "z_loss": 16.34678816795349, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22280532121658325, "tokens_per_second": 1334014.0163728318, "eta_seconds": 20562.170663380624, "data_wait_seconds": 1.8090009689331055e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20764, "cosmopedia_v2": 0.20045, "finemath": 0.08234}}
{"step": 20170, "total_steps": 72479, "tokens": 10574888960, "loss": 3.3963654041290283, "l_mix": 2.0878762900829315, "l_a": 2.212362140417099, "l_c": 2.1440536081790924, "z_loss": 15.644325971603394, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17460379004478455, "tokens_per_second": 1441170.1152268532, "eta_seconds": 19029.66256532669, "data_wait_seconds": 2.866983413696289e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20764, "cosmopedia_v2": 0.20045, "finemath": 0.08235}}
{"step": 20180, "total_steps": 72479, "tokens": 10580131840, "loss": 3.3854224383831024, "l_mix": 2.080131381750107, "l_a": 2.203385651111603, "l_c": 2.1423440873622894, "z_loss": 15.719850659370422, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1915358155965805, "tokens_per_second": 1431759.698410399, "eta_seconds": 19151.07552087307, "data_wait_seconds": 3.445148468017578e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20763, "cosmopedia_v2": 0.20045, "finemath": 0.08236}}
{"step": 20190, "total_steps": 72479, "tokens": 10585374720, "loss": 4.957890748977661, "l_mix": 3.0722239315509796, "l_a": 3.1610802710056305, "l_c": 3.119095891714096, "z_loss": 16.139976978302002, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1833711415529251, "tokens_per_second": 1379627.46283609, "eta_seconds": 19870.94050421715, "data_wait_seconds": 2.22623348236084e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.20045, "finemath": 0.08232}}
{"step": 20200, "total_steps": 72479, "tokens": 10590617600, "loss": 3.4495063722133636, "l_mix": 2.122066766023636, "l_a": 2.2400968968868256, "l_c": 2.179403930902481, "z_loss": 15.893391966819763, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.25016355514526367, "tokens_per_second": 1432212.052905445, "eta_seconds": 19137.705409193037, "data_wait_seconds": 2.7835369110107422e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20045, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.30006048, "gate_a2b/L1_absmax": 0.64785969, "gate_a2b/L1_rms": 0.31782794, "gate_a2b/L3_absmean": 0.33702335, "gate_a2b/L3_absmax": 0.5076043, "gate_a2b/L3_rms": 0.34144819, "gate_a2b/L5_absmean": 0.51814377, "gate_a2b/L5_absmax": 0.69661838, "gate_a2b/L5_rms": 0.52401209, "gate_c2b/L1_absmean": 0.01405319, "gate_c2b/L1_absmax": 0.43796876, "gate_c2b/L1_rms": 0.05248099, "gate_c2b/L3_absmean": 0.20894995, "gate_c2b/L3_absmax": 0.43857622, "gate_c2b/L3_rms": 0.21314426, "gate_c2b/L5_absmean": 0.5291478, "gate_c2b/L5_absmax": 0.71946305, "gate_c2b/L5_rms": 0.54067206, "gate_b2a/L7_absmean": 0.08521576, "gate_b2a/L7_absmax": 0.26340458, "gate_b2a/L7_rms": 0.08825473, "gate_b2a/L11_absmean": 0.13169333, "gate_b2a/L11_absmax": 0.4248406, "gate_b2a/L11_rms": 0.13427061, "gate_b2a/L14_absmean": 0.31724522, "gate_b2a/L14_absmax": 0.53162789, "gate_b2a/L14_rms": 0.31854752, "gate_b2c/L3_absmean": 0.09814354, "gate_b2c/L3_absmax": 0.59432846, "gate_b2c/L3_rms": 0.11500612, "gate_b2c/L5_absmean": 0.12585658, "gate_b2c/L5_absmax": 0.7606231, "gate_b2c/L5_rms": 0.13329956, "gate_b2c/L6_absmean": 0.18178791, "gate_b2c/L6_absmax": 0.50685078, "gate_b2c/L6_rms": 0.18571049}, "streams": {"cos_a2b/round0": 0.376384, "cos_c2b/round0": 0.268832, "cos_b2a/round0": 0.122336, "cos_b2c/round0": 0.115405, "rms_a/round0": 0.843938, "rms_b/round0": 1.566248, "rms_c/round0": 1.261033, "cos_a2b/round1": 0.412174, "cos_c2b/round1": 0.270492, "cos_b2a/round1": 0.227441, "cos_b2c/round1": 0.130487, "rms_a/round1": 1.503383, "rms_b/round1": 3.173876, "rms_c/round1": 2.098103, "cos_a2b/round2": 0.506159, "cos_c2b/round2": 0.478722, "cos_b2a/round2": 0.041559, "cos_b2c/round2": 0.116894, "rms_a/round2": 3.308362, "rms_b/round2": 7.062815, "rms_c/round2": 3.437772}, "alpha_mean": 0.4055001283918145, "alpha_hist": [0.05106, 0.16057, 0.18561, 0.14699, 0.11371, 0.1039, 0.10257, 0.09589, 0.0376, 0.00212], "alpha_by_type": {"special": {"share": 0.00104, "mean": 0.60456, "std": 0.12533}, "whitespace": {"share": 0.03471, "mean": 0.62675, "std": 0.17285}, "digit": {"share": 0.00887, "mean": 0.37989, "std": 0.17873}, "word_start": {"share": 0.57335, "mean": 0.4123, "std": 0.20743}, "word_cont": {"share": 0.26193, "mean": 0.26242, "std": 0.15251}, "punct": {"share": 0.1201, "mean": 0.62128, "std": 0.18504}}, "alpha_p05": 0.09912, "alpha_p50": 0.38007, "alpha_p95": 0.78826}
{"step": 20210, "total_steps": 72479, "tokens": 10595860480, "loss": 3.43110191822052, "l_mix": 2.109504818916321, "l_a": 2.231564462184906, "l_c": 2.168433964252472, "z_loss": 15.97538161277771, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21638496220111847, "tokens_per_second": 1282589.550401416, "eta_seconds": 21366.156821894645, "data_wait_seconds": 2.110004425048828e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20045, "finemath": 0.08234}}
{"step": 20220, "total_steps": 72479, "tokens": 10601103360, "loss": 3.424854040145874, "l_mix": 2.1068247854709625, "l_a": 2.2240794897079468, "l_c": 2.164183408021927, "z_loss": 15.503462433815002, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19392268359661102, "tokens_per_second": 1348786.6143096834, "eta_seconds": 20313.64064657688, "data_wait_seconds": 2.917647361755371e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20045, "finemath": 0.08234}}
{"step": 20230, "total_steps": 72479, "tokens": 10606346240, "loss": 3.4077900648117065, "l_mix": 2.095183104276657, "l_a": 2.2146134972572327, "l_c": 2.1554251313209534, "z_loss": 15.953281283378601, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22489315271377563, "tokens_per_second": 1423614.6595878138, "eta_seconds": 19242.232107901575, "data_wait_seconds": 1.9341707229614258e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 20240, "total_steps": 72479, "tokens": 10611589120, "loss": 4.081405073404312, "l_mix": 2.521263301372528, "l_a": 2.625380277633667, "l_c": 2.5696266293525696, "z_loss": 16.397294521331787, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22136521339416504, "tokens_per_second": 1433631.5821929325, "eta_seconds": 19104.12770769596, "data_wait_seconds": 1.6361474990844727e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08231}}
{"step": 20250, "total_steps": 72479, "tokens": 10616832000, "loss": 4.050792783498764, "l_mix": 2.5021886825561523, "l_a": 2.6117523312568665, "l_c": 2.5446428656578064, "z_loss": 16.85483455657959, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21726089715957642, "tokens_per_second": 1389912.0727484548, "eta_seconds": 19701.273547363282, "data_wait_seconds": 2.440810203552246e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08232}}
{"step": 20260, "total_steps": 72479, "tokens": 10622074880, "loss": 4.183216363191605, "l_mix": 2.5843281745910645, "l_a": 2.6893958151340485, "l_c": 2.6348008513450623, "z_loss": 16.292102575302124, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2185206562280655, "tokens_per_second": 1400015.4169111415, "eta_seconds": 19555.352563476565, "data_wait_seconds": 2.181529998779297e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 20270, "total_steps": 72479, "tokens": 10627317760, "loss": 4.067737251520157, "l_mix": 2.5122947990894318, "l_a": 2.6193582117557526, "l_c": 2.5597851872444153, "z_loss": 16.993873596191406, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19161634147167206, "tokens_per_second": 1378650.2533279099, "eta_seconds": 19854.602083396912, "data_wait_seconds": 1.6510486602783203e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 20280, "total_steps": 72479, "tokens": 10632560640, "loss": 3.4276435375213623, "l_mix": 2.107970029115677, "l_a": 2.2262518107891083, "l_c": 2.1673623919487, "z_loss": 15.89251708984375, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2053072452545166, "tokens_per_second": 1464464.1480138707, "eta_seconds": 18687.59255671501, "data_wait_seconds": 1.9788742065429688e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 20290, "total_steps": 72479, "tokens": 10637803520, "loss": 3.436879873275757, "l_mix": 2.112663894891739, "l_a": 2.2360455095767975, "l_c": 2.1727707982063293, "z_loss": 15.710383653640747, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20127208530902863, "tokens_per_second": 1306611.5900463038, "eta_seconds": 20941.239646458624, "data_wait_seconds": 2.4527311325073242e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08236}}
{"step": 20300, "total_steps": 72479, "tokens": 10643046400, "loss": 3.4232565462589264, "l_mix": 2.105137288570404, "l_a": 2.2289301455020905, "l_c": 2.159688502550125, "z_loss": 15.336252331733704, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19542916119098663, "tokens_per_second": 1337574.8762129818, "eta_seconds": 20452.554872632027, "data_wait_seconds": 3.5762786865234375e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08236}, "gates": {"gate_a2b/L1_absmean": 0.30137375, "gate_a2b/L1_absmax": 0.66028905, "gate_a2b/L1_rms": 0.31904617, "gate_a2b/L3_absmean": 0.3377718, "gate_a2b/L3_absmax": 0.50843042, "gate_a2b/L3_rms": 0.34218708, "gate_a2b/L5_absmean": 0.51889718, "gate_a2b/L5_absmax": 0.69667059, "gate_a2b/L5_rms": 0.52476144, "gate_c2b/L1_absmean": 0.01414432, "gate_c2b/L1_absmax": 0.43486246, "gate_c2b/L1_rms": 0.05239941, "gate_c2b/L3_absmean": 0.2088733, "gate_c2b/L3_absmax": 0.43359882, "gate_c2b/L3_rms": 0.21301772, "gate_c2b/L5_absmean": 0.52846134, "gate_c2b/L5_absmax": 0.71917063, "gate_c2b/L5_rms": 0.5398888, "gate_b2a/L7_absmean": 0.08500016, "gate_b2a/L7_absmax": 0.26386636, "gate_b2a/L7_rms": 0.08815333, "gate_b2a/L11_absmean": 0.13196902, "gate_b2a/L11_absmax": 0.42694372, "gate_b2a/L11_rms": 0.13457225, "gate_b2a/L14_absmean": 0.31710064, "gate_b2a/L14_absmax": 0.53039783, "gate_b2a/L14_rms": 0.3184146, "gate_b2c/L3_absmean": 0.09839194, "gate_b2c/L3_absmax": 0.59716159, "gate_b2c/L3_rms": 0.1153284, "gate_b2c/L5_absmean": 0.12642887, "gate_b2c/L5_absmax": 0.76373816, "gate_b2c/L5_rms": 0.13391443, "gate_b2c/L6_absmean": 0.18214849, "gate_b2c/L6_absmax": 0.50922728, "gate_b2c/L6_rms": 0.18613969}, "streams": {"cos_a2b/round0": 0.372475, "cos_c2b/round0": 0.260992, "cos_b2a/round0": 0.11819, "cos_b2c/round0": 0.110292, "rms_a/round0": 0.843936, "rms_b/round0": 1.562147, "rms_c/round0": 1.260484, "cos_a2b/round1": 0.406904, "cos_c2b/round1": 0.271687, "cos_b2a/round1": 0.223354, "cos_b2c/round1": 0.134184, "rms_a/round1": 1.502562, "rms_b/round1": 3.153828, "rms_c/round1": 2.115357, "cos_a2b/round2": 0.515203, "cos_c2b/round2": 0.464958, "cos_b2a/round2": 0.035866, "cos_b2c/round2": 0.116581, "rms_a/round2": 3.312799, "rms_b/round2": 6.929353, "rms_c/round2": 3.419144}, "alpha_mean": 0.44327728461043847, "alpha_hist": [0.04128, 0.12422, 0.16295, 0.14984, 0.12437, 0.10976, 0.10857, 0.1057, 0.06721, 0.0061], "alpha_by_type": {"special": {"share": 0.00113, "mean": 0.5932, "std": 0.1197}, "whitespace": {"share": 0.03404, "mean": 0.63162, "std": 0.17385}, "digit": {"share": 0.00839, "mean": 0.41161, "std": 0.16898}, "word_start": {"share": 0.57224, "mean": 0.46634, "std": 0.21429}, "word_cont": {"share": 0.25644, "mean": 0.27492, "std": 0.15888}, "punct": {"share": 0.12776, "mean": 0.62849, "std": 0.19362}}, "alpha_p05": 0.1023, "alpha_p50": 0.40686, "alpha_p95": 0.81699}
{"step": 20310, "total_steps": 72479, "tokens": 10648289280, "loss": 4.990879952907562, "l_mix": 3.0934656262397766, "l_a": 3.18204864859581, "l_c": 3.1372168958187103, "z_loss": 16.345963835716248, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22576764225959778, "tokens_per_second": 1464403.8784196698, "eta_seconds": 18677.6210272789, "data_wait_seconds": 2.09808349609375e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20768, "cosmopedia_v2": 0.20044, "finemath": 0.08232}}
{"step": 20320, "total_steps": 72479, "tokens": 10653532160, "loss": 5.025003731250763, "l_mix": 3.1137699484825134, "l_a": 3.2022668421268463, "l_c": 3.1631580889225006, "z_loss": 16.062965869903564, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19807353615760803, "tokens_per_second": 1449885.813997771, "eta_seconds": 18861.02859134674, "data_wait_seconds": 2.6851892471313477e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20768, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 20330, "total_steps": 72479, "tokens": 10658775040, "loss": 5.097003817558289, "l_mix": 3.1599303781986237, "l_a": 3.245540201663971, "l_c": 3.205984443426132, "z_loss": 16.159669399261475, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20586661994457245, "tokens_per_second": 1437981.3855323994, "eta_seconds": 19013.524922561643, "data_wait_seconds": 3.603100776672363e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 20340, "total_steps": 72479, "tokens": 10664017920, "loss": 3.428876608610153, "l_mix": 2.108008027076721, "l_a": 2.2311633825302124, "l_c": 2.166505306959152, "z_loss": 15.678447961807251, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1876985728740692, "tokens_per_second": 1435370.2531061664, "eta_seconds": 19044.46046087742, "data_wait_seconds": 2.6226043701171875e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 20350, "total_steps": 72479, "tokens": 10669260800, "loss": 3.4272583425045013, "l_mix": 2.106797069311142, "l_a": 2.232085555791855, "l_c": 2.164119213819504, "z_loss": 15.997617840766907, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.23488348722457886, "tokens_per_second": 1408506.1055128372, "eta_seconds": 19403.96924445629, "data_wait_seconds": 1.8477439880371094e-05, "mix": {"fineweb_hq": 0.50953, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08236}}
{"step": 20360, "total_steps": 72479, "tokens": 10674503680, "loss": 4.479790270328522, "l_mix": 2.7709483206272125, "l_a": 2.871635854244232, "l_c": 2.8191491961479187, "z_loss": 16.063267588615417, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.23783987760543823, "tokens_per_second": 1422292.8927061788, "eta_seconds": 19212.1935025692, "data_wait_seconds": 1.9788742065429688e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08232}}
{"step": 20370, "total_steps": 72479, "tokens": 10679746560, "loss": 4.546295374631882, "l_mix": 2.8129525780677795, "l_a": 2.9155526757240295, "l_c": 2.8569739758968353, "z_loss": 15.847660303115845, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2192956954240799, "tokens_per_second": 1276098.0801002763, "eta_seconds": 21409.109392166138, "data_wait_seconds": 1.671910285949707e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 20380, "total_steps": 72479, "tokens": 10684989440, "loss": 4.655840218067169, "l_mix": 2.8810242116451263, "l_a": 2.9851895570755005, "l_c": 2.9253874123096466, "z_loss": 16.429092407226562, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19586065411567688, "tokens_per_second": 1441812.1870013908, "eta_seconds": 18944.82565639019, "data_wait_seconds": 1.96993350982666e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 20390, "total_steps": 72479, "tokens": 10690232320, "loss": 3.3910082578659058, "l_mix": 2.084494650363922, "l_a": 2.2084117233753204, "l_c": 2.1412960588932037, "z_loss": 16.01202642917633, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19471430778503418, "tokens_per_second": 1449223.639238208, "eta_seconds": 18844.32249970436, "data_wait_seconds": 3.314018249511719e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 20400, "total_steps": 72479, "tokens": 10695475200, "loss": 3.443973273038864, "l_mix": 2.117026597261429, "l_a": 2.2399147748947144, "l_c": 2.178063988685608, "z_loss": 15.530709028244019, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21398833394050598, "tokens_per_second": 1371836.9748089353, "eta_seconds": 19903.52735302448, "data_wait_seconds": 3.2454729080200195e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.30098632, "gate_a2b/L1_absmax": 0.65964317, "gate_a2b/L1_rms": 0.31854543, "gate_a2b/L3_absmean": 0.33692351, "gate_a2b/L3_absmax": 0.4993116, "gate_a2b/L3_rms": 0.34148616, "gate_a2b/L5_absmean": 0.51852024, "gate_a2b/L5_absmax": 0.69653475, "gate_a2b/L5_rms": 0.52441221, "gate_c2b/L1_absmean": 0.01526906, "gate_c2b/L1_absmax": 0.44520688, "gate_c2b/L1_rms": 0.05281103, "gate_c2b/L3_absmean": 0.20996796, "gate_c2b/L3_absmax": 0.42869446, "gate_c2b/L3_rms": 0.21415457, "gate_c2b/L5_absmean": 0.52815533, "gate_c2b/L5_absmax": 0.71801823, "gate_c2b/L5_rms": 0.5397011, "gate_b2a/L7_absmean": 0.08538702, "gate_b2a/L7_absmax": 0.26947781, "gate_b2a/L7_rms": 0.08876354, "gate_b2a/L11_absmean": 0.13253906, "gate_b2a/L11_absmax": 0.42860535, "gate_b2a/L11_rms": 0.13521592, "gate_b2a/L14_absmean": 0.31742269, "gate_b2a/L14_absmax": 0.53619266, "gate_b2a/L14_rms": 0.31880659, "gate_b2c/L3_absmean": 0.09865355, "gate_b2c/L3_absmax": 0.60241145, "gate_b2c/L3_rms": 0.11540017, "gate_b2c/L5_absmean": 0.12553954, "gate_b2c/L5_absmax": 0.75671715, "gate_b2c/L5_rms": 0.13297926, "gate_b2c/L6_absmean": 0.18161683, "gate_b2c/L6_absmax": 0.50453842, "gate_b2c/L6_rms": 0.18555287}, "streams": {"cos_a2b/round0": 0.373685, "cos_c2b/round0": 0.254661, "cos_b2a/round0": 0.115108, "cos_b2c/round0": 0.110206, "rms_a/round0": 0.847363, "rms_b/round0": 1.58548, "rms_c/round0": 1.26798, "cos_a2b/round1": 0.406973, "cos_c2b/round1": 0.266677, "cos_b2a/round1": 0.219696, "cos_b2c/round1": 0.136135, "rms_a/round1": 1.521591, "rms_b/round1": 3.224574, "rms_c/round1": 2.135314, "cos_a2b/round2": 0.509057, "cos_c2b/round2": 0.46803, "cos_b2a/round2": 0.035433, "cos_b2c/round2": 0.109473, "rms_a/round2": 3.351007, "rms_b/round2": 7.082774, "rms_c/round2": 3.45523}, "alpha_mean": 0.4434994773737486, "alpha_hist": [0.05083, 0.13335, 0.15843, 0.1387, 0.11739, 0.10571, 0.10252, 0.10326, 0.07619, 0.01363], "alpha_by_type": {"special": {"share": 0.00104, "mean": 0.60609, "std": 0.11364}, "whitespace": {"share": 0.03809, "mean": 0.64478, "std": 0.18053}, "digit": {"share": 0.01213, "mean": 0.45663, "std": 0.16303}, "word_start": {"share": 0.56279, "mean": 0.46449, "std": 0.22268}, "word_cont": {"share": 0.2663, "mean": 0.27126, "std": 0.16585}, "punct": {"share": 0.11966, "mean": 0.66128, "std": 0.1963}}, "alpha_p05": 0.09774, "alpha_p50": 0.39071, "alpha_p95": 0.84947}
{"step": 20410, "total_steps": 72479, "tokens": 10700718080, "loss": 3.4452545046806335, "l_mix": 2.119056671857834, "l_a": 2.2397681772708893, "l_c": 2.175544559955597, "z_loss": 16.03917694091797, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.24652867019176483, "tokens_per_second": 1298758.5237447342, "eta_seconds": 21019.420756745338, "data_wait_seconds": 2.7507543563842773e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08236}}
{"step": 20420, "total_steps": 72479, "tokens": 10705960960, "loss": 4.633608102798462, "l_mix": 2.868324190378189, "l_a": 2.9671491086483, "l_c": 2.9118246138095856, "z_loss": 15.917304992675781, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.23873986303806305, "tokens_per_second": 1430539.925564571, "eta_seconds": 19079.4458122015, "data_wait_seconds": 2.1725893020629883e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08232}}
{"step": 20430, "total_steps": 72479, "tokens": 10711203840, "loss": 4.738081336021423, "l_mix": 2.9332036077976227, "l_a": 3.032246232032776, "l_c": 2.978440970182419, "z_loss": 16.71578049659729, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.24363923072814941, "tokens_per_second": 1429426.9388239041, "eta_seconds": 19090.63371539116, "data_wait_seconds": 1.633167266845703e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 20440, "total_steps": 72479, "tokens": 10716446720, "loss": 4.6189329624176025, "l_mix": 2.858661413192749, "l_a": 2.9557502567768097, "l_c": 2.9064384400844574, "z_loss": 16.150289297103882, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2482631802558899, "tokens_per_second": 1422500.456082849, "eta_seconds": 19179.904734182357, "data_wait_seconds": 2.205371856689453e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20763, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 20450, "total_steps": 72479, "tokens": 10721689600, "loss": 4.697623550891876, "l_mix": 2.907809793949127, "l_a": 3.0068595707416534, "l_c": 2.9537642300128937, "z_loss": 16.26651954650879, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2005823850631714, "tokens_per_second": 1371443.8447719105, "eta_seconds": 19890.118327474596, "data_wait_seconds": 2.1845102310180664e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20763, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 20460, "total_steps": 72479, "tokens": 10726932480, "loss": 4.592584848403931, "l_mix": 2.8440164625644684, "l_a": 2.9293947219848633, "l_c": 2.893906980752945, "z_loss": 15.777485728263855, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.4731072783470154, "tokens_per_second": 1398498.0681037826, "eta_seconds": 19501.591095495223, "data_wait_seconds": 1.6003847122192383e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20762, "cosmopedia_v2": 0.20044, "finemath": 0.08236}}
{"step": 20470, "total_steps": 72479, "tokens": 10732175360, "loss": 4.699526369571686, "l_mix": 2.9087804555892944, "l_a": 3.011796712875366, "l_c": 2.951805830001831, "z_loss": 16.64887237548828, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18875066936016083, "tokens_per_second": 1406831.7789586766, "eta_seconds": 19382.341940116883, "data_wait_seconds": 2.3603439331054688e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08232}}
{"step": 20480, "total_steps": 72479, "tokens": 10737418240, "loss": 4.705512881278992, "l_mix": 2.9124188125133514, "l_a": 3.015572488307953, "l_c": 2.9557204842567444, "z_loss": 17.06180238723755, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.30315253138542175, "tokens_per_second": 1352857.2890010974, "eta_seconds": 20151.757272291183, "data_wait_seconds": 2.002716064453125e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08232}}
{"step": 20490, "total_steps": 72479, "tokens": 10742661120, "loss": 4.6149375438690186, "l_mix": 2.856374680995941, "l_a": 2.954034745693207, "l_c": 2.90233650803566, "z_loss": 16.514036893844604, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2620714008808136, "tokens_per_second": 1372097.7027758323, "eta_seconds": 19865.355635285378, "data_wait_seconds": 3.37064266204834e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 20500, "total_steps": 72479, "tokens": 10747904000, "loss": 3.4340875446796417, "l_mix": 2.111075282096863, "l_a": 2.2371427416801453, "l_c": 2.1678001284599304, "z_loss": 15.293413996696472, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18547476828098297, "tokens_per_second": 1422961.6180720814, "eta_seconds": 19151.58188800812, "data_wait_seconds": 3.001093864440918e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.30064678, "gate_a2b/L1_absmax": 0.65992916, "gate_a2b/L1_rms": 0.31804782, "gate_a2b/L3_absmean": 0.33607978, "gate_a2b/L3_absmax": 0.50620955, "gate_a2b/L3_rms": 0.3406015, "gate_a2b/L5_absmean": 0.5171532, "gate_a2b/L5_absmax": 0.69373053, "gate_a2b/L5_rms": 0.52301127, "gate_c2b/L1_absmean": 0.01438302, "gate_c2b/L1_absmax": 0.44588935, "gate_c2b/L1_rms": 0.05290719, "gate_c2b/L3_absmean": 0.20950368, "gate_c2b/L3_absmax": 0.42932233, "gate_c2b/L3_rms": 0.21366157, "gate_c2b/L5_absmean": 0.52839625, "gate_c2b/L5_absmax": 0.72151208, "gate_c2b/L5_rms": 0.54001933, "gate_b2a/L7_absmean": 0.08534066, "gate_b2a/L7_absmax": 0.26743373, "gate_b2a/L7_rms": 0.0885843, "gate_b2a/L11_absmean": 0.13215631, "gate_b2a/L11_absmax": 0.43012398, "gate_b2a/L11_rms": 0.1347829, "gate_b2a/L14_absmean": 0.31779927, "gate_b2a/L14_absmax": 0.53573883, "gate_b2a/L14_rms": 0.31915128, "gate_b2c/L3_absmean": 0.09771919, "gate_b2c/L3_absmax": 0.60411781, "gate_b2c/L3_rms": 0.11469416, "gate_b2c/L5_absmean": 0.12468632, "gate_b2c/L5_absmax": 0.75894582, "gate_b2c/L5_rms": 0.13219871, "gate_b2c/L6_absmean": 0.18165144, "gate_b2c/L6_absmax": 0.50736731, "gate_b2c/L6_rms": 0.18564835}, "streams": {"cos_a2b/round0": 0.374187, "cos_c2b/round0": 0.261189, "cos_b2a/round0": 0.115807, "cos_b2c/round0": 0.105695, "rms_a/round0": 0.857218, "rms_b/round0": 1.591506, "rms_c/round0": 1.275543, "cos_a2b/round1": 0.410347, "cos_c2b/round1": 0.273182, "cos_b2a/round1": 0.226853, "cos_b2c/round1": 0.134339, "rms_a/round1": 1.515019, "rms_b/round1": 3.192927, "rms_c/round1": 2.112219, "cos_a2b/round2": 0.504744, "cos_c2b/round2": 0.473303, "cos_b2a/round2": 0.03744, "cos_b2c/round2": 0.113216, "rms_a/round2": 3.326087, "rms_b/round2": 7.046212, "rms_c/round2": 3.408723}, "alpha_mean": 0.42205996450949357, "alpha_hist": [0.05481, 0.15817, 0.16913, 0.13377, 0.11052, 0.10191, 0.10524, 0.09955, 0.05972, 0.00717], "alpha_by_type": {"special": {"share": 0.00102, "mean": 0.64813, "std": 0.0929}, "whitespace": {"share": 0.03609, "mean": 0.64139, "std": 0.1727}, "digit": {"share": 0.01225, "mean": 0.48131, "std": 0.16458}, "word_start": {"share": 0.56587, "mean": 0.4352, "std": 0.22073}, "word_cont": {"share": 0.26694, "mean": 0.25995, "std": 0.15862}, "punct": {"share": 0.11783, "mean": 0.65092, "std": 0.18836}}, "alpha_p05": 0.09138, "alpha_p50": 0.39373, "alpha_p95": 0.81257}
{"step": 20510, "total_steps": 72479, "tokens": 10753146880, "loss": 3.4475159347057343, "l_mix": 2.119652897119522, "l_a": 2.2446989119052887, "l_c": 2.176397144794464, "z_loss": 15.341482520103455, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2772335112094879, "tokens_per_second": 1381543.0784336873, "eta_seconds": 19721.94967882633, "data_wait_seconds": 2.619624137878418e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 20520, "total_steps": 72479, "tokens": 10758389760, "loss": 3.479575276374817, "l_mix": 2.138767182826996, "l_a": 2.2659089267253876, "l_c": 2.198313355445862, "z_loss": 15.413786768913269, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2889079749584198, "tokens_per_second": 1379022.619777885, "eta_seconds": 19754.193877100945, "data_wait_seconds": 2.73287296295166e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08236}}
{"step": 20530, "total_steps": 72479, "tokens": 10763632640, "loss": 4.7065547704696655, "l_mix": 2.913312315940857, "l_a": 3.010736733675003, "l_c": 2.9611411094665527, "z_loss": 16.79094934463501, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20460359752178192, "tokens_per_second": 1450559.1193569954, "eta_seconds": 18776.371778678895, "data_wait_seconds": 3.039836883544922e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08232}}
{"step": 20540, "total_steps": 72479, "tokens": 10768875520, "loss": 4.662966191768646, "l_mix": 2.8863906264305115, "l_a": 2.9835882782936096, "l_c": 2.9329878091812134, "z_loss": 16.025864005088806, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21817921102046967, "tokens_per_second": 1463483.5795431517, "eta_seconds": 18606.969570851328, "data_wait_seconds": 2.008676528930664e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 20550, "total_steps": 72479, "tokens": 10774118400, "loss": 4.620514750480652, "l_mix": 2.8600543439388275, "l_a": 2.959572046995163, "l_c": 2.9031496942043304, "z_loss": 16.43724536895752, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18147343397140503, "tokens_per_second": 1455059.5840611586, "eta_seconds": 18711.09049432278, "data_wait_seconds": 1.6629695892333984e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 20560, "total_steps": 72479, "tokens": 10779361280, "loss": 4.669762969017029, "l_mix": 2.8906218707561493, "l_a": 2.984113574028015, "l_c": 2.9408540427684784, "z_loss": 16.506391286849976, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18536800146102905, "tokens_per_second": 1445378.2010299345, "eta_seconds": 18832.793142032624, "data_wait_seconds": 1.7076730728149414e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 20570, "total_steps": 72479, "tokens": 10784604160, "loss": 4.662018179893494, "l_mix": 2.88518562912941, "l_a": 2.987667202949524, "l_c": 2.929642140865326, "z_loss": 16.397850513458252, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2050361931324005, "tokens_per_second": 1437884.2568064202, "eta_seconds": 18927.299372792244, "data_wait_seconds": 1.5914440155029297e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20763, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 20580, "total_steps": 72479, "tokens": 10789847040, "loss": 4.778116762638092, "l_mix": 2.958086460828781, "l_a": 3.054513067007065, "l_c": 3.0067054331302643, "z_loss": 16.645946979522705, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2776552140712738, "tokens_per_second": 1432357.4899499917, "eta_seconds": 18996.670246720314, "data_wait_seconds": 1.6689300537109375e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20763, "cosmopedia_v2": 0.20044, "finemath": 0.08236}}
{"step": 20590, "total_steps": 72479, "tokens": 10795089920, "loss": 4.912774324417114, "l_mix": 3.0438268780708313, "l_a": 3.13465878367424, "l_c": 3.0896321535110474, "z_loss": 16.60221028327942, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1816454529762268, "tokens_per_second": 1396513.7159833591, "eta_seconds": 19480.496124482153, "data_wait_seconds": 2.714991569519043e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08232}}
{"step": 20600, "total_steps": 72479, "tokens": 10800332800, "loss": 4.984189212322235, "l_mix": 3.0878494679927826, "l_a": 3.180807441473007, "l_c": 3.134847342967987, "z_loss": 16.43251371383667, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19910988211631775, "tokens_per_second": 1323890.1583455822, "eta_seconds": 20545.1615306139, "data_wait_seconds": 1.6450881958007812e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.30083627, "gate_a2b/L1_absmax": 0.66182315, "gate_a2b/L1_rms": 0.31829607, "gate_a2b/L3_absmean": 0.3355335, "gate_a2b/L3_absmax": 0.50437486, "gate_a2b/L3_rms": 0.34003857, "gate_a2b/L5_absmean": 0.51700395, "gate_a2b/L5_absmax": 0.68557775, "gate_a2b/L5_rms": 0.52285349, "gate_c2b/L1_absmean": 0.0144156, "gate_c2b/L1_absmax": 0.44362801, "gate_c2b/L1_rms": 0.05286335, "gate_c2b/L3_absmean": 0.20909138, "gate_c2b/L3_absmax": 0.43724191, "gate_c2b/L3_rms": 0.21324003, "gate_c2b/L5_absmean": 0.52870262, "gate_c2b/L5_absmax": 0.71959639, "gate_c2b/L5_rms": 0.54024297, "gate_b2a/L7_absmean": 0.08523224, "gate_b2a/L7_absmax": 0.26753703, "gate_b2a/L7_rms": 0.0884361, "gate_b2a/L11_absmean": 0.13206638, "gate_b2a/L11_absmax": 0.42793366, "gate_b2a/L11_rms": 0.13473575, "gate_b2a/L14_absmean": 0.31750745, "gate_b2a/L14_absmax": 0.53651416, "gate_b2a/L14_rms": 0.31888047, "gate_b2c/L3_absmean": 0.09760576, "gate_b2c/L3_absmax": 0.6021716, "gate_b2c/L3_rms": 0.11445798, "gate_b2c/L5_absmean": 0.12454055, "gate_b2c/L5_absmax": 0.75318927, "gate_b2c/L5_rms": 0.13200785, "gate_b2c/L6_absmean": 0.18200877, "gate_b2c/L6_absmax": 0.51171666, "gate_b2c/L6_rms": 0.18607788}, "streams": {"cos_a2b/round0": 0.384057, "cos_c2b/round0": 0.301329, "cos_b2a/round0": 0.129584, "cos_b2c/round0": 0.154729, "rms_a/round0": 0.826003, "rms_b/round0": 1.575367, "rms_c/round0": 1.235745, "cos_a2b/round1": 0.411666, "cos_c2b/round1": 0.281262, "cos_b2a/round1": 0.236967, "cos_b2c/round1": 0.14146, "rms_a/round1": 1.448098, "rms_b/round1": 3.025013, "rms_c/round1": 2.027263, "cos_a2b/round2": 0.549147, "cos_c2b/round2": 0.491964, "cos_b2a/round2": 0.09383, "cos_b2c/round2": 0.147568, "rms_a/round2": 3.265707, "rms_b/round2": 6.688918, "rms_c/round2": 3.249502}, "alpha_mean": 0.48721187088496976, "alpha_hist": [0.00786, 0.04343, 0.11325, 0.1553, 0.18484, 0.21202, 0.16721, 0.0966, 0.01912, 0.00037], "alpha_by_type": {"special": {"share": 0.0004, "mean": 0.59942, "std": 0.11651}, "whitespace": {"share": 0.07698, "mean": 0.55761, "std": 0.09277}, "digit": {"share": 0.02811, "mean": 0.52136, "std": 0.11536}, "word_start": {"share": 0.5587, "mean": 0.50686, "std": 0.16249}, "word_cont": {"share": 0.23663, "mean": 0.35666, "std": 0.14426}, "punct": {"share": 0.09918, "mean": 0.62326, "std": 0.15539}}, "alpha_p05": 0.20594, "alpha_p50": 0.48533, "alpha_p95": 0.76629}
{"step": 20610, "total_steps": 72479, "tokens": 10805575680, "loss": 4.93532532453537, "l_mix": 3.058143138885498, "l_a": 3.1479009091854095, "l_c": 3.103968024253845, "z_loss": 16.215258836746216, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19011764228343964, "tokens_per_second": 1454328.3273522868, "eta_seconds": 18698.86858458519, "data_wait_seconds": 1.7136335372924805e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 20620, "total_steps": 72479, "tokens": 10810818560, "loss": 3.450565069913864, "l_mix": 2.1216444969177246, "l_a": 2.244389593601227, "l_c": 2.18014332652092, "z_loss": 15.60646367073059, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21507731080055237, "tokens_per_second": 1412119.4240668584, "eta_seconds": 19254.073648881913, "data_wait_seconds": 2.440810203552246e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 20630, "total_steps": 72479, "tokens": 10816061440, "loss": 3.4944024085998535, "l_mix": 2.149113565683365, "l_a": 2.2710889875888824, "l_c": 2.2078493535518646, "z_loss": 16.072938561439514, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.24971379339694977, "tokens_per_second": 1404637.0773331085, "eta_seconds": 19352.905423521996, "data_wait_seconds": 2.154707908630371e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08236}}
{"step": 20640, "total_steps": 72479, "tokens": 10821304320, "loss": 4.663652658462524, "l_mix": 2.886852353811264, "l_a": 2.9825549721717834, "l_c": 2.934732288122177, "z_loss": 16.140756487846375, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2042866051197052, "tokens_per_second": 1476703.2914434846, "eta_seconds": 18404.89270219803, "data_wait_seconds": 1.582503318786621e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08232}}
{"step": 20650, "total_steps": 72479, "tokens": 10826547200, "loss": 4.807271480560303, "l_mix": 2.976827323436737, "l_a": 3.076445132493973, "l_c": 3.019500195980072, "z_loss": 16.604251384735107, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22027137875556946, "tokens_per_second": 1431974.0439391932, "eta_seconds": 18976.12800107002, "data_wait_seconds": 1.9431114196777344e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08232}}
{"step": 20660, "total_steps": 72479, "tokens": 10831790080, "loss": 4.709032356739044, "l_mix": 2.9156775772571564, "l_a": 3.0120943784713745, "l_c": 2.9601182341575623, "z_loss": 16.90926170349121, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22012990713119507, "tokens_per_second": 1446048.656656705, "eta_seconds": 18787.804785776138, "data_wait_seconds": 1.84476375579834e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 20670, "total_steps": 72479, "tokens": 10837032960, "loss": 4.652303695678711, "l_mix": 2.878869205713272, "l_a": 2.9806312024593353, "l_c": 2.925268203020096, "z_loss": 16.645342350006104, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20818686485290527, "tokens_per_second": 1362917.7706619776, "eta_seconds": 19929.91622583866, "data_wait_seconds": 1.5944242477416992e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 20680, "total_steps": 72479, "tokens": 10842275840, "loss": 4.676470220088959, "l_mix": 2.894368827342987, "l_a": 2.995958834886551, "l_c": 2.9388403594493866, "z_loss": 16.61545443534851, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.24353161454200745, "tokens_per_second": 1422858.6821556257, "eta_seconds": 19086.641879892348, "data_wait_seconds": 1.5497207641601562e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 20690, "total_steps": 72479, "tokens": 10847518720, "loss": 4.694212734699249, "l_mix": 2.90582999587059, "l_a": 3.001914143562317, "l_c": 2.953955590724945, "z_loss": 16.21733593940735, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2655071020126343, "tokens_per_second": 1428644.2650719727, "eta_seconds": 19005.67684750557, "data_wait_seconds": 4.750490188598633e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20764, "cosmopedia_v2": 0.20043, "finemath": 0.08236}}
{"step": 20700, "total_steps": 72479, "tokens": 10852761600, "loss": 4.9630491733551025, "l_mix": 3.074564665555954, "l_a": 3.1668766736984253, "l_c": 3.122698962688446, "z_loss": 16.118177890777588, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.40785473585128784, "tokens_per_second": 1411180.3288753384, "eta_seconds": 19237.164660334587, "data_wait_seconds": 1.856684684753418e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20768, "cosmopedia_v2": 0.20043, "finemath": 0.08232}, "gates": {"gate_a2b/L1_absmean": 0.30123624, "gate_a2b/L1_absmax": 0.66594338, "gate_a2b/L1_rms": 0.3185989, "gate_a2b/L3_absmean": 0.33451188, "gate_a2b/L3_absmax": 0.50560606, "gate_a2b/L3_rms": 0.33911312, "gate_a2b/L5_absmean": 0.51592463, "gate_a2b/L5_absmax": 0.68914646, "gate_a2b/L5_rms": 0.52183694, "gate_c2b/L1_absmean": 0.0146242, "gate_c2b/L1_absmax": 0.44311023, "gate_c2b/L1_rms": 0.05287071, "gate_c2b/L3_absmean": 0.20930243, "gate_c2b/L3_absmax": 0.43332645, "gate_c2b/L3_rms": 0.21347916, "gate_c2b/L5_absmean": 0.52852172, "gate_c2b/L5_absmax": 0.71326965, "gate_c2b/L5_rms": 0.54001063, "gate_b2a/L7_absmean": 0.08506718, "gate_b2a/L7_absmax": 0.27122313, "gate_b2a/L7_rms": 0.08842889, "gate_b2a/L11_absmean": 0.13091972, "gate_b2a/L11_absmax": 0.43280539, "gate_b2a/L11_rms": 0.13371682, "gate_b2a/L14_absmean": 0.31745934, "gate_b2a/L14_absmax": 0.53783518, "gate_b2a/L14_rms": 0.31885257, "gate_b2c/L3_absmean": 0.09772458, "gate_b2c/L3_absmax": 0.60087961, "gate_b2c/L3_rms": 0.11469574, "gate_b2c/L5_absmean": 0.12482619, "gate_b2c/L5_absmax": 0.76143724, "gate_b2c/L5_rms": 0.13244686, "gate_b2c/L6_absmean": 0.18139192, "gate_b2c/L6_absmax": 0.50650334, "gate_b2c/L6_rms": 0.18547928}, "streams": {"cos_a2b/round0": 0.379277, "cos_c2b/round0": 0.293389, "cos_b2a/round0": 0.131807, "cos_b2c/round0": 0.145674, "rms_a/round0": 0.830514, "rms_b/round0": 1.589825, "rms_c/round0": 1.244004, "cos_a2b/round1": 0.420052, "cos_c2b/round1": 0.286493, "cos_b2a/round1": 0.235115, "cos_b2c/round1": 0.131219, "rms_a/round1": 1.466315, "rms_b/round1": 3.026554, "rms_c/round1": 2.079671, "cos_a2b/round2": 0.546165, "cos_c2b/round2": 0.500473, "cos_b2a/round2": 0.085205, "cos_b2c/round2": 0.145655, "rms_a/round2": 3.25704, "rms_b/round2": 6.671929, "rms_c/round2": 3.308304}, "alpha_mean": 0.45227554368040046, "alpha_hist": [0.00859, 0.05945, 0.15396, 0.19562, 0.18156, 0.16931, 0.13974, 0.07887, 0.01279, 0.00011], "alpha_by_type": {"special": {"share": 0.00063, "mean": 0.5569, "std": 0.12698}, "whitespace": {"share": 0.05769, "mean": 0.45441, "std": 0.11835}, "digit": {"share": 0.01474, "mean": 0.39861, "std": 0.12366}, "word_start": {"share": 0.59373, "mean": 0.48349, "std": 0.16476}, "word_cont": {"share": 0.23064, "mean": 0.31966, "std": 0.13333}, "punct": {"share": 0.10257, "mean": 0.57565, "std": 0.16831}}, "alpha_p05": 0.18126, "alpha_p50": 0.43903, "alpha_p95": 0.72759}
{"step": 20710, "total_steps": 72479, "tokens": 10858004480, "loss": 4.932847201824188, "l_mix": 3.0561848878860474, "l_a": 3.1486217379570007, "l_c": 3.10139462351799, "z_loss": 16.573875665664673, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2686160206794739, "tokens_per_second": 1425977.0161081823, "eta_seconds": 19033.873032593725, "data_wait_seconds": 1.7344951629638672e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 20720, "total_steps": 72479, "tokens": 10863247360, "loss": 4.936439096927643, "l_mix": 3.0593740344047546, "l_a": 3.14666211605072, "l_c": 3.1047706305980682, "z_loss": 16.350867867469788, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1615394949913025, "tokens_per_second": 1373188.8527169058, "eta_seconds": 19761.755666971207, "data_wait_seconds": 2.2351741790771484e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 20730, "total_steps": 72479, "tokens": 10868490240, "loss": 4.876020312309265, "l_mix": 3.0211384296417236, "l_a": 3.1064305305480957, "l_c": 3.070950537919998, "z_loss": 16.675090074539185, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17417071759700775, "tokens_per_second": 1376538.3110235077, "eta_seconds": 19709.861683273313, "data_wait_seconds": 2.0951032638549805e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 20740, "total_steps": 72479, "tokens": 10873733120, "loss": 5.03387588262558, "l_mix": 3.119786649942398, "l_a": 3.2088878750801086, "l_c": 3.1659258902072906, "z_loss": 16.45066738128662, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20513229072093964, "tokens_per_second": 1448617.4153934692, "eta_seconds": 18725.53549594879, "data_wait_seconds": 2.3573637008666992e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 20750, "total_steps": 72479, "tokens": 10878976000, "loss": 4.7082929611206055, "l_mix": 2.9142205119132996, "l_a": 3.01398429274559, "l_c": 2.9606639742851257, "z_loss": 16.779122591018677, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20356284081935883, "tokens_per_second": 1456753.1756598528, "eta_seconds": 18617.357013630866, "data_wait_seconds": 2.0325183868408203e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08231}}
{"step": 20760, "total_steps": 72479, "tokens": 10884218880, "loss": 4.795975208282471, "l_mix": 2.9693102836608887, "l_a": 3.0696051120758057, "l_c": 3.013834536075592, "z_loss": 16.33100128173828, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2004970759153366, "tokens_per_second": 1471279.348870432, "eta_seconds": 18429.98142590523, "data_wait_seconds": 1.7315149307250977e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08232}}
{"step": 20770, "total_steps": 72479, "tokens": 10889461760, "loss": 4.813051164150238, "l_mix": 2.9806756377220154, "l_a": 3.0784159004688263, "l_c": 3.02401340007782, "z_loss": 16.465834856033325, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18393474817276, "tokens_per_second": 1352332.567933084, "eta_seconds": 20047.145824074745, "data_wait_seconds": 5.543231964111328e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 20780, "total_steps": 72479, "tokens": 10894704640, "loss": 4.816446781158447, "l_mix": 2.9822839498519897, "l_a": 3.0771773755550385, "l_c": 3.0313650369644165, "z_loss": 16.00063192844391, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22077316045761108, "tokens_per_second": 1376899.8813724697, "eta_seconds": 19685.647212767602, "data_wait_seconds": 2.3752450942993164e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 20790, "total_steps": 72479, "tokens": 10899947520, "loss": 4.6356281042099, "l_mix": 2.8692815005779266, "l_a": 2.970001667737961, "l_c": 2.9123109877109528, "z_loss": 16.527043342590332, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2029048353433609, "tokens_per_second": 1462725.444117395, "eta_seconds": 18527.0055573225, "data_wait_seconds": 1.868605613708496e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 20800, "total_steps": 72479, "tokens": 10905190400, "loss": 4.705214738845825, "l_mix": 2.9116873145103455, "l_a": 3.012437343597412, "l_c": 2.960573971271515, "z_loss": 16.241125345230103, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.30051684379577637, "tokens_per_second": 1430621.1728861707, "eta_seconds": 18939.10146551132, "data_wait_seconds": 1.627206802368164e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20764, "cosmopedia_v2": 0.20043, "finemath": 0.08236}, "gates": {"gate_a2b/L1_absmean": 0.30098405, "gate_a2b/L1_absmax": 0.66556656, "gate_a2b/L1_rms": 0.31835148, "gate_a2b/L3_absmean": 0.33417353, "gate_a2b/L3_absmax": 0.50681895, "gate_a2b/L3_rms": 0.33876687, "gate_a2b/L5_absmean": 0.51514459, "gate_a2b/L5_absmax": 0.69113111, "gate_a2b/L5_rms": 0.52097183, "gate_c2b/L1_absmean": 0.01373999, "gate_c2b/L1_absmax": 0.44324589, "gate_c2b/L1_rms": 0.05291519, "gate_c2b/L3_absmean": 0.20948279, "gate_c2b/L3_absmax": 0.43813336, "gate_c2b/L3_rms": 0.21363132, "gate_c2b/L5_absmean": 0.52984816, "gate_c2b/L5_absmax": 0.71246904, "gate_c2b/L5_rms": 0.54139256, "gate_b2a/L7_absmean": 0.08530912, "gate_b2a/L7_absmax": 0.26916331, "gate_b2a/L7_rms": 0.08856771, "gate_b2a/L11_absmean": 0.13138479, "gate_b2a/L11_absmax": 0.42867589, "gate_b2a/L11_rms": 0.13405783, "gate_b2a/L14_absmean": 0.31873643, "gate_b2a/L14_absmax": 0.53594112, "gate_b2a/L14_rms": 0.32011649, "gate_b2c/L3_absmean": 0.09801364, "gate_b2c/L3_absmax": 0.60236007, "gate_b2c/L3_rms": 0.11496515, "gate_b2c/L5_absmean": 0.12469523, "gate_b2c/L5_absmax": 0.758192, "gate_b2c/L5_rms": 0.13229287, "gate_b2c/L6_absmean": 0.18198082, "gate_b2c/L6_absmax": 0.49763814, "gate_b2c/L6_rms": 0.18600844}, "streams": {"cos_a2b/round0": 0.388032, "cos_c2b/round0": 0.294206, "cos_b2a/round0": 0.130524, "cos_b2c/round0": 0.145605, "rms_a/round0": 0.828489, "rms_b/round0": 1.582246, "rms_c/round0": 1.257652, "cos_a2b/round1": 0.410006, "cos_c2b/round1": 0.271396, "cos_b2a/round1": 0.223563, "cos_b2c/round1": 0.122571, "rms_a/round1": 1.485033, "rms_b/round1": 3.107477, "rms_c/round1": 2.062366, "cos_a2b/round2": 0.527749, "cos_c2b/round2": 0.492692, "cos_b2a/round2": 0.070606, "cos_b2c/round2": 0.132618, "rms_a/round2": 3.298893, "rms_b/round2": 6.934195, "rms_c/round2": 3.340884}, "alpha_mean": 0.44628096539638307, "alpha_hist": [0.01984, 0.08031, 0.14774, 0.17882, 0.17444, 0.16243, 0.13353, 0.0831, 0.01961, 0.0002], "alpha_by_type": {"special": {"share": 0.00104, "mean": 0.55753, "std": 0.11625}, "whitespace": {"share": 0.02785, "mean": 0.54606, "std": 0.13575}, "digit": {"share": 0.0533, "mean": 0.52878, "std": 0.12282}, "word_start": {"share": 0.55804, "mean": 0.45688, "std": 0.17751}, "word_cont": {"share": 0.24719, "mean": 0.3242, "std": 0.15363}, "punct": {"share": 0.11258, "mean": 0.59703, "std": 0.16583}}, "alpha_p05": 0.15103, "alpha_p50": 0.44432, "alpha_p95": 0.75923}
{"step": 20810, "total_steps": 72479, "tokens": 10910433280, "loss": 3.213371157646179, "l_mix": 1.985660269856453, "l_a": 2.064456522464752, "l_c": 2.0218422412872314, "z_loss": 18.211486101150513, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22823728621006012, "tokens_per_second": 1470413.6088329265, "eta_seconds": 18423.00459494591, "data_wait_seconds": 1.7911195755004883e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20764, "cosmopedia_v2": 0.20043, "finemath": 0.08236}}
{"step": 20820, "total_steps": 72479, "tokens": 10915676160, "loss": 4.953016638755798, "l_mix": 3.0689124166965485, "l_a": 3.1619175374507904, "l_c": 3.113033562898636, "z_loss": 16.187771439552307, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2157294899225235, "tokens_per_second": 1351345.379794706, "eta_seconds": 20042.39197244644, "data_wait_seconds": 2.0205974578857422e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.20043, "finemath": 0.08232}}
{"step": 20830, "total_steps": 72479, "tokens": 10920919040, "loss": 4.8756067752838135, "l_mix": 3.021023213863373, "l_a": 3.111194372177124, "l_c": 3.0652827620506287, "z_loss": 16.403576612472534, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17836272716522217, "tokens_per_second": 1429268.9977687001, "eta_seconds": 18946.014329195023, "data_wait_seconds": 2.5212764739990234e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20768, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 20840, "total_steps": 72479, "tokens": 10926161920, "loss": 4.927887737751007, "l_mix": 3.052733361721039, "l_a": 3.1424473226070404, "l_c": 3.102491408586502, "z_loss": 16.727254152297974, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18599598109722137, "tokens_per_second": 1441149.3365881075, "eta_seconds": 18786.191926574706, "data_wait_seconds": 1.773238182067871e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20768, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 20850, "total_steps": 72479, "tokens": 10931404800, "loss": 4.892550706863403, "l_mix": 3.0310466587543488, "l_a": 3.124301463365555, "l_c": 3.0752022862434387, "z_loss": 16.529046416282654, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19924066960811615, "tokens_per_second": 1336473.8549837277, "eta_seconds": 20253.643609309194, "data_wait_seconds": 2.9921531677246094e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 20860, "total_steps": 72479, "tokens": 10936647680, "loss": 4.794984579086304, "l_mix": 2.968811273574829, "l_a": 3.0652410089969635, "l_c": 3.016532927751541, "z_loss": 16.41081213951111, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17541256546974182, "tokens_per_second": 1451500.7861400896, "eta_seconds": 18644.993189406396, "data_wait_seconds": 1.8268823623657227e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08231}}
{"step": 20870, "total_steps": 72479, "tokens": 10941890560, "loss": 4.687794387340546, "l_mix": 2.901796340942383, "l_a": 2.999766319990158, "l_c": 2.94803249835968, "z_loss": 16.582773685455322, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20681877434253693, "tokens_per_second": 1472485.309654895, "eta_seconds": 18375.721112179755, "data_wait_seconds": 1.6808509826660156e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08232}}
{"step": 20880, "total_steps": 72479, "tokens": 10947133440, "loss": 4.651925086975098, "l_mix": 2.879489302635193, "l_a": 2.976891875267029, "l_c": 2.9256984889507294, "z_loss": 16.585747480392456, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1641547828912735, "tokens_per_second": 1384858.5460058495, "eta_seconds": 19534.6568716526, "data_wait_seconds": 1.8775463104248047e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 20890, "total_steps": 72479, "tokens": 10952376320, "loss": 4.778508543968201, "l_mix": 2.9594629108905792, "l_a": 3.0530883073806763, "l_c": 3.0047698616981506, "z_loss": 16.88051700592041, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16612400114536285, "tokens_per_second": 1331428.7396492197, "eta_seconds": 20314.638573241235, "data_wait_seconds": 2.104043960571289e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 20900, "total_steps": 72479, "tokens": 10957619200, "loss": 4.765956342220306, "l_mix": 2.9507464170455933, "l_a": 3.0512823164463043, "l_c": 2.9938811361789703, "z_loss": 16.60854732990265, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2195100486278534, "tokens_per_second": 1460152.9894486968, "eta_seconds": 18520.148879885674, "data_wait_seconds": 2.345442771911621e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.30115491, "gate_a2b/L1_absmax": 0.65455735, "gate_a2b/L1_rms": 0.31855848, "gate_a2b/L3_absmean": 0.33416298, "gate_a2b/L3_absmax": 0.50497031, "gate_a2b/L3_rms": 0.33875716, "gate_a2b/L5_absmean": 0.5152117, "gate_a2b/L5_absmax": 0.68517005, "gate_a2b/L5_rms": 0.52102, "gate_c2b/L1_absmean": 0.01427929, "gate_c2b/L1_absmax": 0.43792024, "gate_c2b/L1_rms": 0.05238627, "gate_c2b/L3_absmean": 0.20981346, "gate_c2b/L3_absmax": 0.43629053, "gate_c2b/L3_rms": 0.2139985, "gate_c2b/L5_absmean": 0.53016734, "gate_c2b/L5_absmax": 0.72336352, "gate_c2b/L5_rms": 0.54178405, "gate_b2a/L7_absmean": 0.08593353, "gate_b2a/L7_absmax": 0.26709926, "gate_b2a/L7_rms": 0.08913961, "gate_b2a/L11_absmean": 0.13248686, "gate_b2a/L11_absmax": 0.4298127, "gate_b2a/L11_rms": 0.13515128, "gate_b2a/L14_absmean": 0.31869572, "gate_b2a/L14_absmax": 0.54113102, "gate_b2a/L14_rms": 0.32007095, "gate_b2c/L3_absmean": 0.0988446, "gate_b2c/L3_absmax": 0.60284841, "gate_b2c/L3_rms": 0.11545386, "gate_b2c/L5_absmean": 0.12483956, "gate_b2c/L5_absmax": 0.75416172, "gate_b2c/L5_rms": 0.1322978, "gate_b2c/L6_absmean": 0.18224511, "gate_b2c/L6_absmax": 0.50366074, "gate_b2c/L6_rms": 0.18629885}, "streams": {"cos_a2b/round0": 0.380754, "cos_c2b/round0": 0.29444, "cos_b2a/round0": 0.134964, "cos_b2c/round0": 0.150669, "rms_a/round0": 0.823224, "rms_b/round0": 1.578279, "rms_c/round0": 1.246819, "cos_a2b/round1": 0.413263, "cos_c2b/round1": 0.276467, "cos_b2a/round1": 0.234519, "cos_b2c/round1": 0.137584, "rms_a/round1": 1.468942, "rms_b/round1": 3.080496, "rms_c/round1": 2.03925, "cos_a2b/round2": 0.545681, "cos_c2b/round2": 0.486659, "cos_b2a/round2": 0.080282, "cos_b2c/round2": 0.143981, "rms_a/round2": 3.330422, "rms_b/round2": 6.86506, "rms_c/round2": 3.287111}, "alpha_mean": 0.4469728942230091, "alpha_hist": [0.01453, 0.06741, 0.15326, 0.19569, 0.18652, 0.15128, 0.1292, 0.08716, 0.01476, 0.0002], "alpha_by_type": {"special": {"share": 0.00134, "mean": 0.49845, "std": 0.09176}, "whitespace": {"share": 0.02417, "mean": 0.46991, "std": 0.11004}, "digit": {"share": 0.03287, "mean": 0.39726, "std": 0.10789}, "word_start": {"share": 0.59659, "mean": 0.4809, "std": 0.16989}, "word_cont": {"share": 0.23946, "mean": 0.31118, "std": 0.13949}, "punct": {"share": 0.10558, "mean": 0.57284, "std": 0.17072}}, "alpha_p05": 0.16749, "alpha_p50": 0.43686, "alpha_p95": 0.75528}
{"step": 20910, "total_steps": 72479, "tokens": 10962862080, "loss": 4.727457880973816, "l_mix": 2.9262982606887817, "l_a": 3.0229081511497498, "l_c": 2.975432425737381, "z_loss": 16.57439923286438, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2200831025838852, "tokens_per_second": 1409959.101713592, "eta_seconds": 19175.739097070695, "data_wait_seconds": 3.838539123535156e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 20920, "total_steps": 72479, "tokens": 10968104960, "loss": 3.209535002708435, "l_mix": 1.9840127676725388, "l_a": 2.046069324016571, "l_c": 2.032716765999794, "z_loss": 18.862890005111694, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.3438883125782013, "tokens_per_second": 1430046.4045163756, "eta_seconds": 18902.718755578993, "data_wait_seconds": 2.1278858184814453e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08236}}
{"step": 20930, "total_steps": 72479, "tokens": 10973347840, "loss": 3.037274330854416, "l_mix": 1.8761374205350876, "l_a": 1.9466427862644196, "l_c": 1.9179407507181168, "z_loss": 17.617986917495728, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.37286192178726196, "tokens_per_second": 1415919.4248968286, "eta_seconds": 19087.6130638361, "data_wait_seconds": 2.5540590286254883e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20764, "cosmopedia_v2": 0.20043, "finemath": 0.08237}}
{"step": 20940, "total_steps": 72479, "tokens": 10978590720, "loss": 4.882542073726654, "l_mix": 3.0249379575252533, "l_a": 3.1130805015563965, "l_c": 3.0735729932785034, "z_loss": 16.08001947402954, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22455616295337677, "tokens_per_second": 1337209.3481267623, "eta_seconds": 20207.21682050228, "data_wait_seconds": 1.8149614334106445e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20769, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 20950, "total_steps": 72479, "tokens": 10983833600, "loss": 4.682935118675232, "l_mix": 2.9001955687999725, "l_a": 2.992927372455597, "l_c": 2.944275379180908, "z_loss": 15.786243200302124, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2047627866268158, "tokens_per_second": 1432008.173594804, "eta_seconds": 18865.839490413666, "data_wait_seconds": 1.8328428268432617e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20764, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 20960, "total_steps": 72479, "tokens": 10989076480, "loss": 4.683792889118195, "l_mix": 2.8988540172576904, "l_a": 2.99695947766304, "l_c": 2.947136700153351, "z_loss": 17.098485231399536, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16191470623016357, "tokens_per_second": 1391395.7827119606, "eta_seconds": 19412.732026076315, "data_wait_seconds": 2.187490463256836e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20763, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 20970, "total_steps": 72479, "tokens": 10994319360, "loss": 4.6837387681007385, "l_mix": 2.8995874226093292, "l_a": 2.9975718557834625, "l_c": 2.944035589694977, "z_loss": 16.69113564491272, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1906735599040985, "tokens_per_second": 1407304.180591247, "eta_seconds": 19189.561833500862, "data_wait_seconds": 2.1666288375854492e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20763, "cosmopedia_v2": 0.20043, "finemath": 0.08236}}
{"step": 20980, "total_steps": 72479, "tokens": 10999562240, "loss": 4.741339385509491, "l_mix": 2.934990704059601, "l_a": 3.0367586612701416, "l_c": 2.978933811187744, "z_loss": 16.408276319503784, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1658516526222229, "tokens_per_second": 1460076.3995998942, "eta_seconds": 18492.393767476082, "data_wait_seconds": 1.659989356994629e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08232}}
{"step": 20990, "total_steps": 72479, "tokens": 11004805120, "loss": 4.620993077754974, "l_mix": 2.8609625101089478, "l_a": 2.9558673202991486, "l_c": 2.905384838581085, "z_loss": 16.548494577407837, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1925610452890396, "tokens_per_second": 1470220.825335401, "eta_seconds": 18361.23143327236, "data_wait_seconds": 1.582503318786621e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08232}}
{"step": 21000, "total_steps": 72479, "tokens": 11010048000, "loss": 4.664673805236816, "l_mix": 2.8857762813568115, "l_a": 2.9904342889785767, "l_c": 2.93359637260437, "z_loss": 16.88197088241577, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17916272580623627, "tokens_per_second": 1293416.4888519146, "eta_seconds": 20867.07737579346, "data_wait_seconds": 2.0712614059448242e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.30141708, "gate_a2b/L1_absmax": 0.65347201, "gate_a2b/L1_rms": 0.31870803, "gate_a2b/L3_absmean": 0.33358899, "gate_a2b/L3_absmax": 0.50474805, "gate_a2b/L3_rms": 0.33821106, "gate_a2b/L5_absmean": 0.51460135, "gate_a2b/L5_absmax": 0.6871177, "gate_a2b/L5_rms": 0.52044088, "gate_c2b/L1_absmean": 0.01484851, "gate_c2b/L1_absmax": 0.43801653, "gate_c2b/L1_rms": 0.05238888, "gate_c2b/L3_absmean": 0.20964964, "gate_c2b/L3_absmax": 0.432836, "gate_c2b/L3_rms": 0.21381374, "gate_c2b/L5_absmean": 0.52915162, "gate_c2b/L5_absmax": 0.71498716, "gate_c2b/L5_rms": 0.54059613, "gate_b2a/L7_absmean": 0.08560204, "gate_b2a/L7_absmax": 0.27130356, "gate_b2a/L7_rms": 0.08893847, "gate_b2a/L11_absmean": 0.132357, "gate_b2a/L11_absmax": 0.43441966, "gate_b2a/L11_rms": 0.1351196, "gate_b2a/L14_absmean": 0.31770587, "gate_b2a/L14_absmax": 0.53730321, "gate_b2a/L14_rms": 0.3191697, "gate_b2c/L3_absmean": 0.09817824, "gate_b2c/L3_absmax": 0.60847282, "gate_b2c/L3_rms": 0.11515526, "gate_b2c/L5_absmean": 0.12406319, "gate_b2c/L5_absmax": 0.74711788, "gate_b2c/L5_rms": 0.13159709, "gate_b2c/L6_absmean": 0.18165693, "gate_b2c/L6_absmax": 0.51049435, "gate_b2c/L6_rms": 0.18580432}, "streams": {"cos_a2b/round0": 0.386568, "cos_c2b/round0": 0.293481, "cos_b2a/round0": 0.122633, "cos_b2c/round0": 0.146842, "rms_a/round0": 0.836393, "rms_b/round0": 1.593235, "rms_c/round0": 1.256058, "cos_a2b/round1": 0.408986, "cos_c2b/round1": 0.260149, "cos_b2a/round1": 0.223226, "cos_b2c/round1": 0.126477, "rms_a/round1": 1.498942, "rms_b/round1": 3.141448, "rms_c/round1": 2.072827, "cos_a2b/round2": 0.52948, "cos_c2b/round2": 0.484949, "cos_b2a/round2": 0.068762, "cos_b2c/round2": 0.129902, "rms_a/round2": 3.319749, "rms_b/round2": 6.94934, "rms_c/round2": 3.329259}, "alpha_mean": 0.43320712219372837, "alpha_hist": [0.02126, 0.08269, 0.16751, 0.19313, 0.1722, 0.15176, 0.11485, 0.07526, 0.02081, 0.00053], "alpha_by_type": {"special": {"share": 0.00145, "mean": 0.57007, "std": 0.11976}, "whitespace": {"share": 0.02814, "mean": 0.51876, "std": 0.13441}, "digit": {"share": 0.0442, "mean": 0.44936, "std": 0.12821}, "word_start": {"share": 0.55261, "mean": 0.46447, "std": 0.17375}, "word_cont": {"share": 0.26871, "mean": 0.29566, "std": 0.13528}, "punct": {"share": 0.10489, "mean": 0.58921, "std": 0.17907}}, "alpha_p05": 0.15406, "alpha_p50": 0.41751, "alpha_p95": 0.75958}
{"step": 21010, "total_steps": 72479, "tokens": 11015290880, "loss": 4.658294916152954, "l_mix": 2.883201837539673, "l_a": 2.980864644050598, "l_c": 2.9306488633155823, "z_loss": 16.388715267181396, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18302522599697113, "tokens_per_second": 1440247.6489479656, "eta_seconds": 18736.068822407724, "data_wait_seconds": 1.862645149230957e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 21020, "total_steps": 72479, "tokens": 11020533760, "loss": 4.80383563041687, "l_mix": 2.9742500483989716, "l_a": 3.0736688375473022, "l_c": 3.019239842891693, "z_loss": 17.128649950027466, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22279338538646698, "tokens_per_second": 1457761.270650571, "eta_seconds": 18507.37616314888, "data_wait_seconds": 3.400444984436035e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 21030, "total_steps": 72479, "tokens": 11025776640, "loss": 4.643965482711792, "l_mix": 2.8733710944652557, "l_a": 2.9707068502902985, "l_c": 2.92567902803421, "z_loss": 16.785693168640137, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2251436859369278, "tokens_per_second": 1441825.2327972366, "eta_seconds": 18708.29605309963, "data_wait_seconds": 2.5928020477294922e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08236}}
{"step": 21040, "total_steps": 72479, "tokens": 11031019520, "loss": 3.2261780500411987, "l_mix": 1.9949790090322495, "l_a": 2.0677601993083954, "l_c": 2.0301654636859894, "z_loss": 18.212390422821045, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.27240997552871704, "tokens_per_second": 1429786.4312765042, "eta_seconds": 18862.153005552293, "data_wait_seconds": 2.765655517578125e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08237}}
{"step": 21050, "total_steps": 72479, "tokens": 11036262400, "loss": 4.718651652336121, "l_mix": 2.9215677976608276, "l_a": 3.0152288675308228, "l_c": 2.9695971608161926, "z_loss": 16.361165046691895, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1876683235168457, "tokens_per_second": 1326737.0376187814, "eta_seconds": 20323.24928562641, "data_wait_seconds": 4.315376281738281e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 21060, "total_steps": 72479, "tokens": 11041505280, "loss": 4.682923138141632, "l_mix": 2.8986165523529053, "l_a": 2.9991842806339264, "l_c": 2.9429900348186493, "z_loss": 16.54147481918335, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18131916224956512, "tokens_per_second": 1324088.8077576454, "eta_seconds": 20359.936972546577, "data_wait_seconds": 2.345442771911621e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20764, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 21070, "total_steps": 72479, "tokens": 11046748160, "loss": 4.723547995090485, "l_mix": 2.9238798320293427, "l_a": 3.02340367436409, "l_c": 2.969977378845215, "z_loss": 16.5391104221344, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1618828922510147, "tokens_per_second": 1415739.206697412, "eta_seconds": 19038.196911191942, "data_wait_seconds": 1.952052116394043e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20764, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 21080, "total_steps": 72479, "tokens": 11051991040, "loss": 4.6000697016716, "l_mix": 2.847405791282654, "l_a": 2.945465326309204, "l_c": 2.8911391496658325, "z_loss": 16.82518172264099, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18065966665744781, "tokens_per_second": 1421845.953932161, "eta_seconds": 18952.741566324235, "data_wait_seconds": 1.9490718841552734e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 21090, "total_steps": 72479, "tokens": 11057233920, "loss": 4.670780420303345, "l_mix": 2.891153484582901, "l_a": 2.990049183368683, "l_c": 2.9364591240882874, "z_loss": 16.74400782585144, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1634153425693512, "tokens_per_second": 1304276.6747571693, "eta_seconds": 20657.14779191017, "data_wait_seconds": 1.7464160919189453e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20763, "cosmopedia_v2": 0.20043, "finemath": 0.08236}}
{"step": 21100, "total_steps": 72479, "tokens": 11062476800, "loss": 4.651363790035248, "l_mix": 2.879514694213867, "l_a": 2.978366494178772, "l_c": 2.9223171174526215, "z_loss": 16.43968939781189, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1834379881620407, "tokens_per_second": 1477880.9169566056, "eta_seconds": 18227.039027929306, "data_wait_seconds": 1.6182661056518555e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20768, "cosmopedia_v2": 0.20043, "finemath": 0.08232}, "gates": {"gate_a2b/L1_absmean": 0.30322242, "gate_a2b/L1_absmax": 0.6511296, "gate_a2b/L1_rms": 0.32028735, "gate_a2b/L3_absmean": 0.3346152, "gate_a2b/L3_absmax": 0.5058381, "gate_a2b/L3_rms": 0.33915174, "gate_a2b/L5_absmean": 0.51581019, "gate_a2b/L5_absmax": 0.69161761, "gate_a2b/L5_rms": 0.5215891, "gate_c2b/L1_absmean": 0.01409577, "gate_c2b/L1_absmax": 0.43530762, "gate_c2b/L1_rms": 0.05225269, "gate_c2b/L3_absmean": 0.20938797, "gate_c2b/L3_absmax": 0.4324941, "gate_c2b/L3_rms": 0.2135413, "gate_c2b/L5_absmean": 0.52931231, "gate_c2b/L5_absmax": 0.71934861, "gate_c2b/L5_rms": 0.54083788, "gate_b2a/L7_absmean": 0.08600629, "gate_b2a/L7_absmax": 0.27041802, "gate_b2a/L7_rms": 0.08919897, "gate_b2a/L11_absmean": 0.13253814, "gate_b2a/L11_absmax": 0.43627158, "gate_b2a/L11_rms": 0.13525149, "gate_b2a/L14_absmean": 0.3179926, "gate_b2a/L14_absmax": 0.53659689, "gate_b2a/L14_rms": 0.31938207, "gate_b2c/L3_absmean": 0.09859522, "gate_b2c/L3_absmax": 0.60539699, "gate_b2c/L3_rms": 0.11533476, "gate_b2c/L5_absmean": 0.12531558, "gate_b2c/L5_absmax": 0.75247633, "gate_b2c/L5_rms": 0.13279685, "gate_b2c/L6_absmean": 0.18270902, "gate_b2c/L6_absmax": 0.50798458, "gate_b2c/L6_rms": 0.18680187}, "streams": {"cos_a2b/round0": 0.38275, "cos_c2b/round0": 0.292777, "cos_b2a/round0": 0.125888, "cos_b2c/round0": 0.143694, "rms_a/round0": 0.830921, "rms_b/round0": 1.583331, "rms_c/round0": 1.259945, "cos_a2b/round1": 0.404163, "cos_c2b/round1": 0.265501, "cos_b2a/round1": 0.227783, "cos_b2c/round1": 0.122271, "rms_a/round1": 1.475005, "rms_b/round1": 3.101397, "rms_c/round1": 2.069934, "cos_a2b/round2": 0.533149, "cos_c2b/round2": 0.487119, "cos_b2a/round2": 0.073432, "cos_b2c/round2": 0.134946, "rms_a/round2": 3.353251, "rms_b/round2": 6.968554, "rms_c/round2": 3.335365}, "alpha_mean": 0.4573159347727227, "alpha_hist": [0.01416, 0.054, 0.13385, 0.19748, 0.20059, 0.16661, 0.12982, 0.0843, 0.01878, 0.0004], "alpha_by_type": {"special": {"share": 0.00136, "mean": 0.55044, "std": 0.10306}, "whitespace": {"share": 0.02678, "mean": 0.54382, "std": 0.10098}, "digit": {"share": 0.04617, "mean": 0.5037, "std": 0.13802}, "word_start": {"share": 0.53824, "mean": 0.48373, "std": 0.16465}, "word_cont": {"share": 0.27106, "mean": 0.3343, "std": 0.14159}, "punct": {"share": 0.11639, "mean": 0.58227, "std": 0.16664}}, "alpha_p05": 0.18627, "alpha_p50": 0.45145, "alpha_p95": 0.75054}
{"step": 21110, "total_steps": 72479, "tokens": 11067719680, "loss": 4.931183278560638, "l_mix": 3.05585840344429, "l_a": 3.143823057413101, "l_c": 3.1018051505088806, "z_loss": 16.36364448070526, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17938555777072906, "tokens_per_second": 1452500.375044973, "eta_seconds": 18541.923110461237, "data_wait_seconds": 5.8650970458984375e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 21120, "total_steps": 72479, "tokens": 11072962560, "loss": 4.776929438114166, "l_mix": 2.957250267267227, "l_a": 3.0567813217639923, "l_c": 3.003198027610779, "z_loss": 16.852671146392822, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2111537903547287, "tokens_per_second": 1192134.5192490553, "eta_seconds": 22587.13841199875, "data_wait_seconds": 3.1560659408569336e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 21130, "total_steps": 72479, "tokens": 11078205440, "loss": 4.989014148712158, "l_mix": 3.0906293094158173, "l_a": 3.1790832579135895, "l_c": 3.1433306336402893, "z_loss": 16.605820417404175, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21070723235607147, "tokens_per_second": 1426055.526443031, "eta_seconds": 18878.412525177002, "data_wait_seconds": 3.185868263244629e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 21140, "total_steps": 72479, "tokens": 11083448320, "loss": 4.925685167312622, "l_mix": 3.0527279675006866, "l_a": 3.140944391489029, "l_c": 3.096668154001236, "z_loss": 16.733165502548218, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1885838359594345, "tokens_per_second": 1117079.149606456, "eta_seconds": 24095.357648992536, "data_wait_seconds": 2.0235776901245117e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08236}}
{"step": 21150, "total_steps": 72479, "tokens": 11088691200, "loss": 3.252982348203659, "l_mix": 2.011174440383911, "l_a": 2.089291200041771, "l_c": 2.0437694787979126, "z_loss": 18.896835565567017, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.29441025853157043, "tokens_per_second": 1380093.3743804786, "eta_seconds": 19499.53477900028, "data_wait_seconds": 2.321600914001465e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08236}}
{"step": 21160, "total_steps": 72479, "tokens": 11093934080, "loss": 4.708333075046539, "l_mix": 2.9143719375133514, "l_a": 3.0123836994171143, "l_c": 2.9619002044200897, "z_loss": 16.760109424591064, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16092678904533386, "tokens_per_second": 1472026.4747433988, "eta_seconds": 18278.160300540923, "data_wait_seconds": 1.704692840576172e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 21170, "total_steps": 72479, "tokens": 11099176960, "loss": 4.714843034744263, "l_mix": 2.918547123670578, "l_a": 3.0156460404396057, "l_c": 2.966483920812607, "z_loss": 16.56791090965271, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18845267593860626, "tokens_per_second": 1399524.4693681765, "eta_seconds": 19221.30950961113, "data_wait_seconds": 4.044175148010254e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 21180, "total_steps": 72479, "tokens": 11104419840, "loss": 4.737904131412506, "l_mix": 2.9335880279541016, "l_a": 3.030460476875305, "l_c": 2.9784851372241974, "z_loss": 16.324958086013794, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1849360466003418, "tokens_per_second": 1353590.0167248773, "eta_seconds": 19869.716664338113, "data_wait_seconds": 2.1517276763916016e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 21190, "total_steps": 72479, "tokens": 11109662720, "loss": 4.758658468723297, "l_mix": 2.9459360539913177, "l_a": 3.044066309928894, "l_c": 2.9928522408008575, "z_loss": 16.468069314956665, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2334440052509308, "tokens_per_second": 1444208.4376949521, "eta_seconds": 18619.33951508999, "data_wait_seconds": 4.8160552978515625e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 21200, "total_steps": 72479, "tokens": 11114905600, "loss": 4.753288686275482, "l_mix": 2.9433341324329376, "l_a": 3.0389592051506042, "l_c": 2.9886141419410706, "z_loss": 16.82553720474243, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1539379060268402, "tokens_per_second": 1345926.800076654, "eta_seconds": 19975.0568533659, "data_wait_seconds": 1.519918441772461e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20042, "finemath": 0.08236}, "gates": {"gate_a2b/L1_absmean": 0.30306551, "gate_a2b/L1_absmax": 0.6565448, "gate_a2b/L1_rms": 0.32034063, "gate_a2b/L3_absmean": 0.33478001, "gate_a2b/L3_absmax": 0.50868315, "gate_a2b/L3_rms": 0.33928329, "gate_a2b/L5_absmean": 0.51563305, "gate_a2b/L5_absmax": 0.68764895, "gate_a2b/L5_rms": 0.52145326, "gate_c2b/L1_absmean": 0.01416982, "gate_c2b/L1_absmax": 0.4346129, "gate_c2b/L1_rms": 0.052558, "gate_c2b/L3_absmean": 0.20935698, "gate_c2b/L3_absmax": 0.43618307, "gate_c2b/L3_rms": 0.21340491, "gate_c2b/L5_absmean": 0.52924854, "gate_c2b/L5_absmax": 0.72544295, "gate_c2b/L5_rms": 0.54082334, "gate_b2a/L7_absmean": 0.08593577, "gate_b2a/L7_absmax": 0.27100769, "gate_b2a/L7_rms": 0.08906785, "gate_b2a/L11_absmean": 0.13250743, "gate_b2a/L11_absmax": 0.4399474, "gate_b2a/L11_rms": 0.13528126, "gate_b2a/L14_absmean": 0.3178798, "gate_b2a/L14_absmax": 0.53847831, "gate_b2a/L14_rms": 0.31929302, "gate_b2c/L3_absmean": 0.09920906, "gate_b2c/L3_absmax": 0.60401076, "gate_b2c/L3_rms": 0.1158439, "gate_b2c/L5_absmean": 0.12469895, "gate_b2c/L5_absmax": 0.75799179, "gate_b2c/L5_rms": 0.13231222, "gate_b2c/L6_absmean": 0.1823768, "gate_b2c/L6_absmax": 0.51074529, "gate_b2c/L6_rms": 0.18653423}, "streams": {"cos_a2b/round0": 0.38828, "cos_c2b/round0": 0.298252, "cos_b2a/round0": 0.127193, "cos_b2c/round0": 0.152066, "rms_a/round0": 0.825657, "rms_b/round0": 1.579909, "rms_c/round0": 1.245461, "cos_a2b/round1": 0.410721, "cos_c2b/round1": 0.274073, "cos_b2a/round1": 0.228662, "cos_b2c/round1": 0.130222, "rms_a/round1": 1.468452, "rms_b/round1": 3.068541, "rms_c/round1": 2.046214, "cos_a2b/round2": 0.546321, "cos_c2b/round2": 0.486787, "cos_b2a/round2": 0.074083, "cos_b2c/round2": 0.135804, "rms_a/round2": 3.309101, "rms_b/round2": 6.83708, "rms_c/round2": 3.300456}, "alpha_mean": 0.44416609706016175, "alpha_hist": [0.01393, 0.06494, 0.15198, 0.19403, 0.19312, 0.17174, 0.12553, 0.07219, 0.01237, 0.00017], "alpha_by_type": {"special": {"share": 0.00121, "mean": 0.54796, "std": 0.10643}, "whitespace": {"share": 0.02615, "mean": 0.52883, "std": 0.11933}, "digit": {"share": 0.03615, "mean": 0.41452, "std": 0.14559}, "word_start": {"share": 0.58255, "mean": 0.4637, "std": 0.1639}, "word_cont": {"share": 0.25131, "mean": 0.32929, "std": 0.13712}, "punct": {"share": 0.10263, "mean": 0.60223, "std": 0.1561}}, "alpha_p05": 0.16345, "alpha_p50": 0.44794, "alpha_p95": 0.72037}
{"step": 21210, "total_steps": 72479, "tokens": 11120148480, "loss": 4.988853633403778, "l_mix": 3.091563642024994, "l_a": 3.1746989488601685, "l_c": 3.1440407633781433, "z_loss": 16.680633306503296, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18198049068450928, "tokens_per_second": 1263205.6474472256, "eta_seconds": 21278.975063419344, "data_wait_seconds": 3.1441450119018555e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20769, "cosmopedia_v2": 0.20042, "finemath": 0.08232}}
{"step": 21220, "total_steps": 72479, "tokens": 11125391360, "loss": 4.9886046051979065, "l_mix": 3.0921507477760315, "l_a": 3.18303245306015, "l_c": 3.1328722536563873, "z_loss": 16.824706554412842, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20495326817035675, "tokens_per_second": 1313014.4783255786, "eta_seconds": 20467.770185041427, "data_wait_seconds": 4.303455352783203e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20768, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 21230, "total_steps": 72479, "tokens": 11130634240, "loss": 4.974561393260956, "l_mix": 3.0828205049037933, "l_a": 3.165759801864624, "l_c": 3.1344184279441833, "z_loss": 16.873818397521973, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1947852224111557, "tokens_per_second": 1448830.8247008352, "eta_seconds": 18545.46110830307, "data_wait_seconds": 2.6702880859375e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 21240, "total_steps": 72479, "tokens": 11135877120, "loss": 5.000201225280762, "l_mix": 3.099814713001251, "l_a": 3.1874857544898987, "l_c": 3.1417932212352753, "z_loss": 16.027469754219055, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2003016471862793, "tokens_per_second": 1477047.472805889, "eta_seconds": 18187.629935121535, "data_wait_seconds": 3.585219383239746e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 21250, "total_steps": 72479, "tokens": 11141120000, "loss": 4.940575480461121, "l_mix": 3.0613581836223602, "l_a": 3.145271748304367, "l_c": 3.1132456958293915, "z_loss": 16.619964122772217, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18735574185848236, "tokens_per_second": 1351884.211297944, "eta_seconds": 19867.64082865715, "data_wait_seconds": 2.238154411315918e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 21260, "total_steps": 72479, "tokens": 11146362880, "loss": 3.106790989637375, "l_mix": 1.9199469238519669, "l_a": 1.9956972301006317, "l_c": 1.9540977776050568, "z_loss": 19.054822206497192, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2522014379501343, "tokens_per_second": 1389497.1902957885, "eta_seconds": 19326.06072149277, "data_wait_seconds": 1.8537044525146484e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08236}}
{"step": 21270, "total_steps": 72479, "tokens": 11151605760, "loss": 4.714604556560516, "l_mix": 2.9189851880073547, "l_a": 3.01367723941803, "l_c": 2.966152608394623, "z_loss": 16.703357219696045, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17082221806049347, "tokens_per_second": 1458604.5285722616, "eta_seconds": 18406.815326619148, "data_wait_seconds": 1.5348196029663086e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08232}}
{"step": 21280, "total_steps": 72479, "tokens": 11156848640, "loss": 4.756342589855194, "l_mix": 2.9455040097236633, "l_a": 3.0383912920951843, "l_c": 2.9921603202819824, "z_loss": 16.729414463043213, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17921467125415802, "tokens_per_second": 1395394.0525444383, "eta_seconds": 19236.875249004363, "data_wait_seconds": 2.4557113647460938e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 21290, "total_steps": 72479, "tokens": 11162091520, "loss": 4.8344244956970215, "l_mix": 2.9938787817955017, "l_a": 3.088479518890381, "l_c": 3.0411029160022736, "z_loss": 16.710197687149048, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17149217426776886, "tokens_per_second": 1419942.3502735377, "eta_seconds": 18900.61130075455, "data_wait_seconds": 2.244114875793457e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 21300, "total_steps": 72479, "tokens": 11167334400, "loss": 4.80870795249939, "l_mix": 2.977980822324753, "l_a": 3.0723289251327515, "l_c": 3.024520069360733, "z_loss": 16.724225997924805, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17387425899505615, "tokens_per_second": 1384698.4415298847, "eta_seconds": 19377.89106078148, "data_wait_seconds": 2.256035804748535e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.30455521, "gate_a2b/L1_absmax": 0.65955257, "gate_a2b/L1_rms": 0.32163846, "gate_a2b/L3_absmean": 0.33505461, "gate_a2b/L3_absmax": 0.51495767, "gate_a2b/L3_rms": 0.33967084, "gate_a2b/L5_absmean": 0.51583523, "gate_a2b/L5_absmax": 0.68864971, "gate_a2b/L5_rms": 0.52164102, "gate_c2b/L1_absmean": 0.01462064, "gate_c2b/L1_absmax": 0.43902618, "gate_c2b/L1_rms": 0.0526048, "gate_c2b/L3_absmean": 0.20892341, "gate_c2b/L3_absmax": 0.42980066, "gate_c2b/L3_rms": 0.21300635, "gate_c2b/L5_absmean": 0.52944803, "gate_c2b/L5_absmax": 0.71920973, "gate_c2b/L5_rms": 0.54093146, "gate_b2a/L7_absmean": 0.08560178, "gate_b2a/L7_absmax": 0.27187112, "gate_b2a/L7_rms": 0.08884416, "gate_b2a/L11_absmean": 0.13249667, "gate_b2a/L11_absmax": 0.43743554, "gate_b2a/L11_rms": 0.13519849, "gate_b2a/L14_absmean": 0.3185899, "gate_b2a/L14_absmax": 0.53848428, "gate_b2a/L14_rms": 0.31995824, "gate_b2c/L3_absmean": 0.10022404, "gate_b2c/L3_absmax": 0.60313123, "gate_b2c/L3_rms": 0.11692687, "gate_b2c/L5_absmean": 0.12487642, "gate_b2c/L5_absmax": 0.75940174, "gate_b2c/L5_rms": 0.1325545, "gate_b2c/L6_absmean": 0.18301146, "gate_b2c/L6_absmax": 0.51441115, "gate_b2c/L6_rms": 0.1872323}, "streams": {"cos_a2b/round0": 0.385298, "cos_c2b/round0": 0.293454, "cos_b2a/round0": 0.128827, "cos_b2c/round0": 0.151559, "rms_a/round0": 0.829871, "rms_b/round0": 1.586635, "rms_c/round0": 1.251539, "cos_a2b/round1": 0.417826, "cos_c2b/round1": 0.273369, "cos_b2a/round1": 0.225464, "cos_b2c/round1": 0.130739, "rms_a/round1": 1.483696, "rms_b/round1": 3.096369, "rms_c/round1": 2.055217, "cos_a2b/round2": 0.547273, "cos_c2b/round2": 0.486738, "cos_b2a/round2": 0.077581, "cos_b2c/round2": 0.139623, "rms_a/round2": 3.367822, "rms_b/round2": 6.974264, "rms_c/round2": 3.338761}, "alpha_mean": 0.4603161581156172, "alpha_hist": [0.00964, 0.0594, 0.14668, 0.18484, 0.17747, 0.17522, 0.13683, 0.08974, 0.01999, 0.00018], "alpha_by_type": {"special": {"share": 0.00099, "mean": 0.56236, "std": 0.10601}, "whitespace": {"share": 0.02567, "mean": 0.52857, "std": 0.11393}, "digit": {"share": 0.03712, "mean": 0.43875, "std": 0.1307}, "word_start": {"share": 0.56735, "mean": 0.48976, "std": 0.16878}, "word_cont": {"share": 0.26361, "mean": 0.33435, "std": 0.1399}, "punct": {"share": 0.10526, "mean": 0.60707, "std": 0.16161}}, "alpha_p05": 0.18213, "alpha_p50": 0.45871, "alpha_p95": 0.75672}
{"step": 21310, "total_steps": 72479, "tokens": 11172577280, "loss": 4.7140613198280334, "l_mix": 2.9186622202396393, "l_a": 3.0131374299526215, "l_c": 2.9659258127212524, "z_loss": 16.801024436950684, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18272408843040466, "tokens_per_second": 1467082.365997887, "eta_seconds": 18286.153043460847, "data_wait_seconds": 1.665949821472168e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08236}}
{"step": 21320, "total_steps": 72479, "tokens": 11177820160, "loss": 3.1662658154964447, "l_mix": 1.9566715955734253, "l_a": 2.0294378101825714, "l_c": 1.9963514655828476, "z_loss": 18.57419228553772, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2551957070827484, "tokens_per_second": 1476096.3561560656, "eta_seconds": 18170.934221291544, "data_wait_seconds": 2.86102294921875e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08236}}
{"step": 21330, "total_steps": 72479, "tokens": 11183063040, "loss": 4.91542261838913, "l_mix": 3.0452674329280853, "l_a": 3.1337923407554626, "l_c": 3.0946764051914215, "z_loss": 16.14453363418579, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22903379797935486, "tokens_per_second": 1442252.0928679847, "eta_seconds": 18593.702893280984, "data_wait_seconds": 2.893805503845215e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20769, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 21340, "total_steps": 72479, "tokens": 11188305920, "loss": 4.9432631731033325, "l_mix": 3.06317076086998, "l_a": 3.1515199840068817, "l_c": 3.109874039888382, "z_loss": 16.740904331207275, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1960655003786087, "tokens_per_second": 1390519.2978527232, "eta_seconds": 19281.69143240452, "data_wait_seconds": 2.3692846298217773e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20769, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 21350, "total_steps": 72479, "tokens": 11193548800, "loss": 4.861980140209198, "l_mix": 3.0119424760341644, "l_a": 3.1039856672286987, "l_c": 3.0573328137397766, "z_loss": 16.42114543914795, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18417298793792725, "tokens_per_second": 1446787.314481918, "eta_seconds": 18528.169886255266, "data_wait_seconds": 2.3245811462402344e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 21360, "total_steps": 72479, "tokens": 11198791680, "loss": 4.851995289325714, "l_mix": 3.007436126470566, "l_a": 3.0903043150901794, "l_c": 3.0526740849018097, "z_loss": 16.655099868774414, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1921197474002838, "tokens_per_second": 1431764.0797773893, "eta_seconds": 18718.920701074603, "data_wait_seconds": 2.3603439331054688e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20768, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 21370, "total_steps": 72479, "tokens": 11204034560, "loss": 4.974123418331146, "l_mix": 3.0833104848861694, "l_a": 3.168186843395233, "l_c": 3.128965437412262, "z_loss": 16.671303272247314, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19776351749897003, "tokens_per_second": 1425552.8954280817, "eta_seconds": 18796.80191309452, "data_wait_seconds": 1.7821788787841797e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08236}}
{"step": 21380, "total_steps": 72479, "tokens": 11209277440, "loss": 4.678761065006256, "l_mix": 2.895626723766327, "l_a": 2.993077129125595, "l_c": 2.945046752691269, "z_loss": 16.969640970230103, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15591293573379517, "tokens_per_second": 1367087.7481710499, "eta_seconds": 19596.834620046615, "data_wait_seconds": 1.71661376953125e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08232}}
{"step": 21390, "total_steps": 72479, "tokens": 11214520320, "loss": 4.754953563213348, "l_mix": 2.9445262253284454, "l_a": 3.0394095480442047, "l_c": 2.989687144756317, "z_loss": 16.98298978805542, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1518370509147644, "tokens_per_second": 1470551.9604256516, "eta_seconds": 18214.487044882775, "data_wait_seconds": 1.615285873413086e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 21400, "total_steps": 72479, "tokens": 11219763200, "loss": 4.7285409569740295, "l_mix": 2.926697164773941, "l_a": 3.0246108174324036, "l_c": 2.9759690165519714, "z_loss": 16.697350025177002, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16555851697921753, "tokens_per_second": 1305173.0277017227, "eta_seconds": 20518.43409540653, "data_wait_seconds": 1.6003847122192383e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.30521959, "gate_a2b/L1_absmax": 0.67040479, "gate_a2b/L1_rms": 0.32227546, "gate_a2b/L3_absmean": 0.33603942, "gate_a2b/L3_absmax": 0.51403278, "gate_a2b/L3_rms": 0.34060019, "gate_a2b/L5_absmean": 0.51646227, "gate_a2b/L5_absmax": 0.69134569, "gate_a2b/L5_rms": 0.52218819, "gate_c2b/L1_absmean": 0.01427399, "gate_c2b/L1_absmax": 0.43684474, "gate_c2b/L1_rms": 0.05221622, "gate_c2b/L3_absmean": 0.20948021, "gate_c2b/L3_absmax": 0.43434677, "gate_c2b/L3_rms": 0.21356709, "gate_c2b/L5_absmean": 0.53020024, "gate_c2b/L5_absmax": 0.71644282, "gate_c2b/L5_rms": 0.54186195, "gate_b2a/L7_absmean": 0.08638856, "gate_b2a/L7_absmax": 0.27188191, "gate_b2a/L7_rms": 0.08956373, "gate_b2a/L11_absmean": 0.13256423, "gate_b2a/L11_absmax": 0.43848595, "gate_b2a/L11_rms": 0.1352386, "gate_b2a/L14_absmean": 0.31921139, "gate_b2a/L14_absmax": 0.53477287, "gate_b2a/L14_rms": 0.32052895, "gate_b2c/L3_absmean": 0.099914, "gate_b2c/L3_absmax": 0.60214275, "gate_b2c/L3_rms": 0.11646193, "gate_b2c/L5_absmean": 0.12537906, "gate_b2c/L5_absmax": 0.76543248, "gate_b2c/L5_rms": 0.13303337, "gate_b2c/L6_absmean": 0.1835137, "gate_b2c/L6_absmax": 0.51493341, "gate_b2c/L6_rms": 0.1876248}, "streams": {"cos_a2b/round0": 0.385151, "cos_c2b/round0": 0.297651, "cos_b2a/round0": 0.128748, "cos_b2c/round0": 0.154981, "rms_a/round0": 0.830013, "rms_b/round0": 1.572156, "rms_c/round0": 1.242642, "cos_a2b/round1": 0.41609, "cos_c2b/round1": 0.273453, "cos_b2a/round1": 0.228291, "cos_b2c/round1": 0.129749, "rms_a/round1": 1.464247, "rms_b/round1": 3.044227, "rms_c/round1": 2.037885, "cos_a2b/round2": 0.551997, "cos_c2b/round2": 0.482481, "cos_b2a/round2": 0.077698, "cos_b2c/round2": 0.140411, "rms_a/round2": 3.304275, "rms_b/round2": 6.761096, "rms_c/round2": 3.271291}, "alpha_mean": 0.46827763555847923, "alpha_hist": [0.01085, 0.0551, 0.13031, 0.17545, 0.18454, 0.18503, 0.15163, 0.08882, 0.01807, 0.00021], "alpha_by_type": {"special": {"share": 0.00119, "mean": 0.5685, "std": 0.10082}, "whitespace": {"share": 0.02551, "mean": 0.5323, "std": 0.11411}, "digit": {"share": 0.0274, "mean": 0.48781, "std": 0.11617}, "word_start": {"share": 0.58409, "mean": 0.49153, "std": 0.16315}, "word_cont": {"share": 0.25363, "mean": 0.34221, "std": 0.14889}, "punct": {"share": 0.10817, "mean": 0.61714, "std": 0.15329}}, "alpha_p05": 0.18922, "alpha_p50": 0.48036, "alpha_p95": 0.75815}
{"step": 21410, "total_steps": 72479, "tokens": 11225006080, "loss": 4.706778347492218, "l_mix": 2.914288103580475, "l_a": 3.0102740824222565, "l_c": 2.959095597267151, "z_loss": 16.792523622512817, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1791011393070221, "tokens_per_second": 1332774.4036515814, "eta_seconds": 20089.56939647198, "data_wait_seconds": 2.3096799850463867e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 21420, "total_steps": 72479, "tokens": 11230248960, "loss": 4.781394302845001, "l_mix": 2.961169093847275, "l_a": 3.0530137717723846, "l_c": 3.0088115632534027, "z_loss": 16.775124311447144, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17920391261577606, "tokens_per_second": 1406442.0861555948, "eta_seconds": 19033.575044083595, "data_wait_seconds": 2.384185791015625e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 21430, "total_steps": 72479, "tokens": 11235491840, "loss": 4.075247257947922, "l_mix": 2.513102874159813, "l_a": 2.673790827393532, "l_c": 2.527128279209137, "z_loss": 18.685316681861877, "lr": 0.0023, "wd": 0.1, "grad_norm": 2.329261302947998, "tokens_per_second": 1456424.3666770186, "eta_seconds": 18376.77171871662, "data_wait_seconds": 2.0116567611694336e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08236}}
{"step": 21440, "total_steps": 72479, "tokens": 11240734720, "loss": 3.4447342455387115, "l_mix": 2.115467131137848, "l_a": 2.2697382420301437, "l_c": 2.1553199887275696, "z_loss": 17.49581527709961, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.6369282007217407, "tokens_per_second": 1329681.4074423036, "eta_seconds": 20124.471232151984, "data_wait_seconds": 2.16066837310791e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08237}}
{"step": 21450, "total_steps": 72479, "tokens": 11245977600, "loss": 4.858233332633972, "l_mix": 3.009232312440872, "l_a": 3.0999327898025513, "l_c": 3.0578120350837708, "z_loss": 16.775267124176025, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17438597977161407, "tokens_per_second": 1403894.483124588, "eta_seconds": 19056.91109523773, "data_wait_seconds": 1.633167266845703e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20769, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 21460, "total_steps": 72479, "tokens": 11251220480, "loss": 4.840499639511108, "l_mix": 2.999500423669815, "l_a": 3.0848843157291412, "l_c": 3.0462599992752075, "z_loss": 16.55776286125183, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.23034510016441345, "tokens_per_second": 1468079.6236521883, "eta_seconds": 18220.162613153458, "data_wait_seconds": 2.053380012512207e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20769, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 21470, "total_steps": 72479, "tokens": 11256463360, "loss": 4.788694679737091, "l_mix": 2.9644819796085358, "l_a": 3.0603961646556854, "l_c": 3.014754205942154, "z_loss": 16.67497682571411, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18205861747264862, "tokens_per_second": 1412997.7555176048, "eta_seconds": 18926.715550374985, "data_wait_seconds": 1.862645149230957e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20764, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 21480, "total_steps": 72479, "tokens": 11261706240, "loss": 4.82470703125, "l_mix": 2.9875660836696625, "l_a": 3.0837980210781097, "l_c": 3.0343950986862183, "z_loss": 16.829173803329468, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.24238204956054688, "tokens_per_second": 1466089.1013742224, "eta_seconds": 18237.74809248447, "data_wait_seconds": 2.0205974578857422e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20764, "cosmopedia_v2": 0.20042, "finemath": 0.08236}}
{"step": 21490, "total_steps": 72479, "tokens": 11266949120, "loss": 4.818813920021057, "l_mix": 2.9840305745601654, "l_a": 3.0799346268177032, "l_c": 3.0304075479507446, "z_loss": 16.806172609329224, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17473767697811127, "tokens_per_second": 1378211.401073389, "eta_seconds": 19396.821714854243, "data_wait_seconds": 2.7626752853393555e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20768, "cosmopedia_v2": 0.20042, "finemath": 0.08232}}
{"step": 21500, "total_steps": 72479, "tokens": 11272192000, "loss": 4.7521087527275085, "l_mix": 2.942109853029251, "l_a": 3.0389585494995117, "l_c": 2.988812416791916, "z_loss": 16.675974369049072, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1690068244934082, "tokens_per_second": 1426966.471088888, "eta_seconds": 18730.417633152007, "data_wait_seconds": 1.6897916793823242e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.30319044, "gate_a2b/L1_absmax": 0.65000755, "gate_a2b/L1_rms": 0.3201181, "gate_a2b/L3_absmean": 0.3345249, "gate_a2b/L3_absmax": 0.51251364, "gate_a2b/L3_rms": 0.33915707, "gate_a2b/L5_absmean": 0.51497507, "gate_a2b/L5_absmax": 0.68363482, "gate_a2b/L5_rms": 0.52072883, "gate_c2b/L1_absmean": 0.01351072, "gate_c2b/L1_absmax": 0.43505973, "gate_c2b/L1_rms": 0.05162637, "gate_c2b/L3_absmean": 0.20955014, "gate_c2b/L3_absmax": 0.43780714, "gate_c2b/L3_rms": 0.21362612, "gate_c2b/L5_absmean": 0.52962977, "gate_c2b/L5_absmax": 0.72934765, "gate_c2b/L5_rms": 0.54125887, "gate_b2a/L7_absmean": 0.08590219, "gate_b2a/L7_absmax": 0.28893983, "gate_b2a/L7_rms": 0.08915475, "gate_b2a/L11_absmean": 0.1330089, "gate_b2a/L11_absmax": 0.45671445, "gate_b2a/L11_rms": 0.13584585, "gate_b2a/L14_absmean": 0.31923497, "gate_b2a/L14_absmax": 0.52005011, "gate_b2a/L14_rms": 0.32054666, "gate_b2c/L3_absmean": 0.0993927, "gate_b2c/L3_absmax": 0.61829442, "gate_b2c/L3_rms": 0.11599898, "gate_b2c/L5_absmean": 0.12503825, "gate_b2c/L5_absmax": 0.77567285, "gate_b2c/L5_rms": 0.13293321, "gate_b2c/L6_absmean": 0.18232743, "gate_b2c/L6_absmax": 0.49521407, "gate_b2c/L6_rms": 0.1864368}, "streams": {"cos_a2b/round0": 0.388391, "cos_c2b/round0": 0.296952, "cos_b2a/round0": 0.126536, "cos_b2c/round0": 0.149051, "rms_a/round0": 0.834162, "rms_b/round0": 1.558743, "rms_c/round0": 1.247668, "cos_a2b/round1": 0.420564, "cos_c2b/round1": 0.277859, "cos_b2a/round1": 0.225596, "cos_b2c/round1": 0.124967, "rms_a/round1": 1.478749, "rms_b/round1": 3.059268, "rms_c/round1": 2.019925, "cos_a2b/round2": 0.548065, "cos_c2b/round2": 0.482701, "cos_b2a/round2": 0.074222, "cos_b2c/round2": 0.13807, "rms_a/round2": 3.336616, "rms_b/round2": 6.863041, "rms_c/round2": 3.227845}, "alpha_mean": 0.46350403043882693, "alpha_hist": [0.01085, 0.05876, 0.13657, 0.17802, 0.17899, 0.18501, 0.14926, 0.08774, 0.01465, 0.00015], "alpha_by_type": {"special": {"share": 0.00085, "mean": 0.62438, "std": 0.10676}, "whitespace": {"share": 0.02531, "mean": 0.58963, "std": 0.12137}, "digit": {"share": 0.02835, "mean": 0.5144, "std": 0.15076}, "word_start": {"share": 0.60252, "mean": 0.47952, "std": 0.16078}, "word_cont": {"share": 0.22871, "mean": 0.32469, "std": 0.14146}, "punct": {"share": 0.11424, "mean": 0.61514, "std": 0.14789}}, "alpha_p05": 0.20915, "alpha_p50": 0.47013, "alpha_p95": 0.74503}
{"step": 21510, "total_steps": 72479, "tokens": 11277434880, "loss": 4.676509380340576, "l_mix": 2.8941804468631744, "l_a": 2.9942470490932465, "l_c": 2.941200405359268, "z_loss": 16.944158792495728, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15937773883342743, "tokens_per_second": 1428835.6753376604, "eta_seconds": 18702.24514493942, "data_wait_seconds": 2.2798776626586914e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 21520, "total_steps": 72479, "tokens": 11282677760, "loss": 4.720978379249573, "l_mix": 2.921837419271469, "l_a": 3.022184044122696, "l_c": 2.969362884759903, "z_loss": 16.7680242061615, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15309292078018188, "tokens_per_second": 1447560.2683283614, "eta_seconds": 18456.704550790786, "data_wait_seconds": 2.5898218154907227e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 21530, "total_steps": 72479, "tokens": 11287920640, "loss": 4.772269546985626, "l_mix": 2.954273283481598, "l_a": 3.049424856901169, "l_c": 3.00489941239357, "z_loss": 16.989879369735718, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1838235706090927, "tokens_per_second": 1283438.4297509906, "eta_seconds": 20812.801528143882, "data_wait_seconds": 2.467632293701172e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 21540, "total_steps": 72479, "tokens": 11293163520, "loss": 3.144465297460556, "l_mix": 1.9423873275518417, "l_a": 2.0188503861427307, "l_c": 1.9817412048578262, "z_loss": 19.004430770874023, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22028960287570953, "tokens_per_second": 1395446.296034804, "eta_seconds": 19138.46954045296, "data_wait_seconds": 1.9282102584838867e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08236}}
{"step": 21550, "total_steps": 72479, "tokens": 11298406400, "loss": 3.221352696418762, "l_mix": 1.989553913474083, "l_a": 2.0718432664871216, "l_c": 2.027814283967018, "z_loss": 19.014885663986206, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.29043763875961304, "tokens_per_second": 1349351.1435688296, "eta_seconds": 19788.37286295891, "data_wait_seconds": 1.817941665649414e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08237}}
{"step": 21560, "total_steps": 72479, "tokens": 11303649280, "loss": 4.725863993167877, "l_mix": 2.9259139001369476, "l_a": 3.023197203874588, "l_c": 2.9710660576820374, "z_loss": 16.710792303085327, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16119474172592163, "tokens_per_second": 1347719.8436637, "eta_seconds": 19808.434814929962, "data_wait_seconds": 2.1457672119140625e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 21570, "total_steps": 72479, "tokens": 11308892160, "loss": 4.738091468811035, "l_mix": 2.933059483766556, "l_a": 3.0312460362911224, "l_c": 2.9799452126026154, "z_loss": 16.74537706375122, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.14831651747226715, "tokens_per_second": 1410377.8827071863, "eta_seconds": 18924.69962785244, "data_wait_seconds": 2.0235776901245117e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 21580, "total_steps": 72479, "tokens": 11314135040, "loss": 4.724182903766632, "l_mix": 2.9238228499889374, "l_a": 3.0243337154388428, "l_c": 2.9711652398109436, "z_loss": 17.10279417037964, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1616828441619873, "tokens_per_second": 1480670.9877149374, "eta_seconds": 18022.731000614167, "data_wait_seconds": 1.5527009963989258e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 21590, "total_steps": 72479, "tokens": 11319377920, "loss": 4.744436502456665, "l_mix": 2.9376251697540283, "l_a": 3.0323130190372467, "l_c": 2.9847834706306458, "z_loss": 16.82357692718506, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17379088699817657, "tokens_per_second": 1313973.5247902176, "eta_seconds": 20305.197577142713, "data_wait_seconds": 1.8209218978881836e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20764, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 21600, "total_steps": 72479, "tokens": 11324620800, "loss": 4.691781222820282, "l_mix": 2.9043959975242615, "l_a": 2.9991644620895386, "l_c": 2.9532392024993896, "z_loss": 16.64054250717163, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1778872311115265, "tokens_per_second": 1457615.5567707748, "eta_seconds": 18300.606787633897, "data_wait_seconds": 1.9371509552001953e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20042, "finemath": 0.08236}, "gates": {"gate_a2b/L1_absmean": 0.30488434, "gate_a2b/L1_absmax": 0.65143979, "gate_a2b/L1_rms": 0.32170996, "gate_a2b/L3_absmean": 0.33572373, "gate_a2b/L3_absmax": 0.51705825, "gate_a2b/L3_rms": 0.34028274, "gate_a2b/L5_absmean": 0.51576012, "gate_a2b/L5_absmax": 0.68967068, "gate_a2b/L5_rms": 0.52155364, "gate_c2b/L1_absmean": 0.01398911, "gate_c2b/L1_absmax": 0.43632448, "gate_c2b/L1_rms": 0.05195127, "gate_c2b/L3_absmean": 0.20941873, "gate_c2b/L3_absmax": 0.43297264, "gate_c2b/L3_rms": 0.21351315, "gate_c2b/L5_absmean": 0.53006452, "gate_c2b/L5_absmax": 0.72122639, "gate_c2b/L5_rms": 0.54170918, "gate_b2a/L7_absmean": 0.08621293, "gate_b2a/L7_absmax": 0.28064382, "gate_b2a/L7_rms": 0.08942061, "gate_b2a/L11_absmean": 0.13270117, "gate_b2a/L11_absmax": 0.4579435, "gate_b2a/L11_rms": 0.13563256, "gate_b2a/L14_absmean": 0.31980073, "gate_b2a/L14_absmax": 0.53217489, "gate_b2a/L14_rms": 0.32112548, "gate_b2c/L3_absmean": 0.09980832, "gate_b2c/L3_absmax": 0.61613071, "gate_b2c/L3_rms": 0.11657031, "gate_b2c/L5_absmean": 0.1250575, "gate_b2c/L5_absmax": 0.77130908, "gate_b2c/L5_rms": 0.13288055, "gate_b2c/L6_absmean": 0.18307728, "gate_b2c/L6_absmax": 0.50873977, "gate_b2c/L6_rms": 0.18724027}, "streams": {"cos_a2b/round0": 0.388054, "cos_c2b/round0": 0.299104, "cos_b2a/round0": 0.124302, "cos_b2c/round0": 0.15483, "rms_a/round0": 0.837857, "rms_b/round0": 1.577809, "rms_c/round0": 1.246389, "cos_a2b/round1": 0.416976, "cos_c2b/round1": 0.270605, "cos_b2a/round1": 0.228235, "cos_b2c/round1": 0.124248, "rms_a/round1": 1.482118, "rms_b/round1": 3.079022, "rms_c/round1": 2.045941, "cos_a2b/round2": 0.545379, "cos_c2b/round2": 0.485618, "cos_b2a/round2": 0.071456, "cos_b2c/round2": 0.140193, "rms_a/round2": 3.341088, "rms_b/round2": 6.919624, "rms_c/round2": 3.267713}, "alpha_mean": 0.4604764310894893, "alpha_hist": [0.01526, 0.06345, 0.13741, 0.17953, 0.18134, 0.17363, 0.1368, 0.08783, 0.02414, 0.00063], "alpha_by_type": {"special": {"share": 0.00102, "mean": 0.58984, "std": 0.10844}, "whitespace": {"share": 0.02493, "mean": 0.56732, "std": 0.1232}, "digit": {"share": 0.04099, "mean": 0.47174, "std": 0.12875}, "word_start": {"share": 0.56223, "mean": 0.48399, "std": 0.16741}, "word_cont": {"share": 0.25314, "mean": 0.32366, "std": 0.14315}, "punct": {"share": 0.11769, "mean": 0.61477, "std": 0.16856}}, "alpha_p05": 0.17024, "alpha_p50": 0.48231, "alpha_p95": 0.77287}
{"step": 21610, "total_steps": 72479, "tokens": 11329863680, "loss": 4.6963977217674255, "l_mix": 2.906841456890106, "l_a": 3.0060278177261353, "l_c": 2.953574687242508, "z_loss": 16.754589080810547, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17501550912857056, "tokens_per_second": 1396928.8951349463, "eta_seconds": 19091.88532421589, "data_wait_seconds": 1.621246337890625e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20768, "cosmopedia_v2": 0.20042, "finemath": 0.08232}}
{"step": 21620, "total_steps": 72479, "tokens": 11335106560, "loss": 4.652064800262451, "l_mix": 2.879282146692276, "l_a": 2.978616416454315, "l_c": 2.924979478120804, "z_loss": 17.038124799728394, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1662101000547409, "tokens_per_second": 1457096.7118677716, "eta_seconds": 18299.92695393562, "data_wait_seconds": 1.940131187438965e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20768, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 21630, "total_steps": 72479, "tokens": 11340349440, "loss": 4.822332859039307, "l_mix": 2.987152338027954, "l_a": 3.077087640762329, "l_c": 3.034633755683899, "z_loss": 16.63957691192627, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18156485259532928, "tokens_per_second": 1369991.2633933297, "eta_seconds": 19459.628119063378, "data_wait_seconds": 3.266334533691406e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 21640, "total_steps": 72479, "tokens": 11345592320, "loss": 4.995989084243774, "l_mix": 3.0953144133090973, "l_a": 3.1850191950798035, "l_c": 3.1450655460357666, "z_loss": 16.492722034454346, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17037492990493774, "tokens_per_second": 1423410.8249358486, "eta_seconds": 18725.63926384449, "data_wait_seconds": 3.311038017272949e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 21650, "total_steps": 72479, "tokens": 11350835200, "loss": 4.927717983722687, "l_mix": 3.053661823272705, "l_a": 3.1398905813694, "l_c": 3.101342052221298, "z_loss": 16.862004041671753, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18206748366355896, "tokens_per_second": 1461060.6573959284, "eta_seconds": 18239.512929940225, "data_wait_seconds": 2.6673078536987305e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08236}}
{"step": 21660, "total_steps": 72479, "tokens": 11356078080, "loss": 3.2451529502868652, "l_mix": 2.005177915096283, "l_a": 2.085606098175049, "l_c": 2.041149154305458, "z_loss": 19.48415184020996, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.263446569442749, "tokens_per_second": 1217522.1541022367, "eta_seconds": 21883.619761848448, "data_wait_seconds": 2.7745962142944336e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08236}}
{"step": 21670, "total_steps": 72479, "tokens": 11361320960, "loss": 4.694383978843689, "l_mix": 2.906019300222397, "l_a": 3.0029331743717194, "l_c": 2.952716886997223, "z_loss": 16.69637441635132, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15013669431209564, "tokens_per_second": 1415800.641637413, "eta_seconds": 18815.18358488083, "data_wait_seconds": 2.2530555725097656e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 21680, "total_steps": 72479, "tokens": 11366563840, "loss": 4.7749879360198975, "l_mix": 2.9565626084804535, "l_a": 3.0510909259319305, "l_c": 3.004691630601883, "z_loss": 16.906537890434265, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16391190886497498, "tokens_per_second": 1314195.4409386853, "eta_seconds": 20265.864027786254, "data_wait_seconds": 2.2739171981811523e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 21690, "total_steps": 72479, "tokens": 11371806720, "loss": 4.617952048778534, "l_mix": 2.858400583267212, "l_a": 2.956868350505829, "l_c": 2.9027107059955597, "z_loss": 16.776187658309937, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15651951730251312, "tokens_per_second": 1462060.3380747736, "eta_seconds": 18212.69788842201, "data_wait_seconds": 3.6150217056274414e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20765, "cosmopedia_v2": 0.20041, "finemath": 0.08234}}
{"step": 21700, "total_steps": 72479, "tokens": 11377049600, "loss": 4.666201531887054, "l_mix": 2.8886047303676605, "l_a": 2.985021084547043, "l_c": 2.9347949624061584, "z_loss": 16.52013599872589, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1730557233095169, "tokens_per_second": 1421429.3400675233, "eta_seconds": 18729.61223013401, "data_wait_seconds": 1.9818544387817383e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20765, "cosmopedia_v2": 0.20041, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.30621883, "gate_a2b/L1_absmax": 0.65815371, "gate_a2b/L1_rms": 0.32300067, "gate_a2b/L3_absmean": 0.3359856, "gate_a2b/L3_absmax": 0.51960939, "gate_a2b/L3_rms": 0.34064311, "gate_a2b/L5_absmean": 0.51618284, "gate_a2b/L5_absmax": 0.68978643, "gate_a2b/L5_rms": 0.52199984, "gate_c2b/L1_absmean": 0.01469463, "gate_c2b/L1_absmax": 0.43523964, "gate_c2b/L1_rms": 0.05211915, "gate_c2b/L3_absmean": 0.20895982, "gate_c2b/L3_absmax": 0.43296364, "gate_c2b/L3_rms": 0.21303114, "gate_c2b/L5_absmean": 0.52957141, "gate_c2b/L5_absmax": 0.71919519, "gate_c2b/L5_rms": 0.5411613, "gate_b2a/L7_absmean": 0.08632223, "gate_b2a/L7_absmax": 0.27964956, "gate_b2a/L7_rms": 0.08961208, "gate_b2a/L11_absmean": 0.13261542, "gate_b2a/L11_absmax": 0.45685536, "gate_b2a/L11_rms": 0.13554987, "gate_b2a/L14_absmean": 0.31903568, "gate_b2a/L14_absmax": 0.54075396, "gate_b2a/L14_rms": 0.32041982, "gate_b2c/L3_absmean": 0.09973826, "gate_b2c/L3_absmax": 0.61796367, "gate_b2c/L3_rms": 0.11650938, "gate_b2c/L5_absmean": 0.12499296, "gate_b2c/L5_absmax": 0.76926482, "gate_b2c/L5_rms": 0.13281541, "gate_b2c/L6_absmean": 0.18309128, "gate_b2c/L6_absmax": 0.51219559, "gate_b2c/L6_rms": 0.18733066}, "streams": {"cos_a2b/round0": 0.390539, "cos_c2b/round0": 0.290081, "cos_b2a/round0": 0.127525, "cos_b2c/round0": 0.154729, "rms_a/round0": 0.830115, "rms_b/round0": 1.570192, "rms_c/round0": 1.240811, "cos_a2b/round1": 0.415461, "cos_c2b/round1": 0.275771, "cos_b2a/round1": 0.230316, "cos_b2c/round1": 0.13012, "rms_a/round1": 1.474445, "rms_b/round1": 3.066298, "rms_c/round1": 2.033061, "cos_a2b/round2": 0.549831, "cos_c2b/round2": 0.489035, "cos_b2a/round2": 0.083251, "cos_b2c/round2": 0.146823, "rms_a/round2": 3.34483, "rms_b/round2": 6.905101, "rms_c/round2": 3.252811}, "alpha_mean": 0.45907373748653413, "alpha_hist": [0.01312, 0.06335, 0.14201, 0.17734, 0.1825, 0.18239, 0.13089, 0.08438, 0.02368, 0.00034], "alpha_by_type": {"special": {"share": 0.0009, "mean": 0.57513, "std": 0.10743}, "whitespace": {"share": 0.02475, "mean": 0.52945, "std": 0.11968}, "digit": {"share": 0.03247, "mean": 0.50829, "std": 0.10843}, "word_start": {"share": 0.59984, "mean": 0.48458, "std": 0.16467}, "word_cont": {"share": 0.24394, "mean": 0.31428, "std": 0.13908}, "punct": {"share": 0.0981, "mean": 0.62807, "std": 0.16122}}, "alpha_p05": 0.16749, "alpha_p50": 0.45193, "alpha_p95": 0.77594}
{"step": 21710, "total_steps": 72479, "tokens": 11382292480, "loss": 4.745805144309998, "l_mix": 2.9370696544647217, "l_a": 3.0376544892787933, "l_c": 2.9857553839683533, "z_loss": 17.125312328338623, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17994318902492523, "tokens_per_second": 1455588.7334466593, "eta_seconds": 18286.468464875223, "data_wait_seconds": 2.250075340270996e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20041, "finemath": 0.08236}}
{"step": 21720, "total_steps": 72479, "tokens": 11387535360, "loss": 3.7044096887111664, "l_mix": 2.292009085416794, "l_a": 2.3769021928310394, "l_c": 2.324842184782028, "z_loss": 18.772151470184326, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.3075809180736542, "tokens_per_second": 1419665.3237674253, "eta_seconds": 18745.498778104782, "data_wait_seconds": 2.619624137878418e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20041, "finemath": 0.08237}}
{"step": 21730, "total_steps": 72479, "tokens": 11392778240, "loss": 4.94368714094162, "l_mix": 3.063481330871582, "l_a": 3.1503758430480957, "l_c": 3.111368805170059, "z_loss": 16.823484897613525, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16288617253303528, "tokens_per_second": 1464414.605654432, "eta_seconds": 18169.09747366905, "data_wait_seconds": 1.868605613708496e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20769, "cosmopedia_v2": 0.20041, "finemath": 0.08233}}
{"step": 21740, "total_steps": 72479, "tokens": 11398021120, "loss": 4.933119714260101, "l_mix": 3.05635142326355, "l_a": 3.14436537027359, "l_c": 3.1059722006320953, "z_loss": 16.669636011123657, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19877837598323822, "tokens_per_second": 1452056.6891410619, "eta_seconds": 18320.117272925374, "data_wait_seconds": 1.9669532775878906e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20768, "cosmopedia_v2": 0.20041, "finemath": 0.08234}}
{"step": 21750, "total_steps": 72479, "tokens": 11403264000, "loss": 5.053986072540283, "l_mix": 3.1337258517742157, "l_a": 3.2150011360645294, "l_c": 3.1803808510303497, "z_loss": 16.45479905605316, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19497953355312347, "tokens_per_second": 1465536.110555836, "eta_seconds": 18148.038632710774, "data_wait_seconds": 2.4199485778808594e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.20041, "finemath": 0.08234}}
{"step": 21760, "total_steps": 72479, "tokens": 11408506880, "loss": 4.9907695055007935, "l_mix": 3.0928570330142975, "l_a": 3.18188938498497, "l_c": 3.1388201117515564, "z_loss": 16.994737148284912, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17160628736019135, "tokens_per_second": 1356199.0069644616, "eta_seconds": 19607.272188997267, "data_wait_seconds": 2.98917293548584e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 21770, "total_steps": 72479, "tokens": 11413749760, "loss": 5.004967212677002, "l_mix": 3.1021352112293243, "l_a": 3.1877934634685516, "l_c": 3.149483799934387, "z_loss": 16.485557317733765, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18539069592952728, "tokens_per_second": 1450954.2120590373, "eta_seconds": 18323.197224998476, "data_wait_seconds": 3.254413604736328e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08236}}
{"step": 21780, "total_steps": 72479, "tokens": 11418992640, "loss": 4.760137677192688, "l_mix": 2.946899712085724, "l_a": 3.044481337070465, "l_c": 2.9939653277397156, "z_loss": 17.038306713104248, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16027891635894775, "tokens_per_second": 1424543.158603097, "eta_seconds": 18659.229207253455, "data_wait_seconds": 2.0205974578857422e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08232}}
{"step": 21790, "total_steps": 72479, "tokens": 11424235520, "loss": 4.713673949241638, "l_mix": 2.918332576751709, "l_a": 3.0134575068950653, "l_c": 2.965402364730835, "z_loss": 16.83364772796631, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16206692159175873, "tokens_per_second": 1381318.660784577, "eta_seconds": 19239.321951174734, "data_wait_seconds": 2.6106834411621094e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20766, "cosmopedia_v2": 0.20041, "finemath": 0.08233}}
{"step": 21800, "total_steps": 72479, "tokens": 11429478400, "loss": 4.6465983390808105, "l_mix": 2.8758564889431, "l_a": 2.9778894782066345, "l_c": 2.918901950120926, "z_loss": 17.043574571609497, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21786734461784363, "tokens_per_second": 1322943.725945269, "eta_seconds": 20084.29461579323, "data_wait_seconds": 2.467632293701172e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20766, "cosmopedia_v2": 0.20041, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.30629048, "gate_a2b/L1_absmax": 0.66356987, "gate_a2b/L1_rms": 0.32306549, "gate_a2b/L3_absmean": 0.3354803, "gate_a2b/L3_absmax": 0.51942199, "gate_a2b/L3_rms": 0.34006375, "gate_a2b/L5_absmean": 0.515926, "gate_a2b/L5_absmax": 0.68901032, "gate_a2b/L5_rms": 0.52173847, "gate_c2b/L1_absmean": 0.01464002, "gate_c2b/L1_absmax": 0.43360412, "gate_c2b/L1_rms": 0.05238542, "gate_c2b/L3_absmean": 0.20979571, "gate_c2b/L3_absmax": 0.42759901, "gate_c2b/L3_rms": 0.21386696, "gate_c2b/L5_absmean": 0.5294348, "gate_c2b/L5_absmax": 0.72011197, "gate_c2b/L5_rms": 0.54104304, "gate_b2a/L7_absmean": 0.08572796, "gate_b2a/L7_absmax": 0.27780345, "gate_b2a/L7_rms": 0.08902153, "gate_b2a/L11_absmean": 0.13281403, "gate_b2a/L11_absmax": 0.45261052, "gate_b2a/L11_rms": 0.13568546, "gate_b2a/L14_absmean": 0.31941137, "gate_b2a/L14_absmax": 0.55103177, "gate_b2a/L14_rms": 0.32084087, "gate_b2c/L3_absmean": 0.09986558, "gate_b2c/L3_absmax": 0.61881876, "gate_b2c/L3_rms": 0.11659125, "gate_b2c/L5_absmean": 0.1250058, "gate_b2c/L5_absmax": 0.76627201, "gate_b2c/L5_rms": 0.13284205, "gate_b2c/L6_absmean": 0.18295036, "gate_b2c/L6_absmax": 0.52001888, "gate_b2c/L6_rms": 0.18723372}, "streams": {"cos_a2b/round0": 0.381938, "cos_c2b/round0": 0.285697, "cos_b2a/round0": 0.12685, "cos_b2c/round0": 0.151204, "rms_a/round0": 0.839445, "rms_b/round0": 1.58948, "rms_c/round0": 1.238588, "cos_a2b/round1": 0.423523, "cos_c2b/round1": 0.276955, "cos_b2a/round1": 0.224713, "cos_b2c/round1": 0.132894, "rms_a/round1": 1.50723, "rms_b/round1": 3.117717, "rms_c/round1": 2.043891, "cos_a2b/round2": 0.553638, "cos_c2b/round2": 0.479465, "cos_b2a/round2": 0.077658, "cos_b2c/round2": 0.146207, "rms_a/round2": 3.402478, "rms_b/round2": 6.968014, "rms_c/round2": 3.30456}, "alpha_mean": 0.47156625447446743, "alpha_hist": [0.01454, 0.06248, 0.12807, 0.15768, 0.17244, 0.19746, 0.15245, 0.08826, 0.02631, 0.00031], "alpha_by_type": {"special": {"share": 0.00073, "mean": 0.53948, "std": 0.11128}, "whitespace": {"share": 0.02356, "mean": 0.52026, "std": 0.11111}, "digit": {"share": 0.02171, "mean": 0.41797, "std": 0.12928}, "word_start": {"share": 0.57802, "mean": 0.50688, "std": 0.16301}, "word_cont": {"share": 0.26048, "mean": 0.32935, "std": 0.1493}, "punct": {"share": 0.11549, "mean": 0.61528, "std": 0.16635}}, "alpha_p05": 0.16318, "alpha_p50": 0.48746, "alpha_p95": 0.75707}
{"step": 21810, "total_steps": 72479, "tokens": 11434721280, "loss": 4.763576090335846, "l_mix": 2.947807103395462, "l_a": 3.047493726015091, "l_c": 2.9994554221630096, "z_loss": 16.842862844467163, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18605291843414307, "tokens_per_second": 1485150.379207088, "eta_seconds": 17887.17765145302, "data_wait_seconds": 1.5527009963989258e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 21820, "total_steps": 72479, "tokens": 11439964160, "loss": 4.729968190193176, "l_mix": 2.9279634952545166, "l_a": 3.0270348489284515, "l_c": 2.9739151298999786, "z_loss": 17.196736335754395, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19879533350467682, "tokens_per_second": 1410974.7881888736, "eta_seconds": 18823.79898941517, "data_wait_seconds": 2.148747444152832e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20041, "finemath": 0.08236}}
{"step": 21830, "total_steps": 72479, "tokens": 11445207040, "loss": 3.2399507462978363, "l_mix": 2.002823084592819, "l_a": 2.0802847743034363, "l_c": 2.0374587178230286, "z_loss": 18.04652500152588, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.3021581172943115, "tokens_per_second": 1394876.3451403917, "eta_seconds": 19037.288147091866, "data_wait_seconds": 1.9729137420654297e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20041, "finemath": 0.08236}}
{"step": 21840, "total_steps": 72479, "tokens": 11450449920, "loss": 3.4431717693805695, "l_mix": 2.1166549026966095, "l_a": 2.24072602391243, "l_c": 2.1757507026195526, "z_loss": 15.738446474075317, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21722473204135895, "tokens_per_second": 1462693.9208329779, "eta_seconds": 18151.042848992347, "data_wait_seconds": 3.361701965332031e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20046, "finemath": 0.08233}}
{"step": 21850, "total_steps": 72479, "tokens": 11455692800, "loss": 3.45966836810112, "l_mix": 2.127353608608246, "l_a": 2.2525859773159027, "l_c": 2.1831697821617126, "z_loss": 15.880078315734863, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22324125468730927, "tokens_per_second": 1462570.6622681378, "eta_seconds": 18148.98783135414, "data_wait_seconds": 2.2709369659423828e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20764, "cosmopedia_v2": 0.20046, "finemath": 0.08233}}
{"step": 21860, "total_steps": 72479, "tokens": 11460935680, "loss": 4.957410931587219, "l_mix": 3.0717695355415344, "l_a": 3.1598106026649475, "l_c": 3.1200761795043945, "z_loss": 16.751691818237305, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16585087776184082, "tokens_per_second": 1482783.311650582, "eta_seconds": 17898.052981495857, "data_wait_seconds": 2.5898218154907227e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20769, "cosmopedia_v2": 0.20041, "finemath": 0.08234}}
{"step": 21870, "total_steps": 72479, "tokens": 11466178560, "loss": 4.993281185626984, "l_mix": 3.0942614376544952, "l_a": 3.1795068085193634, "l_c": 3.145015984773636, "z_loss": 16.627164363861084, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.193254753947258, "tokens_per_second": 1463924.7264070788, "eta_seconds": 18125.03806607723, "data_wait_seconds": 2.03549861907959e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 21880, "total_steps": 72479, "tokens": 11471421440, "loss": 4.905746638774872, "l_mix": 3.0406835079193115, "l_a": 3.1249621212482452, "l_c": 3.08623144030571, "z_loss": 17.05057406425476, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1533309519290924, "tokens_per_second": 1425003.6146703963, "eta_seconds": 18616.407873558997, "data_wait_seconds": 3.1828880310058594e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20768, "cosmopedia_v2": 0.20041, "finemath": 0.08236}}
{"step": 21890, "total_steps": 72479, "tokens": 11476664320, "loss": 4.759865164756775, "l_mix": 2.9471087753772736, "l_a": 3.039603739976883, "l_c": 2.997250109910965, "z_loss": 17.002386331558228, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20565031468868256, "tokens_per_second": 1475942.89238792, "eta_seconds": 17970.34679918289, "data_wait_seconds": 2.0325183868408203e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08232}}
{"step": 21900, "total_steps": 72479, "tokens": 11481907200, "loss": 4.864246726036072, "l_mix": 3.0126373171806335, "l_a": 3.106637179851532, "l_c": 3.0597443282604218, "z_loss": 16.948426485061646, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16208651661872864, "tokens_per_second": 1382256.1271214641, "eta_seconds": 19184.550700616837, "data_wait_seconds": 3.412365913391113e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.30697677, "gate_a2b/L1_absmax": 0.658961, "gate_a2b/L1_rms": 0.32349873, "gate_a2b/L3_absmean": 0.33532783, "gate_a2b/L3_absmax": 0.51687723, "gate_a2b/L3_rms": 0.33990017, "gate_a2b/L5_absmean": 0.51561874, "gate_a2b/L5_absmax": 0.68762809, "gate_a2b/L5_rms": 0.52143699, "gate_c2b/L1_absmean": 0.01403331, "gate_c2b/L1_absmax": 0.4339197, "gate_c2b/L1_rms": 0.05239573, "gate_c2b/L3_absmean": 0.21000433, "gate_c2b/L3_absmax": 0.43559411, "gate_c2b/L3_rms": 0.2140979, "gate_c2b/L5_absmean": 0.52952623, "gate_c2b/L5_absmax": 0.72061002, "gate_c2b/L5_rms": 0.54114944, "gate_b2a/L7_absmean": 0.0859836, "gate_b2a/L7_absmax": 0.27438453, "gate_b2a/L7_rms": 0.08922188, "gate_b2a/L11_absmean": 0.13256103, "gate_b2a/L11_absmax": 0.45155054, "gate_b2a/L11_rms": 0.13538148, "gate_b2a/L14_absmean": 0.31993273, "gate_b2a/L14_absmax": 0.55126488, "gate_b2a/L14_rms": 0.3213352, "gate_b2c/L3_absmean": 0.10039622, "gate_b2c/L3_absmax": 0.6176132, "gate_b2c/L3_rms": 0.11690066, "gate_b2c/L5_absmean": 0.12527815, "gate_b2c/L5_absmax": 0.76824266, "gate_b2c/L5_rms": 0.13312837, "gate_b2c/L6_absmean": 0.18328677, "gate_b2c/L6_absmax": 0.52118015, "gate_b2c/L6_rms": 0.18757044}, "streams": {"cos_a2b/round0": 0.385759, "cos_c2b/round0": 0.294652, "cos_b2a/round0": 0.124941, "cos_b2c/round0": 0.147221, "rms_a/round0": 0.835869, "rms_b/round0": 1.565343, "rms_c/round0": 1.254911, "cos_a2b/round1": 0.413956, "cos_c2b/round1": 0.268124, "cos_b2a/round1": 0.231679, "cos_b2c/round1": 0.121912, "rms_a/round1": 1.477869, "rms_b/round1": 3.090945, "rms_c/round1": 2.051611, "cos_a2b/round2": 0.538385, "cos_c2b/round2": 0.491316, "cos_b2a/round2": 0.076063, "cos_b2c/round2": 0.136847, "rms_a/round2": 3.3164, "rms_b/round2": 6.893703, "rms_c/round2": 3.304847}, "alpha_mean": 0.4332973410239873, "alpha_hist": [0.01163, 0.06517, 0.15869, 0.20753, 0.20531, 0.17386, 0.11455, 0.05664, 0.00655, 8e-05], "alpha_by_type": {"special": {"share": 0.00142, "mean": 0.55171, "std": 0.10226}, "whitespace": {"share": 0.02971, "mean": 0.52627, "std": 0.11019}, "digit": {"share": 0.04253, "mean": 0.47068, "std": 0.11802}, "word_start": {"share": 0.54156, "mean": 0.45201, "std": 0.15747}, "word_cont": {"share": 0.26038, "mean": 0.32453, "std": 0.13768}, "punct": {"share": 0.1244, "mean": 0.54316, "std": 0.15682}}, "alpha_p05": 0.17982, "alpha_p50": 0.43518, "alpha_p95": 0.70881}
{"step": 21910, "total_steps": 72479, "tokens": 11487150080, "loss": 4.754666328430176, "l_mix": 2.943745106458664, "l_a": 3.037681996822357, "l_c": 2.9929617643356323, "z_loss": 17.280073881149292, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17609207332134247, "tokens_per_second": 1440925.1542979283, "eta_seconds": 18399.789741277695, "data_wait_seconds": 1.9729137420654297e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08234}}
{"step": 21920, "total_steps": 72479, "tokens": 11492392960, "loss": 4.691079795360565, "l_mix": 2.9034229516983032, "l_a": 3.0022372007369995, "l_c": 2.9509848058223724, "z_loss": 16.90148115158081, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15708495676517487, "tokens_per_second": 1426552.9593927138, "eta_seconds": 18581.488207268718, "data_wait_seconds": 1.671910285949707e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20041, "finemath": 0.08234}}
{"step": 21930, "total_steps": 72479, "tokens": 11497635840, "loss": 4.755388557910919, "l_mix": 2.9439550042152405, "l_a": 3.039539724588394, "l_c": 2.9929469525814056, "z_loss": 16.87542223930359, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1598156988620758, "tokens_per_second": 1429206.945776111, "eta_seconds": 18543.31466155052, "data_wait_seconds": 2.0951032638549805e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 21940, "total_steps": 72479, "tokens": 11502878720, "loss": 4.72368311882019, "l_mix": 2.9244548678398132, "l_a": 3.0198774337768555, "l_c": 2.971792072057724, "z_loss": 17.273228883743286, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20213943719863892, "tokens_per_second": 1426893.0453111941, "eta_seconds": 18569.7108266592, "data_wait_seconds": 1.8268823623657227e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20041, "finemath": 0.08236}}
{"step": 21950, "total_steps": 72479, "tokens": 11508121600, "loss": 3.5238469541072845, "l_mix": 2.16620209813118, "l_a": 2.2926391065120697, "l_c": 2.227655977010727, "z_loss": 15.562458395957947, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22046566009521484, "tokens_per_second": 1473959.0621429717, "eta_seconds": 17973.191408371928, "data_wait_seconds": 2.6226043701171875e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20046, "finemath": 0.08232}}
{"step": 21960, "total_steps": 72479, "tokens": 11513364480, "loss": 3.4416992366313934, "l_mix": 2.115978091955185, "l_a": 2.2394557893276215, "l_c": 2.1744415163993835, "z_loss": 15.519039869308472, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22144824266433716, "tokens_per_second": 1423642.8619256474, "eta_seconds": 18604.740121531486, "data_wait_seconds": 3.364682197570801e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20046, "finemath": 0.08233}}
{"step": 21970, "total_steps": 72479, "tokens": 11518607360, "loss": 3.464682102203369, "l_mix": 2.1300489604473114, "l_a": 2.2564720809459686, "l_c": 2.187060207128525, "z_loss": 15.733950853347778, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17156259715557098, "tokens_per_second": 1472252.5544228905, "eta_seconds": 17986.902119779585, "data_wait_seconds": 2.065300941467285e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20046, "finemath": 0.08234}}
{"step": 21980, "total_steps": 72479, "tokens": 11523850240, "loss": 3.4876529574394226, "l_mix": 2.1448687314987183, "l_a": 2.267567604780197, "l_c": 2.203076273202896, "z_loss": 15.90971326828003, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20977091789245605, "tokens_per_second": 1444557.1845477503, "eta_seconds": 18328.121583008768, "data_wait_seconds": 3.057718276977539e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20764, "cosmopedia_v2": 0.20045, "finemath": 0.08235}}
{"step": 21990, "total_steps": 72479, "tokens": 11529093120, "loss": 3.4858180582523346, "l_mix": 2.1423808336257935, "l_a": 2.270698308944702, "l_c": 2.202120065689087, "z_loss": 15.917224526405334, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.24587886035442352, "tokens_per_second": 1421636.1000156642, "eta_seconds": 18619.93855650425, "data_wait_seconds": 2.60770320892334e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20764, "cosmopedia_v2": 0.20045, "finemath": 0.08236}}
{"step": 22000, "total_steps": 72479, "tokens": 11534336000, "loss": 4.741678357124329, "l_mix": 2.934418499469757, "l_a": 3.036366045475006, "l_c": 2.9822793006896973, "z_loss": 16.661499500274658, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1993112862110138, "tokens_per_second": 1469606.241618354, "eta_seconds": 18008.588424921036, "data_wait_seconds": 1.963973045349121e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20764, "cosmopedia_v2": 0.20041, "finemath": 0.08236}, "gates": {"gate_a2b/L1_absmean": 0.30728808, "gate_a2b/L1_absmax": 0.662395, "gate_a2b/L1_rms": 0.32383603, "gate_a2b/L3_absmean": 0.33623296, "gate_a2b/L3_absmax": 0.5168283, "gate_a2b/L3_rms": 0.34076276, "gate_a2b/L5_absmean": 0.51623571, "gate_a2b/L5_absmax": 0.68938804, "gate_a2b/L5_rms": 0.5220437, "gate_c2b/L1_absmean": 0.01483373, "gate_c2b/L1_absmax": 0.43450147, "gate_c2b/L1_rms": 0.05262426, "gate_c2b/L3_absmean": 0.20911913, "gate_c2b/L3_absmax": 0.43303403, "gate_c2b/L3_rms": 0.21311432, "gate_c2b/L5_absmean": 0.52980447, "gate_c2b/L5_absmax": 0.72392893, "gate_c2b/L5_rms": 0.54140759, "gate_b2a/L7_absmean": 0.08581873, "gate_b2a/L7_absmax": 0.27432457, "gate_b2a/L7_rms": 0.08907838, "gate_b2a/L11_absmean": 0.13229078, "gate_b2a/L11_absmax": 0.45022774, "gate_b2a/L11_rms": 0.13509442, "gate_b2a/L14_absmean": 0.31978595, "gate_b2a/L14_absmax": 0.55186623, "gate_b2a/L14_rms": 0.32120895, "gate_b2c/L3_absmean": 0.10059192, "gate_b2c/L3_absmax": 0.61823815, "gate_b2c/L3_rms": 0.11697954, "gate_b2c/L5_absmean": 0.12565708, "gate_b2c/L5_absmax": 0.76939744, "gate_b2c/L5_rms": 0.13347557, "gate_b2c/L6_absmean": 0.18331103, "gate_b2c/L6_absmax": 0.5318979, "gate_b2c/L6_rms": 0.18774302}, "streams": {"cos_a2b/round0": 0.382565, "cos_c2b/round0": 0.293484, "cos_b2a/round0": 0.122001, "cos_b2c/round0": 0.146095, "rms_a/round0": 0.822417, "rms_b/round0": 1.56061, "rms_c/round0": 1.250704, "cos_a2b/round1": 0.414562, "cos_c2b/round1": 0.269306, "cos_b2a/round1": 0.227113, "cos_b2c/round1": 0.122708, "rms_a/round1": 1.463664, "rms_b/round1": 3.073097, "rms_c/round1": 2.036823, "cos_a2b/round2": 0.5302, "cos_c2b/round2": 0.498518, "cos_b2a/round2": 0.068329, "cos_b2c/round2": 0.131301, "rms_a/round2": 3.298272, "rms_b/round2": 6.900546, "rms_c/round2": 3.312754}, "alpha_mean": 0.4491278470723472, "alpha_hist": [0.01282, 0.06126, 0.14896, 0.20241, 0.19116, 0.16187, 0.12312, 0.07663, 0.02127, 0.0005], "alpha_by_type": {"special": {"share": 0.00108, "mean": 0.57292, "std": 0.09251}, "whitespace": {"share": 0.02977, "mean": 0.54545, "std": 0.12508}, "digit": {"share": 0.04022, "mean": 0.45205, "std": 0.12001}, "word_start": {"share": 0.55049, "mean": 0.47137, "std": 0.16944}, "word_cont": {"share": 0.2648, "mean": 0.33278, "std": 0.13978}, "punct": {"share": 0.11363, "mean": 0.58506, "std": 0.1717}}, "alpha_p05": 0.17953, "alpha_p50": 0.42632, "alpha_p95": 0.75923}
{"step": 22010, "total_steps": 72479, "tokens": 11539578880, "loss": 4.711372554302216, "l_mix": 2.9160050749778748, "l_a": 3.0126257836818695, "l_c": 2.9663349390029907, "z_loss": 16.7924222946167, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21649137139320374, "tokens_per_second": 1405432.193983638, "eta_seconds": 18827.15593485832, "data_wait_seconds": 1.9788742065429688e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20768, "cosmopedia_v2": 0.20041, "finemath": 0.08233}}
{"step": 22020, "total_steps": 72479, "tokens": 11544821760, "loss": 4.809196293354034, "l_mix": 2.978349506855011, "l_a": 3.0731201469898224, "l_c": 3.024114668369293, "z_loss": 16.76335096359253, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1685904711484909, "tokens_per_second": 1391947.2969333997, "eta_seconds": 19005.78294184208, "data_wait_seconds": 1.5735626220703125e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08233}}
{"step": 22030, "total_steps": 72479, "tokens": 11550064640, "loss": 4.873739719390869, "l_mix": 3.0182072818279266, "l_a": 3.1132847368717194, "l_c": 3.0662083327770233, "z_loss": 16.844366908073425, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16239377856254578, "tokens_per_second": 1483875.1279665127, "eta_seconds": 17824.81882302761, "data_wait_seconds": 1.6987323760986328e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08234}}
{"step": 22040, "total_steps": 72479, "tokens": 11555307520, "loss": 4.67382550239563, "l_mix": 2.893040806055069, "l_a": 2.9925876557826996, "l_c": 2.937727600336075, "z_loss": 16.900983333587646, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16334770619869232, "tokens_per_second": 1329358.0299642233, "eta_seconds": 19892.73155608177, "data_wait_seconds": 2.0295381546020508e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 22050, "total_steps": 72479, "tokens": 11560550400, "loss": 4.6516430377960205, "l_mix": 2.87788125872612, "l_a": 2.9781758785247803, "l_c": 2.928663283586502, "z_loss": 17.09933614730835, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1915515959262848, "tokens_per_second": 1336226.7322194704, "eta_seconds": 19786.551873636246, "data_wait_seconds": 3.1948089599609375e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20041, "finemath": 0.08236}}
{"step": 22060, "total_steps": 72479, "tokens": 11565793280, "loss": 3.451564759016037, "l_mix": 2.122030407190323, "l_a": 2.2452515959739685, "l_c": 2.1812108159065247, "z_loss": 15.955925703048706, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21168407797813416, "tokens_per_second": 1332234.8755239903, "eta_seconds": 19841.904124903678, "data_wait_seconds": 4.8726797103881836e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20045, "finemath": 0.08232}}
{"step": 22070, "total_steps": 72479, "tokens": 11571036160, "loss": 3.411700665950775, "l_mix": 2.0968263745307922, "l_a": 2.2192322313785553, "l_c": 2.158462643623352, "z_loss": 15.65766727924347, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20246660709381104, "tokens_per_second": 1451982.9595867037, "eta_seconds": 18201.889779424666, "data_wait_seconds": 2.47955322265625e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20045, "finemath": 0.08233}}
{"step": 22080, "total_steps": 72479, "tokens": 11576279040, "loss": 3.4559971392154694, "l_mix": 2.124389261007309, "l_a": 2.2510659098625183, "l_c": 2.182395875453949, "z_loss": 15.692691326141357, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20317649841308594, "tokens_per_second": 1430068.9102222768, "eta_seconds": 18477.1452082634, "data_wait_seconds": 4.5925378799438477e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20045, "finemath": 0.08234}}
{"step": 22090, "total_steps": 72479, "tokens": 11581521920, "loss": 3.4122759997844696, "l_mix": 2.0974991619586945, "l_a": 2.224449634552002, "l_c": 2.152994304895401, "z_loss": 15.435882449150085, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20150379836559296, "tokens_per_second": 1459825.4544875184, "eta_seconds": 18096.9224442482, "data_wait_seconds": 3.364682197570801e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20765, "cosmopedia_v2": 0.20045, "finemath": 0.08234}}
{"step": 22100, "total_steps": 72479, "tokens": 11586764800, "loss": 3.455884277820587, "l_mix": 2.123983711004257, "l_a": 2.250155448913574, "l_c": 2.1842488646507263, "z_loss": 15.792147994041443, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22790288925170898, "tokens_per_second": 1451971.1673876594, "eta_seconds": 18191.204994463922, "data_wait_seconds": 2.002716064453125e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20765, "cosmopedia_v2": 0.20045, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.30742174, "gate_a2b/L1_absmax": 0.65904504, "gate_a2b/L1_rms": 0.32418081, "gate_a2b/L3_absmean": 0.33564022, "gate_a2b/L3_absmax": 0.52524579, "gate_a2b/L3_rms": 0.34027487, "gate_a2b/L5_absmean": 0.51629269, "gate_a2b/L5_absmax": 0.69124943, "gate_a2b/L5_rms": 0.52211088, "gate_c2b/L1_absmean": 0.01440584, "gate_c2b/L1_absmax": 0.43219405, "gate_c2b/L1_rms": 0.05263732, "gate_c2b/L3_absmean": 0.20915072, "gate_c2b/L3_absmax": 0.43377963, "gate_c2b/L3_rms": 0.21329266, "gate_c2b/L5_absmean": 0.52810442, "gate_c2b/L5_absmax": 0.71976095, "gate_c2b/L5_rms": 0.53963315, "gate_b2a/L7_absmean": 0.08648493, "gate_b2a/L7_absmax": 0.27157792, "gate_b2a/L7_rms": 0.08981058, "gate_b2a/L11_absmean": 0.13429525, "gate_b2a/L11_absmax": 0.44712675, "gate_b2a/L11_rms": 0.13716131, "gate_b2a/L14_absmean": 0.31796962, "gate_b2a/L14_absmax": 0.54818559, "gate_b2a/L14_rms": 0.31954578, "gate_b2c/L3_absmean": 0.10060689, "gate_b2c/L3_absmax": 0.62571526, "gate_b2c/L3_rms": 0.11708268, "gate_b2c/L5_absmean": 0.12454482, "gate_b2c/L5_absmax": 0.76966125, "gate_b2c/L5_rms": 0.13251919, "gate_b2c/L6_absmean": 0.18318331, "gate_b2c/L6_absmax": 0.53132749, "gate_b2c/L6_rms": 0.18778425}, "streams": {"cos_a2b/round0": 0.374709, "cos_c2b/round0": 0.261051, "cos_b2a/round0": 0.10713, "cos_b2c/round0": 0.10807, "rms_a/round0": 0.867655, "rms_b/round0": 1.572246, "rms_c/round0": 1.247835, "cos_a2b/round1": 0.407131, "cos_c2b/round1": 0.269558, "cos_b2a/round1": 0.2113, "cos_b2c/round1": 0.12361, "rms_a/round1": 1.539103, "rms_b/round1": 3.21807, "rms_c/round1": 2.151132, "cos_a2b/round2": 0.499326, "cos_c2b/round2": 0.481243, "cos_b2a/round2": 0.025697, "cos_b2c/round2": 0.102666, "rms_a/round2": 3.347219, "rms_b/round2": 7.159681, "rms_c/round2": 3.494868}, "alpha_mean": 0.42326616762638736, "alpha_hist": [0.06067, 0.14827, 0.16676, 0.13728, 0.11368, 0.10332, 0.10394, 0.09615, 0.0609, 0.00903], "alpha_by_type": {"special": {"share": 0.0011, "mean": 0.61808, "std": 0.08005}, "whitespace": {"share": 0.04636, "mean": 0.629, "std": 0.17276}, "digit": {"share": 0.01205, "mean": 0.42341, "std": 0.16527}, "word_start": {"share": 0.53802, "mean": 0.43492, "std": 0.22206}, "word_cont": {"share": 0.25993, "mean": 0.25625, "std": 0.15846}, "punct": {"share": 0.14253, "mean": 0.61545, "std": 0.20467}}, "alpha_p05": 0.09947, "alpha_p50": 0.38514, "alpha_p95": 0.79724}
{"step": 22110, "total_steps": 72479, "tokens": 11592007680, "loss": 4.745141983032227, "l_mix": 2.938531130552292, "l_a": 3.035182476043701, "l_c": 2.9812238812446594, "z_loss": 16.88922691345215, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.23968584835529327, "tokens_per_second": 1422166.0475784854, "eta_seconds": 18568.761585164073, "data_wait_seconds": 2.047419548034668e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20764, "cosmopedia_v2": 0.20041, "finemath": 0.08236}}
{"step": 22120, "total_steps": 72479, "tokens": 11597250560, "loss": 4.67540055513382, "l_mix": 2.8936559855937958, "l_a": 2.9929828345775604, "l_c": 2.9403545260429382, "z_loss": 17.43294382095337, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18852384388446808, "tokens_per_second": 1461179.9721136186, "eta_seconds": 18069.382208824158, "data_wait_seconds": 1.6182661056518555e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20045, "finemath": 0.08232}}
{"step": 22130, "total_steps": 72479, "tokens": 11602493440, "loss": 4.776333391666412, "l_mix": 2.9571141600608826, "l_a": 3.0544138848781586, "l_c": 3.0040103495121, "z_loss": 16.91973900794983, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17091931402683258, "tokens_per_second": 1351792.1325620105, "eta_seconds": 19527.68911442757, "data_wait_seconds": 2.3931264877319336e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20045, "finemath": 0.08233}}
{"step": 22140, "total_steps": 72479, "tokens": 11607736320, "loss": 4.717680335044861, "l_mix": 2.9201062619686127, "l_a": 3.0176795721054077, "l_c": 2.96853768825531, "z_loss": 17.088582515716553, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17042042315006256, "tokens_per_second": 1410455.408434447, "eta_seconds": 18711.78165163994, "data_wait_seconds": 2.4378299713134766e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20768, "cosmopedia_v2": 0.20041, "finemath": 0.08234}}
{"step": 22150, "total_steps": 72479, "tokens": 11612979200, "loss": 4.894705235958099, "l_mix": 3.0327517688274384, "l_a": 3.1183697283267975, "l_c": 3.0825155079364777, "z_loss": 16.87805414199829, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15719503164291382, "tokens_per_second": 1132342.6575548805, "eta_seconds": 23302.920344781876, "data_wait_seconds": 2.2530555725097656e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 22160, "total_steps": 72479, "tokens": 11618222080, "loss": 4.964138746261597, "l_mix": 3.0761643946170807, "l_a": 3.1605958342552185, "l_c": 3.127012997865677, "z_loss": 16.91592264175415, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19157876074314117, "tokens_per_second": 1452525.8957078941, "eta_seconds": 18162.60071504116, "data_wait_seconds": 1.7702579498291016e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08236}}
{"step": 22170, "total_steps": 72479, "tokens": 11623464960, "loss": 3.4933879375457764, "l_mix": 2.1485320329666138, "l_a": 2.2721382081508636, "l_c": 2.2053742110729218, "z_loss": 16.02124559879303, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.23690754175186157, "tokens_per_second": 1416399.045149738, "eta_seconds": 18622.15671658516, "data_wait_seconds": 2.473592758178711e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20045, "finemath": 0.08232}}
{"step": 22180, "total_steps": 72479, "tokens": 11628707840, "loss": 3.4716654121875763, "l_mix": 2.1347004175186157, "l_a": 2.257513105869293, "l_c": 2.19381582736969, "z_loss": 15.662708520889282, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.214420348405838, "tokens_per_second": 1413090.0073886025, "eta_seconds": 18662.0540617466, "data_wait_seconds": 2.41696834564209e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20045, "finemath": 0.08233}}
{"step": 22190, "total_steps": 72479, "tokens": 11633950720, "loss": 3.4784035682678223, "l_mix": 2.1383361518383026, "l_a": 2.2666441202163696, "l_c": 2.195012718439102, "z_loss": 15.7032071352005, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2163669466972351, "tokens_per_second": 1398500.3805305366, "eta_seconds": 18852.993963432313, "data_wait_seconds": 2.658367156982422e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20766, "cosmopedia_v2": 0.20045, "finemath": 0.08233}}
{"step": 22200, "total_steps": 72479, "tokens": 11639193600, "loss": 3.4585886001586914, "l_mix": 2.126461982727051, "l_a": 2.2502973079681396, "l_c": 2.184942603111267, "z_loss": 15.546295642852783, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2080092579126358, "tokens_per_second": 1273654.0839306577, "eta_seconds": 20696.888334584237, "data_wait_seconds": 1.6808509826660156e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20766, "cosmopedia_v2": 0.20045, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.3089264, "gate_a2b/L1_absmax": 0.64898968, "gate_a2b/L1_rms": 0.32530373, "gate_a2b/L3_absmean": 0.33566949, "gate_a2b/L3_absmax": 0.52807724, "gate_a2b/L3_rms": 0.34031805, "gate_a2b/L5_absmean": 0.51529062, "gate_a2b/L5_absmax": 0.69420093, "gate_a2b/L5_rms": 0.52112347, "gate_c2b/L1_absmean": 0.01443472, "gate_c2b/L1_absmax": 0.43653399, "gate_c2b/L1_rms": 0.05273506, "gate_c2b/L3_absmean": 0.2095366, "gate_c2b/L3_absmax": 0.4273003, "gate_c2b/L3_rms": 0.21357198, "gate_c2b/L5_absmean": 0.5280931, "gate_c2b/L5_absmax": 0.70586991, "gate_c2b/L5_rms": 0.5397023, "gate_b2a/L7_absmean": 0.08651776, "gate_b2a/L7_absmax": 0.26737475, "gate_b2a/L7_rms": 0.08959445, "gate_b2a/L11_absmean": 0.13321796, "gate_b2a/L11_absmax": 0.44989026, "gate_b2a/L11_rms": 0.13603196, "gate_b2a/L14_absmean": 0.3198131, "gate_b2a/L14_absmax": 0.5496822, "gate_b2a/L14_rms": 0.32132477, "gate_b2c/L3_absmean": 0.09988647, "gate_b2c/L3_absmax": 0.62274921, "gate_b2c/L3_rms": 0.1162345, "gate_b2c/L5_absmean": 0.12458535, "gate_b2c/L5_absmax": 0.76623827, "gate_b2c/L5_rms": 0.13256285, "gate_b2c/L6_absmean": 0.18315819, "gate_b2c/L6_absmax": 0.52533764, "gate_b2c/L6_rms": 0.18768054}, "streams": {"cos_a2b/round0": 0.382938, "cos_c2b/round0": 0.258943, "cos_b2a/round0": 0.107709, "cos_b2c/round0": 0.108837, "rms_a/round0": 0.872693, "rms_b/round0": 1.580273, "rms_c/round0": 1.263337, "cos_a2b/round1": 0.415256, "cos_c2b/round1": 0.269234, "cos_b2a/round1": 0.220005, "cos_b2c/round1": 0.136129, "rms_a/round1": 1.550625, "rms_b/round1": 3.244153, "rms_c/round1": 2.145623, "cos_a2b/round2": 0.503051, "cos_c2b/round2": 0.476074, "cos_b2a/round2": 0.034559, "cos_b2c/round2": 0.108794, "rms_a/round2": 3.374801, "rms_b/round2": 7.114795, "rms_c/round2": 3.500075}, "alpha_mean": 0.42383083671972344, "alpha_hist": [0.0563, 0.15103, 0.16563, 0.13448, 0.11333, 0.10403, 0.11363, 0.1033, 0.0515, 0.00676], "alpha_by_type": {"special": {"share": 0.00116, "mean": 0.59915, "std": 0.13625}, "whitespace": {"share": 0.03645, "mean": 0.65721, "std": 0.18057}, "digit": {"share": 0.01086, "mean": 0.41345, "std": 0.18029}, "word_start": {"share": 0.56279, "mean": 0.44149, "std": 0.21409}, "word_cont": {"share": 0.26186, "mean": 0.25255, "std": 0.15813}, "punct": {"share": 0.12688, "mean": 0.63123, "std": 0.18774}}, "alpha_p05": 0.10485, "alpha_p50": 0.40498, "alpha_p95": 0.81641}
{"step": 22210, "total_steps": 72479, "tokens": 11644436480, "loss": 3.502595543861389, "l_mix": 2.153383821249008, "l_a": 2.2794603407382965, "l_c": 2.21267831325531, "z_loss": 15.701276540756226, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22340409457683563, "tokens_per_second": 1463688.2394006862, "eta_seconds": 18006.17970585823, "data_wait_seconds": 2.60770320892334e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20765, "cosmopedia_v2": 0.20045, "finemath": 0.08235}}
{"step": 22220, "total_steps": 72479, "tokens": 11649679360, "loss": 4.6315547823905945, "l_mix": 2.8666515350341797, "l_a": 2.965796023607254, "l_c": 2.9114387929439545, "z_loss": 17.328349590301514, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19693754613399506, "tokens_per_second": 1457546.4784730482, "eta_seconds": 18078.456489157677, "data_wait_seconds": 1.6301870346069336e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20765, "cosmopedia_v2": 0.20041, "finemath": 0.08236}}
{"step": 22230, "total_steps": 72479, "tokens": 11654922240, "loss": 4.835039854049683, "l_mix": 2.9946154057979584, "l_a": 3.0851419866085052, "l_c": 3.0439235866069794, "z_loss": 17.04708743095398, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19274231791496277, "tokens_per_second": 1463986.710917344, "eta_seconds": 17995.346211504937, "data_wait_seconds": 1.9252300262451172e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20045, "finemath": 0.08232}}
{"step": 22240, "total_steps": 72479, "tokens": 11660165120, "loss": 4.780636966228485, "l_mix": 2.960946947336197, "l_a": 3.053986608982086, "l_c": 3.0060702562332153, "z_loss": 16.72773313522339, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.23257556557655334, "tokens_per_second": 1464652.6931920745, "eta_seconds": 17983.584063601495, "data_wait_seconds": 2.41696834564209e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20045, "finemath": 0.08233}}
{"step": 22250, "total_steps": 72479, "tokens": 11665408000, "loss": 3.471891701221466, "l_mix": 2.1346241235733032, "l_a": 2.2588119506835938, "l_c": 2.193402796983719, "z_loss": 16.030646681785583, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19784976541996002, "tokens_per_second": 1282855.6213199599, "eta_seconds": 20528.00137002468, "data_wait_seconds": 2.16066837310791e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20764, "cosmopedia_v2": 0.20045, "finemath": 0.08234}}
{"step": 22260, "total_steps": 72479, "tokens": 11670650880, "loss": 3.507492631673813, "l_mix": 2.157684236764908, "l_a": 2.28132626414299, "l_c": 2.2128127217292786, "z_loss": 15.666475892066956, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2580975890159607, "tokens_per_second": 1470504.463660735, "eta_seconds": 17904.888915777206, "data_wait_seconds": 2.154707908630371e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20764, "cosmopedia_v2": 0.20045, "finemath": 0.08235}}
{"step": 22270, "total_steps": 72479, "tokens": 11675893760, "loss": 4.874237298965454, "l_mix": 3.0203595757484436, "l_a": 3.1067385971546173, "l_c": 3.0671707093715668, "z_loss": 17.048319578170776, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.172567218542099, "tokens_per_second": 1287024.2380282553, "eta_seconds": 20453.36475739479, "data_wait_seconds": 1.9341707229614258e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.2004, "finemath": 0.08235}}
{"step": 22280, "total_steps": 72479, "tokens": 11681136640, "loss": 4.844928443431854, "l_mix": 3.0028637647628784, "l_a": 3.091119945049286, "l_c": 3.0434463918209076, "z_loss": 16.94837236404419, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18517225980758667, "tokens_per_second": 1345881.0816485211, "eta_seconds": 19555.02136917114, "data_wait_seconds": 2.3186206817626953e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.2004, "finemath": 0.08236}}
{"step": 22290, "total_steps": 72479, "tokens": 11686379520, "loss": 3.4135841727256775, "l_mix": 2.0979965329170227, "l_a": 2.2226964235305786, "l_c": 2.157459646463394, "z_loss": 15.408264756202698, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21647755801677704, "tokens_per_second": 1452651.1134575242, "eta_seconds": 18114.11576271057, "data_wait_seconds": 2.3603439331054688e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20045, "finemath": 0.08232}}
{"step": 22300, "total_steps": 72479, "tokens": 11691622400, "loss": 3.472263753414154, "l_mix": 2.1348152458667755, "l_a": 2.26132008433342, "l_c": 2.1917490661144257, "z_loss": 15.278084635734558, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20234110951423645, "tokens_per_second": 1314147.6119082866, "eta_seconds": 20019.24845702648, "data_wait_seconds": 2.3156404495239258e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20045, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.30879325, "gate_a2b/L1_absmax": 0.64867568, "gate_a2b/L1_rms": 0.32509732, "gate_a2b/L3_absmean": 0.33592525, "gate_a2b/L3_absmax": 0.52742428, "gate_a2b/L3_rms": 0.34050921, "gate_a2b/L5_absmean": 0.51533616, "gate_a2b/L5_absmax": 0.68496108, "gate_a2b/L5_rms": 0.52114326, "gate_c2b/L1_absmean": 0.01417445, "gate_c2b/L1_absmax": 0.43676183, "gate_c2b/L1_rms": 0.05294625, "gate_c2b/L3_absmean": 0.20966302, "gate_c2b/L3_absmax": 0.43100911, "gate_c2b/L3_rms": 0.21371621, "gate_c2b/L5_absmean": 0.52900469, "gate_c2b/L5_absmax": 0.71271187, "gate_c2b/L5_rms": 0.54053688, "gate_b2a/L7_absmean": 0.08671258, "gate_b2a/L7_absmax": 0.26881847, "gate_b2a/L7_rms": 0.08979283, "gate_b2a/L11_absmean": 0.1329245, "gate_b2a/L11_absmax": 0.45248845, "gate_b2a/L11_rms": 0.13575597, "gate_b2a/L14_absmean": 0.32050622, "gate_b2a/L14_absmax": 0.55301577, "gate_b2a/L14_rms": 0.32201955, "gate_b2c/L3_absmean": 0.10038531, "gate_b2c/L3_absmax": 0.62165511, "gate_b2c/L3_rms": 0.11665078, "gate_b2c/L5_absmean": 0.1245323, "gate_b2c/L5_absmax": 0.768287, "gate_b2c/L5_rms": 0.13244779, "gate_b2c/L6_absmean": 0.18331222, "gate_b2c/L6_absmax": 0.52873743, "gate_b2c/L6_rms": 0.18778925}, "streams": {"cos_a2b/round0": 0.375773, "cos_c2b/round0": 0.255887, "cos_b2a/round0": 0.106277, "cos_b2c/round0": 0.105556, "rms_a/round0": 0.85883, "rms_b/round0": 1.572344, "rms_c/round0": 1.266853, "cos_a2b/round1": 0.412916, "cos_c2b/round1": 0.268222, "cos_b2a/round1": 0.219106, "cos_b2c/round1": 0.128691, "rms_a/round1": 1.517906, "rms_b/round1": 3.202717, "rms_c/round1": 2.084325, "cos_a2b/round2": 0.505362, "cos_c2b/round2": 0.470419, "cos_b2a/round2": 0.033891, "cos_b2c/round2": 0.114334, "rms_a/round2": 3.337667, "rms_b/round2": 7.081007, "rms_c/round2": 3.332341}, "alpha_mean": 0.41282933279075706, "alpha_hist": [0.06039, 0.1561, 0.17294, 0.14778, 0.11154, 0.09605, 0.09845, 0.09398, 0.05722, 0.00554], "alpha_by_type": {"special": {"share": 0.00105, "mean": 0.65399, "std": 0.08148}, "whitespace": {"share": 0.03735, "mean": 0.56957, "std": 0.19363}, "digit": {"share": 0.01392, "mean": 0.31781, "std": 0.17042}, "word_start": {"share": 0.56253, "mean": 0.43964, "std": 0.21906}, "word_cont": {"share": 0.27441, "mean": 0.25517, "std": 0.15866}, "punct": {"share": 0.11073, "mean": 0.62409, "std": 0.20459}}, "alpha_p05": 0.10195, "alpha_p50": 0.36614, "alpha_p95": 0.80622}
{"step": 22310, "total_steps": 72479, "tokens": 11696865280, "loss": 3.4519477486610413, "l_mix": 2.121560662984848, "l_a": 2.24563929438591, "l_c": 2.183867782354355, "z_loss": 15.348777651786804, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2319539338350296, "tokens_per_second": 1300023.6328389, "eta_seconds": 20232.712704277037, "data_wait_seconds": 2.2977590560913086e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20766, "cosmopedia_v2": 0.20045, "finemath": 0.08234}}
{"step": 22320, "total_steps": 72479, "tokens": 11702108160, "loss": 4.6038137674331665, "l_mix": 2.849288761615753, "l_a": 2.946256637573242, "l_c": 2.8965477645397186, "z_loss": 16.837323904037476, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1626920998096466, "tokens_per_second": 1179359.4518476757, "eta_seconds": 22298.343181800843, "data_wait_seconds": 2.3365020751953125e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20766, "cosmopedia_v2": 0.2004, "finemath": 0.08235}}
{"step": 22330, "total_steps": 72479, "tokens": 11707351040, "loss": 4.685723721981049, "l_mix": 2.8996839225292206, "l_a": 3.0001744627952576, "l_c": 2.947554498910904, "z_loss": 17.209723711013794, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1660296767950058, "tokens_per_second": 1385609.3330935584, "eta_seconds": 18975.41989941597, "data_wait_seconds": 2.0205974578857422e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.2004, "finemath": 0.08236}}
{"step": 22340, "total_steps": 72479, "tokens": 11712593920, "loss": 4.666316628456116, "l_mix": 2.8892654180526733, "l_a": 2.9873451590538025, "l_c": 2.9305561780929565, "z_loss": 16.806978464126587, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20593349635601044, "tokens_per_second": 1334834.3873205306, "eta_seconds": 19693.286509323123, "data_wait_seconds": 2.9772520065307617e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20045, "finemath": 0.08232}}
{"step": 22350, "total_steps": 72479, "tokens": 11717836800, "loss": 4.739964962005615, "l_mix": 2.9349789321422577, "l_a": 3.0289711356163025, "l_c": 2.9820410311222076, "z_loss": 16.82245659828186, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19281825423240662, "tokens_per_second": 1402987.416409088, "eta_seconds": 18732.907255339625, "data_wait_seconds": 1.722574234008789e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20045, "finemath": 0.08233}}
{"step": 22360, "total_steps": 72479, "tokens": 11723079680, "loss": 3.4909088015556335, "l_mix": 2.1476865112781525, "l_a": 2.2665053009986877, "l_c": 2.205577254295349, "z_loss": 15.975462794303894, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.24284115433692932, "tokens_per_second": 1319978.6930152196, "eta_seconds": 19906.980628585814, "data_wait_seconds": 3.0249357223510742e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20045, "finemath": 0.08233}}
{"step": 22370, "total_steps": 72479, "tokens": 11728322560, "loss": 3.4945132732391357, "l_mix": 2.148815542459488, "l_a": 2.275448441505432, "l_c": 2.2049501538276672, "z_loss": 15.781304955482483, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.25373953580856323, "tokens_per_second": 1451889.2025165285, "eta_seconds": 18094.732949638368, "data_wait_seconds": 2.0205974578857422e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20764, "cosmopedia_v2": 0.20045, "finemath": 0.08234}}
{"step": 22380, "total_steps": 72479, "tokens": 11733565440, "loss": 3.450608015060425, "l_mix": 2.121742844581604, "l_a": 2.2487286031246185, "l_c": 2.1754700541496277, "z_loss": 16.055896282196045, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21819348633289337, "tokens_per_second": 1291443.3565379584, "eta_seconds": 20338.719758033752, "data_wait_seconds": 2.8342008590698242e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20764, "cosmopedia_v2": 0.20045, "finemath": 0.08235}}
{"step": 22390, "total_steps": 72479, "tokens": 11738808320, "loss": 3.478559970855713, "l_mix": 2.138652890920639, "l_a": 2.2673354148864746, "l_c": 2.193936586380005, "z_loss": 15.254103779792786, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2090820074081421, "tokens_per_second": 1453327.4704418855, "eta_seconds": 18069.610714793205, "data_wait_seconds": 3.096461296081543e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20764, "cosmopedia_v2": 0.20045, "finemath": 0.08236}}
{"step": 22400, "total_steps": 72479, "tokens": 11744051200, "loss": 4.987416505813599, "l_mix": 3.0902159214019775, "l_a": 3.176807463169098, "l_c": 3.1414934694767, "z_loss": 17.104151964187622, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2549061179161072, "tokens_per_second": 1439960.946335731, "eta_seconds": 18233.70197560787, "data_wait_seconds": 2.866983413696289e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20768, "cosmopedia_v2": 0.20045, "finemath": 0.08232}, "gates": {"gate_a2b/L1_absmean": 0.30745304, "gate_a2b/L1_absmax": 0.64324778, "gate_a2b/L1_rms": 0.32361916, "gate_a2b/L3_absmean": 0.3356882, "gate_a2b/L3_absmax": 0.52305132, "gate_a2b/L3_rms": 0.34021199, "gate_a2b/L5_absmean": 0.51545304, "gate_a2b/L5_absmax": 0.69185239, "gate_a2b/L5_rms": 0.52128851, "gate_c2b/L1_absmean": 0.01497346, "gate_c2b/L1_absmax": 0.43542746, "gate_c2b/L1_rms": 0.05309781, "gate_c2b/L3_absmean": 0.20987454, "gate_c2b/L3_absmax": 0.43496782, "gate_c2b/L3_rms": 0.21388274, "gate_c2b/L5_absmean": 0.52993518, "gate_c2b/L5_absmax": 0.72000468, "gate_c2b/L5_rms": 0.54156268, "gate_b2a/L7_absmean": 0.08687289, "gate_b2a/L7_absmax": 0.27306184, "gate_b2a/L7_rms": 0.09000432, "gate_b2a/L11_absmean": 0.13321532, "gate_b2a/L11_absmax": 0.45569974, "gate_b2a/L11_rms": 0.13604005, "gate_b2a/L14_absmean": 0.32128927, "gate_b2a/L14_absmax": 0.54260254, "gate_b2a/L14_rms": 0.32274628, "gate_b2c/L3_absmean": 0.10067041, "gate_b2c/L3_absmax": 0.61778444, "gate_b2c/L3_rms": 0.11695275, "gate_b2c/L5_absmean": 0.12502983, "gate_b2c/L5_absmax": 0.76346445, "gate_b2c/L5_rms": 0.13287015, "gate_b2c/L6_absmean": 0.18367796, "gate_b2c/L6_absmax": 0.52950084, "gate_b2c/L6_rms": 0.18815427}, "streams": {"cos_a2b/round0": 0.385975, "cos_c2b/round0": 0.293212, "cos_b2a/round0": 0.121512, "cos_b2c/round0": 0.153644, "rms_a/round0": 0.826265, "rms_b/round0": 1.57649, "rms_c/round0": 1.239697, "cos_a2b/round1": 0.412557, "cos_c2b/round1": 0.275783, "cos_b2a/round1": 0.240474, "cos_b2c/round1": 0.128936, "rms_a/round1": 1.447996, "rms_b/round1": 3.063695, "rms_c/round1": 2.031971, "cos_a2b/round2": 0.547759, "cos_c2b/round2": 0.497228, "cos_b2a/round2": 0.079816, "cos_b2c/round2": 0.146843, "rms_a/round2": 3.324131, "rms_b/round2": 6.808758, "rms_c/round2": 3.244192}, "alpha_mean": 0.451773506305976, "alpha_hist": [0.0098, 0.05197, 0.14375, 0.1873, 0.19934, 0.20287, 0.13779, 0.06004, 0.00693, 0.00021], "alpha_by_type": {"special": {"share": 0.00046, "mean": 0.52333, "std": 0.16628}, "whitespace": {"share": 0.04941, "mean": 0.55086, "std": 0.11056}, "digit": {"share": 0.03223, "mean": 0.5063, "std": 0.12321}, "word_start": {"share": 0.55116, "mean": 0.47473, "std": 0.15105}, "word_cont": {"share": 0.26479, "mean": 0.33238, "std": 0.13536}, "punct": {"share": 0.10196, "mean": 0.57217, "std": 0.15567}}, "alpha_p05": 0.19163, "alpha_p50": 0.4552, "alpha_p95": 0.71582}
{"step": 22410, "total_steps": 72479, "tokens": 11749294080, "loss": 3.510391414165497, "l_mix": 2.15855672955513, "l_a": 2.2833397686481476, "l_c": 2.217695325613022, "z_loss": 15.240290760993958, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21224980056285858, "tokens_per_second": 1445078.9114169425, "eta_seconds": 18165.49647538662, "data_wait_seconds": 2.8073787689208984e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20768, "cosmopedia_v2": 0.20045, "finemath": 0.08233}}
{"step": 22420, "total_steps": 72479, "tokens": 11754536960, "loss": 3.4244153797626495, "l_mix": 2.1052666008472443, "l_a": 2.2279689610004425, "l_c": 2.164135843515396, "z_loss": 15.173149347305298, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.24281595647335052, "tokens_per_second": 1386774.9955521508, "eta_seconds": 18925.444341135026, "data_wait_seconds": 2.9414892196655273e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20045, "finemath": 0.08234}}
{"step": 22430, "total_steps": 72479, "tokens": 11759779840, "loss": 4.6507678627967834, "l_mix": 2.87841135263443, "l_a": 2.976078122854233, "l_c": 2.926174581050873, "z_loss": 16.806580543518066, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22398380935192108, "tokens_per_second": 1364972.588829664, "eta_seconds": 19223.895283126833, "data_wait_seconds": 1.9341707229614258e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20767, "cosmopedia_v2": 0.2004, "finemath": 0.08235}}
{"step": 22440, "total_steps": 72479, "tokens": 11765022720, "loss": 4.589433312416077, "l_mix": 2.840583473443985, "l_a": 2.938133656978607, "l_c": 2.8856565952301025, "z_loss": 17.126280069351196, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16671520471572876, "tokens_per_second": 1441612.3700387736, "eta_seconds": 18198.267285466194, "data_wait_seconds": 3.191828727722168e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.2004, "finemath": 0.08235}}
{"step": 22450, "total_steps": 72479, "tokens": 11770265600, "loss": 4.6471027135849, "l_mix": 2.8767842054367065, "l_a": 2.9715071320533752, "l_c": 2.923804998397827, "z_loss": 17.246528387069702, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2001129537820816, "tokens_per_second": 1428001.53172617, "eta_seconds": 18368.050572252272, "data_wait_seconds": 1.9341707229614258e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20045, "finemath": 0.08232}}
{"step": 22460, "total_steps": 72479, "tokens": 11775508480, "loss": 4.564088821411133, "l_mix": 2.824193686246872, "l_a": 2.9250389337539673, "l_c": 2.868896335363388, "z_loss": 17.145201921463013, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21489395201206207, "tokens_per_second": 1410885.7994992724, "eta_seconds": 18587.160974550246, "data_wait_seconds": 3.0517578125e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20045, "finemath": 0.08232}}
{"step": 22470, "total_steps": 72479, "tokens": 11780751360, "loss": 3.5000364184379578, "l_mix": 2.1527035534381866, "l_a": 2.274049609899521, "l_c": 2.211777687072754, "z_loss": 15.846363067626953, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.30831122398376465, "tokens_per_second": 1360083.244101312, "eta_seconds": 19277.583710932733, "data_wait_seconds": 2.4020671844482422e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20045, "finemath": 0.08233}}
{"step": 22480, "total_steps": 72479, "tokens": 11785994240, "loss": 3.5362775027751923, "l_mix": 2.1738120019435883, "l_a": 2.3023435473442078, "l_c": 2.234144389629364, "z_loss": 15.190773606300354, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.269093781709671, "tokens_per_second": 1381848.4937929015, "eta_seconds": 18970.151814579964, "data_wait_seconds": 2.518296241760254e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 22490, "total_steps": 72479, "tokens": 11791237120, "loss": 3.517986625432968, "l_mix": 2.162977457046509, "l_a": 2.2889949679374695, "l_c": 2.22264164686203, "z_loss": 15.182183027267456, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.25885769724845886, "tokens_per_second": 1466074.2444093926, "eta_seconds": 17876.743235850336, "data_wait_seconds": 4.628300666809082e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 22500, "total_steps": 72479, "tokens": 11796480000, "loss": 3.4791769087314606, "l_mix": 2.1381512880325317, "l_a": 2.2640947699546814, "l_c": 2.200933128595352, "z_loss": 15.171120285987854, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.26667195558547974, "tokens_per_second": 1442591.6620212076, "eta_seconds": 18164.107447624207, "data_wait_seconds": 1.9818544387817383e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08236}, "gates": {"gate_a2b/L1_absmean": 0.30837235, "gate_a2b/L1_absmax": 0.64739347, "gate_a2b/L1_rms": 0.32454756, "gate_a2b/L3_absmean": 0.33518919, "gate_a2b/L3_absmax": 0.52965617, "gate_a2b/L3_rms": 0.33970395, "gate_a2b/L5_absmean": 0.51421261, "gate_a2b/L5_absmax": 0.68735927, "gate_a2b/L5_rms": 0.52002704, "gate_c2b/L1_absmean": 0.01435984, "gate_c2b/L1_absmax": 0.43898374, "gate_c2b/L1_rms": 0.05294141, "gate_c2b/L3_absmean": 0.20938507, "gate_c2b/L3_absmax": 0.42861265, "gate_c2b/L3_rms": 0.21337359, "gate_c2b/L5_absmean": 0.52973694, "gate_c2b/L5_absmax": 0.7074275, "gate_c2b/L5_rms": 0.54134184, "gate_b2a/L7_absmean": 0.08623527, "gate_b2a/L7_absmax": 0.27262616, "gate_b2a/L7_rms": 0.08937744, "gate_b2a/L11_absmean": 0.13305482, "gate_b2a/L11_absmax": 0.4537496, "gate_b2a/L11_rms": 0.13590679, "gate_b2a/L14_absmean": 0.32127273, "gate_b2a/L14_absmax": 0.552912, "gate_b2a/L14_rms": 0.32275242, "gate_b2c/L3_absmean": 0.10006499, "gate_b2c/L3_absmax": 0.62092841, "gate_b2c/L3_rms": 0.11647917, "gate_b2c/L5_absmean": 0.12435883, "gate_b2c/L5_absmax": 0.7588678, "gate_b2c/L5_rms": 0.13218732, "gate_b2c/L6_absmean": 0.18303291, "gate_b2c/L6_absmax": 0.5236876, "gate_b2c/L6_rms": 0.18750328}, "streams": {"cos_a2b/round0": 0.380084, "cos_c2b/round0": 0.255942, "cos_b2a/round0": 0.103916, "cos_b2c/round0": 0.108611, "rms_a/round0": 0.874415, "rms_b/round0": 1.575002, "rms_c/round0": 1.262774, "cos_a2b/round1": 0.410722, "cos_c2b/round1": 0.273282, "cos_b2a/round1": 0.214386, "cos_b2c/round1": 0.132137, "rms_a/round1": 1.550179, "rms_b/round1": 3.210692, "rms_c/round1": 2.172281, "cos_a2b/round2": 0.504397, "cos_c2b/round2": 0.475428, "cos_b2a/round2": 0.031123, "cos_b2c/round2": 0.107951, "rms_a/round2": 3.377508, "rms_b/round2": 7.068702, "rms_c/round2": 3.472178}, "alpha_mean": 0.4269039959061729, "alpha_hist": [0.06155, 0.145, 0.15604, 0.13432, 0.12009, 0.11037, 0.10922, 0.09782, 0.05658, 0.009], "alpha_by_type": {"special": {"share": 0.00108, "mean": 0.65779, "std": 0.08462}, "whitespace": {"share": 0.03677, "mean": 0.63835, "std": 0.19664}, "digit": {"share": 0.01321, "mean": 0.42253, "std": 0.17631}, "word_start": {"share": 0.57143, "mean": 0.43959, "std": 0.21919}, "word_cont": {"share": 0.25064, "mean": 0.26356, "std": 0.16556}, "punct": {"share": 0.12686, "mean": 0.62966, "std": 0.19276}}, "alpha_p05": 0.08851, "alpha_p50": 0.40568, "alpha_p95": 0.82104}
{"step": 22510, "total_steps": 72479, "tokens": 11801722880, "loss": 3.2203240990638733, "l_mix": 1.9895042181015015, "l_a": 2.0786443054676056, "l_c": 2.0176009982824326, "z_loss": 19.462626695632935, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2935602366924286, "tokens_per_second": 1334152.7388834674, "eta_seconds": 19636.542585015297, "data_wait_seconds": 2.8699636459350586e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08236}}
{"step": 22520, "total_steps": 72479, "tokens": 11806965760, "loss": 4.5388782024383545, "l_mix": 2.8085418343544006, "l_a": 2.9058394730091095, "l_c": 2.8563190698623657, "z_loss": 16.885319709777832, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19651289284229279, "tokens_per_second": 1476113.29956097, "eta_seconds": 17744.507958698272, "data_wait_seconds": 2.7924776077270508e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 22530, "total_steps": 72479, "tokens": 11812208640, "loss": 4.620753109455109, "l_mix": 2.8597512543201447, "l_a": 2.9605840146541595, "l_c": 2.903641700744629, "z_loss": 17.341360330581665, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20801596343517303, "tokens_per_second": 1400523.8346714065, "eta_seconds": 18698.476001405714, "data_wait_seconds": 1.6123056411743164e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20763, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 22540, "total_steps": 72479, "tokens": 11817451520, "loss": 4.683700442314148, "l_mix": 2.8996115028858185, "l_a": 2.9956041872501373, "l_c": 2.945679098367691, "z_loss": 17.040102005004883, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17954087257385254, "tokens_per_second": 1346602.1469301928, "eta_seconds": 19443.32146780491, "data_wait_seconds": 2.8073787689208984e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20763, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 22550, "total_steps": 72479, "tokens": 11822694400, "loss": 4.663843095302582, "l_mix": 2.8870214223861694, "l_a": 2.9809583127498627, "l_c": 2.936067372560501, "z_loss": 17.138758182525635, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16400016844272614, "tokens_per_second": 1471998.490633128, "eta_seconds": 17783.42553920746, "data_wait_seconds": 1.946091651916504e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.2004, "finemath": 0.08235}}
{"step": 22560, "total_steps": 72479, "tokens": 11827937280, "loss": 4.758906245231628, "l_mix": 2.9465225636959076, "l_a": 3.043151617050171, "l_c": 2.9923688769340515, "z_loss": 17.274956226348877, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.23431497812271118, "tokens_per_second": 1355773.1592626965, "eta_seconds": 19304.06461670399, "data_wait_seconds": 3.11732292175293e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.2004, "finemath": 0.08236}}
{"step": 22570, "total_steps": 72479, "tokens": 11833180160, "loss": 4.797584712505341, "l_mix": 2.9701439440250397, "l_a": 3.070071220397949, "l_c": 3.015611320734024, "z_loss": 17.358824014663696, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2614608108997345, "tokens_per_second": 1380327.1846456514, "eta_seconds": 18956.874922895433, "data_wait_seconds": 2.0325183868408203e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08232}}
{"step": 22580, "total_steps": 72479, "tokens": 11838423040, "loss": 3.4864718914031982, "l_mix": 2.143134206533432, "l_a": 2.2694360315799713, "l_c": 2.20307919383049, "z_loss": 15.830599546432495, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21162049472332, "tokens_per_second": 1437855.1113456292, "eta_seconds": 18194.772689938545, "data_wait_seconds": 1.7017126083374023e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 22590, "total_steps": 72479, "tokens": 11843665920, "loss": 3.5528085231781006, "l_mix": 2.18573060631752, "l_a": 2.3082691431045532, "l_c": 2.243514597415924, "z_loss": 15.427345871925354, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2320837378501892, "tokens_per_second": 1417629.2651491167, "eta_seconds": 18450.665963959695, "data_wait_seconds": 2.3156404495239258e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 22600, "total_steps": 72479, "tokens": 11848908800, "loss": 3.428318530321121, "l_mix": 2.107435405254364, "l_a": 2.2334459125995636, "l_c": 2.164325028657913, "z_loss": 15.517634987831116, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19328723847866058, "tokens_per_second": 1459944.6644678987, "eta_seconds": 17912.296122217176, "data_wait_seconds": 2.580881118774414e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.30853319, "gate_a2b/L1_absmax": 0.65067339, "gate_a2b/L1_rms": 0.32479405, "gate_a2b/L3_absmean": 0.33484125, "gate_a2b/L3_absmax": 0.52450138, "gate_a2b/L3_rms": 0.33944273, "gate_a2b/L5_absmean": 0.51426041, "gate_a2b/L5_absmax": 0.68390161, "gate_a2b/L5_rms": 0.52008909, "gate_c2b/L1_absmean": 0.01414107, "gate_c2b/L1_absmax": 0.43442494, "gate_c2b/L1_rms": 0.05266877, "gate_c2b/L3_absmean": 0.20965803, "gate_c2b/L3_absmax": 0.43393278, "gate_c2b/L3_rms": 0.21373506, "gate_c2b/L5_absmean": 0.5301919, "gate_c2b/L5_absmax": 0.71082032, "gate_c2b/L5_rms": 0.54173625, "gate_b2a/L7_absmean": 0.08673261, "gate_b2a/L7_absmax": 0.27087256, "gate_b2a/L7_rms": 0.08983847, "gate_b2a/L11_absmean": 0.13316759, "gate_b2a/L11_absmax": 0.45543799, "gate_b2a/L11_rms": 0.13601215, "gate_b2a/L14_absmean": 0.32163393, "gate_b2a/L14_absmax": 0.55522931, "gate_b2a/L14_rms": 0.32311693, "gate_b2c/L3_absmean": 0.100272, "gate_b2c/L3_absmax": 0.61997133, "gate_b2c/L3_rms": 0.1166537, "gate_b2c/L5_absmean": 0.12478435, "gate_b2c/L5_absmax": 0.76346397, "gate_b2c/L5_rms": 0.13267443, "gate_b2c/L6_absmean": 0.18352184, "gate_b2c/L6_absmax": 0.52546293, "gate_b2c/L6_rms": 0.18802933}, "streams": {"cos_a2b/round0": 0.375405, "cos_c2b/round0": 0.265769, "cos_b2a/round0": 0.104563, "cos_b2c/round0": 0.111263, "rms_a/round0": 0.856282, "rms_b/round0": 1.547766, "rms_c/round0": 1.24975, "cos_a2b/round1": 0.411411, "cos_c2b/round1": 0.2661, "cos_b2a/round1": 0.213206, "cos_b2c/round1": 0.128291, "rms_a/round1": 1.53101, "rms_b/round1": 3.203283, "rms_c/round1": 2.087447, "cos_a2b/round2": 0.499718, "cos_c2b/round2": 0.480002, "cos_b2a/round2": 0.030506, "cos_b2c/round2": 0.108574, "rms_a/round2": 3.356718, "rms_b/round2": 7.156731, "rms_c/round2": 3.395472}, "alpha_mean": 0.3958605156524655, "alpha_hist": [0.06969, 0.16347, 0.17992, 0.14375, 0.11522, 0.10146, 0.09918, 0.08304, 0.03993, 0.00435], "alpha_by_type": {"special": {"share": 0.00105, "mean": 0.59168, "std": 0.11455}, "whitespace": {"share": 0.04222, "mean": 0.57553, "std": 0.18735}, "digit": {"share": 0.01477, "mean": 0.31258, "std": 0.16317}, "word_start": {"share": 0.54811, "mean": 0.41619, "std": 0.21331}, "word_cont": {"share": 0.26488, "mean": 0.23942, "std": 0.15032}, "punct": {"share": 0.12897, "mean": 0.57988, "std": 0.19845}}, "alpha_p05": 0.08756, "alpha_p50": 0.3372, "alpha_p95": 0.79406}
{"step": 22610, "total_steps": 72479, "tokens": 11854151680, "loss": 3.475173532962799, "l_mix": 2.1376135647296906, "l_a": 2.2579973936080933, "l_c": 2.1953265964984894, "z_loss": 15.627297043800354, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2074132263660431, "tokens_per_second": 1421902.3115929852, "eta_seconds": 18387.84426949024, "data_wait_seconds": 1.7464160919189453e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 22620, "total_steps": 72479, "tokens": 11859394560, "loss": 3.2086341381073, "l_mix": 1.982549011707306, "l_a": 2.063022255897522, "l_c": 2.017318546772003, "z_loss": 19.82810878753662, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.25737592577934265, "tokens_per_second": 1349734.1932863987, "eta_seconds": 19367.128373885153, "data_wait_seconds": 3.260374069213867e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08236}}
{"step": 22630, "total_steps": 72479, "tokens": 11864637440, "loss": 4.652939081192017, "l_mix": 2.880348414182663, "l_a": 2.976340502500534, "l_c": 2.9266715049743652, "z_loss": 16.87014865875244, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21351057291030884, "tokens_per_second": 1428903.7302388246, "eta_seconds": 18290.40820519924, "data_wait_seconds": 1.7970800399780273e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08232}}
{"step": 22640, "total_steps": 72479, "tokens": 11869880320, "loss": 4.647943139076233, "l_mix": 2.877830743789673, "l_a": 2.973144054412842, "l_c": 2.921564429998398, "z_loss": 16.998449325561523, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1869906187057495, "tokens_per_second": 1412962.2562847403, "eta_seconds": 18493.055646586417, "data_wait_seconds": 4.634261131286621e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 22650, "total_steps": 72479, "tokens": 11875123200, "loss": 4.630416572093964, "l_mix": 2.866128593683243, "l_a": 2.963504135608673, "l_c": 2.911901891231537, "z_loss": 16.66133999824524, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2147662490606308, "tokens_per_second": 1348356.479089107, "eta_seconds": 19375.252136325835, "data_wait_seconds": 1.800060272216797e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 22660, "total_steps": 72479, "tokens": 11880366080, "loss": 4.581849098205566, "l_mix": 2.8366549611091614, "l_a": 2.9299950897693634, "l_c": 2.881584346294403, "z_loss": 17.202688932418823, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22415722906589508, "tokens_per_second": 1434406.2580102328, "eta_seconds": 18209.279084038735, "data_wait_seconds": 2.3812055587768555e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20763, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 22670, "total_steps": 72479, "tokens": 11885608960, "loss": 4.75531941652298, "l_mix": 2.9452209770679474, "l_a": 3.0344669818878174, "l_c": 2.9934107065200806, "z_loss": 17.349741220474243, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.24248914420604706, "tokens_per_second": 1432904.425863293, "eta_seconds": 18224.70537507534, "data_wait_seconds": 2.0712614059448242e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20763, "cosmopedia_v2": 0.20044, "finemath": 0.08236}}
{"step": 22680, "total_steps": 72479, "tokens": 11890851840, "loss": 5.01225072145462, "l_mix": 3.1054624021053314, "l_a": 3.1967513859272003, "l_c": 3.1537070274353027, "z_loss": 16.506080150604248, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19637633860111237, "tokens_per_second": 1436903.1748245726, "eta_seconds": 18170.339219403268, "data_wait_seconds": 1.7881393432617188e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08232}}
{"step": 22690, "total_steps": 72479, "tokens": 11896094720, "loss": 3.488725483417511, "l_mix": 2.145298182964325, "l_a": 2.272052675485611, "l_c": 2.200764536857605, "z_loss": 15.820721745491028, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2122013121843338, "tokens_per_second": 1309080.4129125755, "eta_seconds": 19940.543739342687, "data_wait_seconds": 1.761317253112793e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 22700, "total_steps": 72479, "tokens": 11901337600, "loss": 3.4396913945674896, "l_mix": 2.1144768595695496, "l_a": 2.238167464733124, "l_c": 2.174022912979126, "z_loss": 15.574256777763367, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18092121183872223, "tokens_per_second": 1364029.305882663, "eta_seconds": 19133.410286307335, "data_wait_seconds": 1.6033649444580078e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.31009635, "gate_a2b/L1_absmax": 0.64595991, "gate_a2b/L1_rms": 0.32620448, "gate_a2b/L3_absmean": 0.33555609, "gate_a2b/L3_absmax": 0.52752721, "gate_a2b/L3_rms": 0.34012753, "gate_a2b/L5_absmean": 0.51460582, "gate_a2b/L5_absmax": 0.6861434, "gate_a2b/L5_rms": 0.52040166, "gate_c2b/L1_absmean": 0.01367465, "gate_c2b/L1_absmax": 0.43496948, "gate_c2b/L1_rms": 0.05262389, "gate_c2b/L3_absmean": 0.21003857, "gate_c2b/L3_absmax": 0.42736369, "gate_c2b/L3_rms": 0.21405596, "gate_c2b/L5_absmean": 0.5304696, "gate_c2b/L5_absmax": 0.71230668, "gate_c2b/L5_rms": 0.54214048, "gate_b2a/L7_absmean": 0.08672225, "gate_b2a/L7_absmax": 0.27277726, "gate_b2a/L7_rms": 0.08982987, "gate_b2a/L11_absmean": 0.13314202, "gate_b2a/L11_absmax": 0.45705929, "gate_b2a/L11_rms": 0.13602239, "gate_b2a/L14_absmean": 0.32164937, "gate_b2a/L14_absmax": 0.5538466, "gate_b2a/L14_rms": 0.32310498, "gate_b2c/L3_absmean": 0.10105407, "gate_b2c/L3_absmax": 0.62088418, "gate_b2c/L3_rms": 0.11746939, "gate_b2c/L5_absmean": 0.12547813, "gate_b2c/L5_absmax": 0.76734769, "gate_b2c/L5_rms": 0.13335328, "gate_b2c/L6_absmean": 0.18413031, "gate_b2c/L6_absmax": 0.52570748, "gate_b2c/L6_rms": 0.188576}, "streams": {"cos_a2b/round0": 0.376379, "cos_c2b/round0": 0.261578, "cos_b2a/round0": 0.108398, "cos_b2c/round0": 0.113699, "rms_a/round0": 0.853151, "rms_b/round0": 1.563019, "rms_c/round0": 1.24357, "cos_a2b/round1": 0.409213, "cos_c2b/round1": 0.268635, "cos_b2a/round1": 0.214147, "cos_b2c/round1": 0.128109, "rms_a/round1": 1.524371, "rms_b/round1": 3.162922, "rms_c/round1": 2.100309, "cos_a2b/round2": 0.50694, "cos_c2b/round2": 0.478992, "cos_b2a/round2": 0.034425, "cos_b2c/round2": 0.107865, "rms_a/round2": 3.334689, "rms_b/round2": 6.999798, "rms_c/round2": 3.392256}, "alpha_mean": 0.42172120112823563, "alpha_hist": [0.05394, 0.1403, 0.17149, 0.15007, 0.11841, 0.10596, 0.10516, 0.09457, 0.0538, 0.00629], "alpha_by_type": {"special": {"share": 0.00111, "mean": 0.58085, "std": 0.10627}, "whitespace": {"share": 0.04164, "mean": 0.55758, "std": 0.205}, "digit": {"share": 0.01276, "mean": 0.34259, "std": 0.15465}, "word_start": {"share": 0.55949, "mean": 0.44934, "std": 0.21348}, "word_cont": {"share": 0.25797, "mean": 0.25104, "std": 0.15227}, "punct": {"share": 0.12703, "mean": 0.60869, "std": 0.19932}}, "alpha_p05": 0.09269, "alpha_p50": 0.41087, "alpha_p95": 0.81316}
{"step": 22710, "total_steps": 72479, "tokens": 11906580480, "loss": 3.4894942939281464, "l_mix": 2.1454667448997498, "l_a": 2.271618753671646, "l_c": 2.2032987475395203, "z_loss": 15.521352648735046, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20741428434848785, "tokens_per_second": 1415759.5325042782, "eta_seconds": 18430.594230818748, "data_wait_seconds": 2.7120113372802734e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 22720, "total_steps": 72479, "tokens": 11911823360, "loss": 3.5453109741210938, "l_mix": 2.1790578961372375, "l_a": 2.3090616166591644, "l_c": 2.2399056255817413, "z_loss": 15.628987669944763, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.25607162714004517, "tokens_per_second": 1384466.7218718245, "eta_seconds": 18843.390151500702, "data_wait_seconds": 3.838539123535156e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 22730, "total_steps": 72479, "tokens": 11917066240, "loss": 3.203948736190796, "l_mix": 1.9811995327472687, "l_a": 2.056083559989929, "l_c": 2.013509824872017, "z_loss": 18.711445331573486, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.25615766644477844, "tokens_per_second": 1424217.3820765011, "eta_seconds": 18313.779932928086, "data_wait_seconds": 3.534555435180664e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08236}}
{"step": 22740, "total_steps": 72479, "tokens": 11922309120, "loss": 4.7556681632995605, "l_mix": 2.9443697929382324, "l_a": 3.041881501674652, "l_c": 2.990057349205017, "z_loss": 17.167075634002686, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1803838014602661, "tokens_per_second": 1448167.3241535819, "eta_seconds": 18007.284377336502, "data_wait_seconds": 1.5288591384887695e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08232}}
{"step": 22750, "total_steps": 72479, "tokens": 11927552000, "loss": 4.565981507301331, "l_mix": 2.8258753418922424, "l_a": 2.9243538975715637, "l_c": 2.8702798187732697, "z_loss": 17.16096067428589, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17318838834762573, "tokens_per_second": 1383605.5583937662, "eta_seconds": 18843.750513887404, "data_wait_seconds": 2.2411346435546875e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 22760, "total_steps": 72479, "tokens": 11932794880, "loss": 4.61665403842926, "l_mix": 2.858566850423813, "l_a": 2.9507243037223816, "l_c": 2.903842866420746, "z_loss": 17.16969633102417, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18862120807170868, "tokens_per_second": 1327773.001372259, "eta_seconds": 19632.17737147808, "data_wait_seconds": 2.148747444152832e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 22770, "total_steps": 72479, "tokens": 11938037760, "loss": 4.622864067554474, "l_mix": 2.8615534603595734, "l_a": 2.9578553438186646, "l_c": 2.9076462984085083, "z_loss": 16.59898281097412, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20513004064559937, "tokens_per_second": 1417315.1382363928, "eta_seconds": 18388.17034327984, "data_wait_seconds": 2.0056962966918945e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 22780, "total_steps": 72479, "tokens": 11943280640, "loss": 4.615485429763794, "l_mix": 2.8569299280643463, "l_a": 2.9548495411872864, "l_c": 2.901322662830353, "z_loss": 17.03782320022583, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21961209177970886, "tokens_per_second": 1436661.0700535707, "eta_seconds": 18136.907761430743, "data_wait_seconds": 1.868605613708496e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 22790, "total_steps": 72479, "tokens": 11948523520, "loss": 4.960493326187134, "l_mix": 3.0744116604328156, "l_a": 3.158410429954529, "l_c": 3.1230831146240234, "z_loss": 16.334688663482666, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22819960117340088, "tokens_per_second": 1413700.1171655066, "eta_seconds": 18427.774119615555, "data_wait_seconds": 2.4050474166870117e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20768, "cosmopedia_v2": 0.20044, "finemath": 0.08232}}
{"step": 22800, "total_steps": 72479, "tokens": 11953766400, "loss": 4.9608882665634155, "l_mix": 3.075215846300125, "l_a": 3.155279904603958, "l_c": 3.1247405409812927, "z_loss": 16.663292407989502, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22467492520809174, "tokens_per_second": 1398873.9378241145, "eta_seconds": 18619.335772681236, "data_wait_seconds": 1.6927719116210938e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.20044, "finemath": 0.08232}, "gates": {"gate_a2b/L1_absmean": 0.3110542, "gate_a2b/L1_absmax": 0.65517867, "gate_a2b/L1_rms": 0.32724726, "gate_a2b/L3_absmean": 0.336481, "gate_a2b/L3_absmax": 0.52546549, "gate_a2b/L3_rms": 0.34102845, "gate_a2b/L5_absmean": 0.51549262, "gate_a2b/L5_absmax": 0.69225228, "gate_a2b/L5_rms": 0.52124989, "gate_c2b/L1_absmean": 0.01422105, "gate_c2b/L1_absmax": 0.43327099, "gate_c2b/L1_rms": 0.05265948, "gate_c2b/L3_absmean": 0.20955896, "gate_c2b/L3_absmax": 0.43304417, "gate_c2b/L3_rms": 0.21354567, "gate_c2b/L5_absmean": 0.53019202, "gate_c2b/L5_absmax": 0.71779764, "gate_c2b/L5_rms": 0.5418492, "gate_b2a/L7_absmean": 0.08695385, "gate_b2a/L7_absmax": 0.27305332, "gate_b2a/L7_rms": 0.09008376, "gate_b2a/L11_absmean": 0.13360071, "gate_b2a/L11_absmax": 0.45577806, "gate_b2a/L11_rms": 0.13641346, "gate_b2a/L14_absmean": 0.3216033, "gate_b2a/L14_absmax": 0.5541988, "gate_b2a/L14_rms": 0.32306567, "gate_b2c/L3_absmean": 0.10220959, "gate_b2c/L3_absmax": 0.62142009, "gate_b2c/L3_rms": 0.11829067, "gate_b2c/L5_absmean": 0.12550065, "gate_b2c/L5_absmax": 0.77488029, "gate_b2c/L5_rms": 0.13350148, "gate_b2c/L6_absmean": 0.18423155, "gate_b2c/L6_absmax": 0.52561998, "gate_b2c/L6_rms": 0.1887157}, "streams": {"cos_a2b/round0": 0.384761, "cos_c2b/round0": 0.301548, "cos_b2a/round0": 0.120225, "cos_b2c/round0": 0.154123, "rms_a/round0": 0.823438, "rms_b/round0": 1.558137, "rms_c/round0": 1.21692, "cos_a2b/round1": 0.414147, "cos_c2b/round1": 0.272323, "cos_b2a/round1": 0.22798, "cos_b2c/round1": 0.137882, "rms_a/round1": 1.457838, "rms_b/round1": 3.022009, "rms_c/round1": 2.030419, "cos_a2b/round2": 0.55031, "cos_c2b/round2": 0.49286, "cos_b2a/round2": 0.072617, "cos_b2c/round2": 0.143732, "rms_a/round2": 3.297181, "rms_b/round2": 6.72821, "rms_c/round2": 3.273295}, "alpha_mean": 0.4479037228538161, "alpha_hist": [0.01147, 0.05696, 0.13535, 0.18625, 0.21466, 0.203, 0.13293, 0.05334, 0.00554, 0.00049], "alpha_by_type": {"special": {"share": 0.00064, "mean": 0.52312, "std": 0.12981}, "whitespace": {"share": 0.06372, "mean": 0.52572, "std": 0.12557}, "digit": {"share": 0.03357, "mean": 0.43992, "std": 0.1322}, "word_start": {"share": 0.53815, "mean": 0.46904, "std": 0.14906}, "word_cont": {"share": 0.24527, "mean": 0.33051, "std": 0.13786}, "punct": {"share": 0.11865, "mean": 0.55476, "std": 0.15724}}, "alpha_p05": 0.1833, "alpha_p50": 0.46332, "alpha_p95": 0.71859}
{"step": 22810, "total_steps": 72479, "tokens": 11959009280, "loss": 5.015648126602173, "l_mix": 3.107966899871826, "l_a": 3.1979164481163025, "l_c": 3.1554411351680756, "z_loss": 16.738293886184692, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19367776811122894, "tokens_per_second": 1433148.6284880084, "eta_seconds": 18170.383834838867, "data_wait_seconds": 2.3752450942993164e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 22820, "total_steps": 72479, "tokens": 11964252160, "loss": 4.992372810840607, "l_mix": 3.0931414663791656, "l_a": 3.1837123930454254, "l_c": 3.1413665413856506, "z_loss": 17.0771427154541, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2054212987422943, "tokens_per_second": 1412463.6397563394, "eta_seconds": 18432.7702739954, "data_wait_seconds": 2.1576881408691406e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 22830, "total_steps": 72479, "tokens": 11969495040, "loss": 3.463632345199585, "l_mix": 2.129314035177231, "l_a": 2.2511351704597473, "l_c": 2.191457688808441, "z_loss": 15.403766632080078, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22173012793064117, "tokens_per_second": 1434400.456989021, "eta_seconds": 18147.21599199772, "data_wait_seconds": 2.3066997528076172e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 22840, "total_steps": 72479, "tokens": 11974737920, "loss": 3.428477644920349, "l_mix": 2.10785710811615, "l_a": 2.2348587810993195, "l_c": 2.162057876586914, "z_loss": 15.454253315925598, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20416364073753357, "tokens_per_second": 1442381.4111680202, "eta_seconds": 18043.169324350354, "data_wait_seconds": 2.130866050720215e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08236}}
{"step": 22850, "total_steps": 72479, "tokens": 11979980800, "loss": 4.672834515571594, "l_mix": 2.893010199069977, "l_a": 2.9918269515037537, "l_c": 2.9351742267608643, "z_loss": 17.238282442092896, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21872904896736145, "tokens_per_second": 1341202.1579232677, "eta_seconds": 19400.42297000885, "data_wait_seconds": 2.2709369659423828e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08232}}
{"step": 22860, "total_steps": 72479, "tokens": 11985223680, "loss": 4.609625697135925, "l_mix": 2.8535705506801605, "l_a": 2.9506695568561554, "l_c": 2.897165447473526, "z_loss": 17.045610904693604, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16794754564762115, "tokens_per_second": 1470379.590325419, "eta_seconds": 17692.46964740753, "data_wait_seconds": 2.3871660232543945e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 22870, "total_steps": 72479, "tokens": 11990466560, "loss": 4.550018489360809, "l_mix": 2.8161239624023438, "l_a": 2.908397078514099, "l_c": 2.8655276894569397, "z_loss": 17.169836044311523, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1724659949541092, "tokens_per_second": 1434306.5244028792, "eta_seconds": 18133.783085751533, "data_wait_seconds": 1.710653305053711e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 22880, "total_steps": 72479, "tokens": 11995709440, "loss": 4.653653979301453, "l_mix": 2.880603700876236, "l_a": 2.978529244661331, "l_c": 2.9258925318717957, "z_loss": 17.237221479415894, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20387281477451324, "tokens_per_second": 1399696.5736579509, "eta_seconds": 18578.426925802232, "data_wait_seconds": 1.6748905181884766e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 22890, "total_steps": 72479, "tokens": 12000952320, "loss": 4.661112427711487, "l_mix": 2.8867017030715942, "l_a": 2.9746580123901367, "l_c": 2.934353858232498, "z_loss": 17.071190118789673, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.27505913376808167, "tokens_per_second": 1399338.605086068, "eta_seconds": 18579.432838845252, "data_wait_seconds": 2.4050474166870117e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 22900, "total_steps": 72479, "tokens": 12006195200, "loss": 3.2834051847457886, "l_mix": 2.0304860919713974, "l_a": 2.0986736863851547, "l_c": 2.0716559290885925, "z_loss": 18.201570749282837, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.4293844401836395, "tokens_per_second": 1410910.2408492628, "eta_seconds": 18423.3369348526, "data_wait_seconds": 2.136826515197754e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08236}, "gates": {"gate_a2b/L1_absmean": 0.31100821, "gate_a2b/L1_absmax": 0.66230816, "gate_a2b/L1_rms": 0.3268033, "gate_a2b/L3_absmean": 0.33640787, "gate_a2b/L3_absmax": 0.5238272, "gate_a2b/L3_rms": 0.34099585, "gate_a2b/L5_absmean": 0.51539576, "gate_a2b/L5_absmax": 0.68916398, "gate_a2b/L5_rms": 0.52109081, "gate_c2b/L1_absmean": 0.01357739, "gate_c2b/L1_absmax": 0.4393293, "gate_c2b/L1_rms": 0.05296429, "gate_c2b/L3_absmean": 0.21016192, "gate_c2b/L3_absmax": 0.42708549, "gate_c2b/L3_rms": 0.21412739, "gate_c2b/L5_absmean": 0.53042328, "gate_c2b/L5_absmax": 0.71440417, "gate_c2b/L5_rms": 0.54196221, "gate_b2a/L7_absmean": 0.08749129, "gate_b2a/L7_absmax": 0.27867466, "gate_b2a/L7_rms": 0.09069479, "gate_b2a/L11_absmean": 0.13354933, "gate_b2a/L11_absmax": 0.45464563, "gate_b2a/L11_rms": 0.13635953, "gate_b2a/L14_absmean": 0.32244313, "gate_b2a/L14_absmax": 0.55396652, "gate_b2a/L14_rms": 0.3238951, "gate_b2c/L3_absmean": 0.10194644, "gate_b2c/L3_absmax": 0.6203891, "gate_b2c/L3_rms": 0.11814331, "gate_b2c/L5_absmean": 0.12561636, "gate_b2c/L5_absmax": 0.76507086, "gate_b2c/L5_rms": 0.13352129, "gate_b2c/L6_absmean": 0.18448389, "gate_b2c/L6_absmax": 0.52334476, "gate_b2c/L6_rms": 0.18895251}, "streams": {"cos_a2b/round0": 0.41281, "cos_c2b/round0": 0.338345, "cos_b2a/round0": 0.124711, "cos_b2c/round0": 0.17454, "rms_a/round0": 0.815895, "rms_b/round0": 1.498726, "rms_c/round0": 1.184283, "cos_a2b/round1": 0.404958, "cos_c2b/round1": 0.243153, "cos_b2a/round1": 0.219681, "cos_b2c/round1": 0.117085, "rms_a/round1": 1.483279, "rms_b/round1": 3.061271, "rms_c/round1": 2.039004, "cos_a2b/round2": 0.509366, "cos_c2b/round2": 0.527205, "cos_b2a/round2": 0.075147, "cos_b2c/round2": 0.111427, "rms_a/round2": 3.200171, "rms_b/round2": 6.896558, "rms_c/round2": 3.354619}, "alpha_mean": 0.40569964455059093, "alpha_hist": [0.02226, 0.06465, 0.18135, 0.24194, 0.21606, 0.14844, 0.0842, 0.03642, 0.00449, 0.00018], "alpha_by_type": {"special": {"share": 0.00055, "mean": 0.55552, "std": 0.14923}, "whitespace": {"share": 0.11987, "mean": 0.49887, "std": 0.13602}, "digit": {"share": 0.10292, "mean": 0.34779, "std": 0.11086}, "word_start": {"share": 0.37128, "mean": 0.41036, "std": 0.16013}, "word_cont": {"share": 0.21481, "mean": 0.31309, "std": 0.12479}, "punct": {"share": 0.19057, "mean": 0.47325, "std": 0.15914}}, "alpha_p05": 0.1561, "alpha_p50": 0.37938, "alpha_p95": 0.68194}
{"step": 22910, "total_steps": 72479, "tokens": 12011438080, "loss": 5.0253342390060425, "l_mix": 3.114603966474533, "l_a": 3.2010010480880737, "l_c": 3.162492036819458, "z_loss": 16.823158740997314, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20604372024536133, "tokens_per_second": 1440303.8711356267, "eta_seconds": 18043.714519429206, "data_wait_seconds": 1.8715858459472656e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.20044, "finemath": 0.08232}}
{"step": 22920, "total_steps": 72479, "tokens": 12016680960, "loss": 4.992181062698364, "l_mix": 3.0943227112293243, "l_a": 3.174709916114807, "l_c": 3.146176278591156, "z_loss": 15.924211621284485, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2585025429725647, "tokens_per_second": 1432554.7493977502, "eta_seconds": 18137.658615088465, "data_wait_seconds": 2.6941299438476562e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20768, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 22930, "total_steps": 72479, "tokens": 12021923840, "loss": 5.02999609708786, "l_mix": 3.1178649365901947, "l_a": 3.2016639411449432, "l_c": 3.1664138436317444, "z_loss": 17.077538013458252, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21082156896591187, "tokens_per_second": 1473991.468067241, "eta_seconds": 17624.217422413825, "data_wait_seconds": 1.77919864654541e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20768, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 22940, "total_steps": 72479, "tokens": 12027166720, "loss": 4.888928651809692, "l_mix": 3.028700739145279, "l_a": 3.1177650094032288, "l_c": 3.077295571565628, "z_loss": 17.097215175628662, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19318699836730957, "tokens_per_second": 1353707.673491132, "eta_seconds": 19186.345575642586, "data_wait_seconds": 2.8640031814575195e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 22950, "total_steps": 72479, "tokens": 12032409600, "loss": 4.9621060490608215, "l_mix": 3.0754718482494354, "l_a": 3.1609999537467957, "l_c": 3.1222755908966064, "z_loss": 16.514338731765747, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19882026314735413, "tokens_per_second": 1320646.8035352363, "eta_seconds": 19662.683680820464, "data_wait_seconds": 2.232193946838379e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 22960, "total_steps": 72479, "tokens": 12037652480, "loss": 4.8502004742622375, "l_mix": 3.0051776468753815, "l_a": 3.0924027264118195, "l_c": 3.0519969165325165, "z_loss": 17.028651475906372, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20441439747810364, "tokens_per_second": 1346955.9977294935, "eta_seconds": 19274.73318784237, "data_wait_seconds": 2.8967857360839844e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08236}}
{"step": 22970, "total_steps": 72479, "tokens": 12042895360, "loss": 4.660384297370911, "l_mix": 2.8849372267723083, "l_a": 2.9816857874393463, "l_c": 2.930709034204483, "z_loss": 17.284769535064697, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17615191638469696, "tokens_per_second": 1407162.2557077936, "eta_seconds": 18446.32663128376, "data_wait_seconds": 2.0384788513183594e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08232}}
{"step": 22980, "total_steps": 72479, "tokens": 12048138240, "loss": 4.484735488891602, "l_mix": 2.775665044784546, "l_a": 2.8700941801071167, "l_c": 2.8211514353752136, "z_loss": 16.966028928756714, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19388426840305328, "tokens_per_second": 1434207.4594382506, "eta_seconds": 18094.82410736084, "data_wait_seconds": 2.321600914001465e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 22990, "total_steps": 72479, "tokens": 12053381120, "loss": 4.714143812656403, "l_mix": 2.9178990721702576, "l_a": 3.0147963762283325, "l_c": 2.9671339988708496, "z_loss": 16.65523672103882, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2661571204662323, "tokens_per_second": 1399227.217049852, "eta_seconds": 18543.442062759397, "data_wait_seconds": 4.118680953979492e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 23000, "total_steps": 72479, "tokens": 12058624000, "loss": 4.5497658252716064, "l_mix": 2.814631998538971, "l_a": 2.9174745678901672, "l_c": 2.8604802191257477, "z_loss": 17.473596811294556, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1994602233171463, "tokens_per_second": 1362343.2697961903, "eta_seconds": 19041.636955332757, "data_wait_seconds": 3.349781036376953e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.31193408, "gate_a2b/L1_absmax": 0.65590519, "gate_a2b/L1_rms": 0.32791698, "gate_a2b/L3_absmean": 0.33665854, "gate_a2b/L3_absmax": 0.52558792, "gate_a2b/L3_rms": 0.3411963, "gate_a2b/L5_absmean": 0.51506251, "gate_a2b/L5_absmax": 0.6958198, "gate_a2b/L5_rms": 0.52078074, "gate_c2b/L1_absmean": 0.0145783, "gate_c2b/L1_absmax": 0.43656871, "gate_c2b/L1_rms": 0.05309067, "gate_c2b/L3_absmean": 0.21013384, "gate_c2b/L3_absmax": 0.4291878, "gate_c2b/L3_rms": 0.21411784, "gate_c2b/L5_absmean": 0.5300011, "gate_c2b/L5_absmax": 0.71559972, "gate_c2b/L5_rms": 0.54158652, "gate_b2a/L7_absmean": 0.08762902, "gate_b2a/L7_absmax": 0.27629274, "gate_b2a/L7_rms": 0.09072006, "gate_b2a/L11_absmean": 0.13433959, "gate_b2a/L11_absmax": 0.45733577, "gate_b2a/L11_rms": 0.13711663, "gate_b2a/L14_absmean": 0.32212746, "gate_b2a/L14_absmax": 0.55134386, "gate_b2a/L14_rms": 0.32360208, "gate_b2c/L3_absmean": 0.10244636, "gate_b2c/L3_absmax": 0.62058747, "gate_b2c/L3_rms": 0.11852463, "gate_b2c/L5_absmean": 0.1259006, "gate_b2c/L5_absmax": 0.76673108, "gate_b2c/L5_rms": 0.1337319, "gate_b2c/L6_absmean": 0.1845018, "gate_b2c/L6_absmax": 0.52597028, "gate_b2c/L6_rms": 0.18903957}, "streams": {"cos_a2b/round0": 0.390045, "cos_c2b/round0": 0.30121, "cos_b2a/round0": 0.117992, "cos_b2c/round0": 0.155666, "rms_a/round0": 0.818202, "rms_b/round0": 1.558693, "rms_c/round0": 1.234348, "cos_a2b/round1": 0.415089, "cos_c2b/round1": 0.268589, "cos_b2a/round1": 0.224456, "cos_b2c/round1": 0.127418, "rms_a/round1": 1.467161, "rms_b/round1": 3.063928, "rms_c/round1": 2.027489, "cos_a2b/round2": 0.54899, "cos_c2b/round2": 0.482214, "cos_b2a/round2": 0.069001, "cos_b2c/round2": 0.13577, "rms_a/round2": 3.31009, "rms_b/round2": 6.789382, "rms_c/round2": 3.260882}, "alpha_mean": 0.4879010597382205, "alpha_hist": [0.00916, 0.0461, 0.11511, 0.16745, 0.17984, 0.1757, 0.16997, 0.11095, 0.02524, 0.00049], "alpha_by_type": {"special": {"share": 0.00128, "mean": 0.63066, "std": 0.12084}, "whitespace": {"share": 0.0287, "mean": 0.60354, "std": 0.11791}, "digit": {"share": 0.03575, "mean": 0.55248, "std": 0.126}, "word_start": {"share": 0.57205, "mean": 0.50242, "std": 0.16705}, "word_cont": {"share": 0.25333, "mean": 0.36733, "std": 0.15466}, "punct": {"share": 0.10889, "mean": 0.63876, "std": 0.14721}}, "alpha_p05": 0.18863, "alpha_p50": 0.487, "alpha_p95": 0.78662}
{"step": 23010, "total_steps": 72479, "tokens": 12063866880, "loss": 3.252284348011017, "l_mix": 2.0067304968833923, "l_a": 2.0789141207933426, "l_c": 2.06676809489727, "z_loss": 18.491610527038574, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.46981555223464966, "tokens_per_second": 1434846.055860661, "eta_seconds": 18075.808875846862, "data_wait_seconds": 2.1696090698242188e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08236}}
{"step": 23020, "total_steps": 72479, "tokens": 12069109760, "loss": 3.255706250667572, "l_mix": 2.012391149997711, "l_a": 2.0936915278434753, "l_c": 2.044397383928299, "z_loss": 18.884240865707397, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.244759663939476, "tokens_per_second": 1452742.1857480693, "eta_seconds": 17849.526534295084, "data_wait_seconds": 3.314018249511719e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08236}}
{"step": 23030, "total_steps": 72479, "tokens": 12074352640, "loss": 4.873915076255798, "l_mix": 3.0188737213611603, "l_a": 3.112256109714508, "l_c": 3.0655913054943085, "z_loss": 16.869962453842163, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20359782874584198, "tokens_per_second": 1398875.5395947883, "eta_seconds": 18533.112187743187, "data_wait_seconds": 2.294778823852539e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20769, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 23040, "total_steps": 72479, "tokens": 12079595520, "loss": 4.625034809112549, "l_mix": 2.862920969724655, "l_a": 2.9561864733695984, "l_c": 2.911844551563263, "z_loss": 17.04391312599182, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1635960191488266, "tokens_per_second": 1400496.1840875766, "eta_seconds": 18507.922211074827, "data_wait_seconds": 2.2172927856445312e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20764, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 23050, "total_steps": 72479, "tokens": 12084838400, "loss": 4.604599833488464, "l_mix": 2.8500972390174866, "l_a": 2.9443224370479584, "l_c": 2.898541808128357, "z_loss": 16.433149337768555, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21124055981636047, "tokens_per_second": 1429734.373605471, "eta_seconds": 18125.766597223283, "data_wait_seconds": 1.800060272216797e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20764, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 23060, "total_steps": 72479, "tokens": 12090081280, "loss": 4.594628214836121, "l_mix": 2.8440481424331665, "l_a": 2.9412178993225098, "l_c": 2.888304591178894, "z_loss": 17.232319831848145, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20721639692783356, "tokens_per_second": 1475441.1133470992, "eta_seconds": 17560.706718564034, "data_wait_seconds": 1.5795230865478516e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20763, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 23070, "total_steps": 72479, "tokens": 12095324160, "loss": 4.646308779716492, "l_mix": 2.8761423528194427, "l_a": 2.9700235724449158, "l_c": 2.924905091524124, "z_loss": 16.87822985649109, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18186229467391968, "tokens_per_second": 1341289.2816578283, "eta_seconds": 19313.168416571618, "data_wait_seconds": 1.531839370727539e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20763, "cosmopedia_v2": 0.20043, "finemath": 0.08236}}
{"step": 23080, "total_steps": 72479, "tokens": 12100567040, "loss": 4.648225665092468, "l_mix": 2.8766189515590668, "l_a": 2.978481113910675, "l_c": 2.921116054058075, "z_loss": 17.27537703514099, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18502084910869598, "tokens_per_second": 1446646.736321192, "eta_seconds": 17902.99059317112, "data_wait_seconds": 2.962350845336914e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08232}}
{"step": 23090, "total_steps": 72479, "tokens": 12105809920, "loss": 4.588666915893555, "l_mix": 2.839408725500107, "l_a": 2.9380095303058624, "l_c": 2.8872509598731995, "z_loss": 16.799144506454468, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17282915115356445, "tokens_per_second": 1413881.0892620177, "eta_seconds": 18314.170992636682, "data_wait_seconds": 2.7745962142944336e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 23100, "total_steps": 72479, "tokens": 12111052800, "loss": 4.629723012447357, "l_mix": 2.8657581508159637, "l_a": 2.9640182554721832, "l_c": 2.9104008972644806, "z_loss": 16.390769600868225, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22004537284374237, "tokens_per_second": 1397683.6803577626, "eta_seconds": 18522.658249378204, "data_wait_seconds": 2.8640031814575195e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.31182459, "gate_a2b/L1_absmax": 0.65320736, "gate_a2b/L1_rms": 0.3277345, "gate_a2b/L3_absmean": 0.33704218, "gate_a2b/L3_absmax": 0.53084773, "gate_a2b/L3_rms": 0.3416039, "gate_a2b/L5_absmean": 0.51625568, "gate_a2b/L5_absmax": 0.68979508, "gate_a2b/L5_rms": 0.52201915, "gate_c2b/L1_absmean": 0.01398075, "gate_c2b/L1_absmax": 0.43323931, "gate_c2b/L1_rms": 0.05272334, "gate_c2b/L3_absmean": 0.20963454, "gate_c2b/L3_absmax": 0.42566434, "gate_c2b/L3_rms": 0.21364301, "gate_c2b/L5_absmean": 0.52984035, "gate_c2b/L5_absmax": 0.71751946, "gate_c2b/L5_rms": 0.54145753, "gate_b2a/L7_absmean": 0.08739372, "gate_b2a/L7_absmax": 0.27402681, "gate_b2a/L7_rms": 0.09060941, "gate_b2a/L11_absmean": 0.13451271, "gate_b2a/L11_absmax": 0.45273501, "gate_b2a/L11_rms": 0.13731869, "gate_b2a/L14_absmean": 0.32308277, "gate_b2a/L14_absmax": 0.55176312, "gate_b2a/L14_rms": 0.32453862, "gate_b2c/L3_absmean": 0.10273751, "gate_b2c/L3_absmax": 0.61533487, "gate_b2c/L3_rms": 0.11879406, "gate_b2c/L5_absmean": 0.1263105, "gate_b2c/L5_absmax": 0.75429189, "gate_b2c/L5_rms": 0.13402723, "gate_b2c/L6_absmean": 0.18537086, "gate_b2c/L6_absmax": 0.53246796, "gate_b2c/L6_rms": 0.18987952}, "streams": {"cos_a2b/round0": 0.384739, "cos_c2b/round0": 0.296653, "cos_b2a/round0": 0.12275, "cos_b2c/round0": 0.151171, "rms_a/round0": 0.820916, "rms_b/round0": 1.545274, "rms_c/round0": 1.243058, "cos_a2b/round1": 0.41667, "cos_c2b/round1": 0.275227, "cos_b2a/round1": 0.234417, "cos_b2c/round1": 0.132174, "rms_a/round1": 1.449688, "rms_b/round1": 3.035846, "rms_c/round1": 2.045255, "cos_a2b/round2": 0.552359, "cos_c2b/round2": 0.481998, "cos_b2a/round2": 0.074261, "cos_b2c/round2": 0.143312, "rms_a/round2": 3.311292, "rms_b/round2": 6.834615, "rms_c/round2": 3.249201}, "alpha_mean": 0.4817614401489294, "alpha_hist": [0.01044, 0.05423, 0.12325, 0.16708, 0.1752, 0.18092, 0.14983, 0.10249, 0.03581, 0.00075], "alpha_by_type": {"special": {"share": 0.00122, "mean": 0.60413, "std": 0.10455}, "whitespace": {"share": 0.0247, "mean": 0.58766, "std": 0.12771}, "digit": {"share": 0.02777, "mean": 0.48509, "std": 0.11629}, "word_start": {"share": 0.58832, "mean": 0.50981, "std": 0.17093}, "word_cont": {"share": 0.25885, "mean": 0.34783, "std": 0.14787}, "punct": {"share": 0.09914, "mean": 0.63617, "std": 0.16752}}, "alpha_p05": 0.17106, "alpha_p50": 0.49292, "alpha_p95": 0.79343}
{"step": 23110, "total_steps": 72479, "tokens": 12116295680, "loss": 4.663029074668884, "l_mix": 2.8856644928455353, "l_a": 2.988975316286087, "l_c": 2.930038720369339, "z_loss": 16.602919101715088, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2092982828617096, "tokens_per_second": 1409978.6290484841, "eta_seconds": 18357.423111772536, "data_wait_seconds": 2.804398536682129e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 23120, "total_steps": 72479, "tokens": 12121538560, "loss": 3.4937626719474792, "l_mix": 2.1488915979862213, "l_a": 2.2706955671310425, "l_c": 2.206931412220001, "z_loss": 15.828596830368042, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.25521886348724365, "tokens_per_second": 1306058.357296489, "eta_seconds": 19814.069752264022, "data_wait_seconds": 1.9311904907226562e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 23130, "total_steps": 72479, "tokens": 12126781440, "loss": 3.350533127784729, "l_mix": 2.070874884724617, "l_a": 2.1514536142349243, "l_c": 2.1079696118831635, "z_loss": 18.311593770980835, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21700991690158844, "tokens_per_second": 1445172.0758852554, "eta_seconds": 17903.11959643364, "data_wait_seconds": 3.1888484954833984e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08236}}
{"step": 23140, "total_steps": 72479, "tokens": 12132024320, "loss": 4.758186161518097, "l_mix": 2.9460282027721405, "l_a": 3.0431061685085297, "l_c": 2.991833060979843, "z_loss": 16.761616945266724, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18401461839675903, "tokens_per_second": 1454756.079580336, "eta_seconds": 17781.569017028807, "data_wait_seconds": 1.9043684005737305e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08232}}
{"step": 23150, "total_steps": 72479, "tokens": 12137267200, "loss": 4.566953718662262, "l_mix": 2.8260277807712555, "l_a": 2.9252441227436066, "l_c": 2.8722282350063324, "z_loss": 16.842142820358276, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19188974797725677, "tokens_per_second": 1440481.9999312193, "eta_seconds": 17954.131154179573, "data_wait_seconds": 2.5033950805664062e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 23160, "total_steps": 72479, "tokens": 12142510080, "loss": 4.577598929405212, "l_mix": 2.833206832408905, "l_a": 2.9302872121334076, "l_c": 2.878719747066498, "z_loss": 16.8983633518219, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1845490038394928, "tokens_per_second": 1263454.0086070243, "eta_seconds": 20465.612278604505, "data_wait_seconds": 1.84476375579834e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20764, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 23170, "total_steps": 72479, "tokens": 12147752960, "loss": 4.643030762672424, "l_mix": 2.874073028564453, "l_a": 2.969530552625656, "l_c": 2.92134428024292, "z_loss": 16.9518723487854, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16768008470535278, "tokens_per_second": 1377873.8331756429, "eta_seconds": 18762.32523584366, "data_wait_seconds": 2.4110078811645508e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 23180, "total_steps": 72479, "tokens": 12152995840, "loss": 4.536711752414703, "l_mix": 2.807391107082367, "l_a": 2.907627433538437, "l_c": 2.8511592745780945, "z_loss": 16.845077514648438, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19657883048057556, "tokens_per_second": 1329407.0529721093, "eta_seconds": 19442.4078420639, "data_wait_seconds": 2.6404857635498047e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20043, "finemath": 0.08236}}
{"step": 23190, "total_steps": 72479, "tokens": 12158238720, "loss": 4.651161193847656, "l_mix": 2.8789546489715576, "l_a": 2.9753905534744263, "l_c": 2.926275461912155, "z_loss": 17.066441297531128, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19140110909938812, "tokens_per_second": 1448615.888540891, "eta_seconds": 17838.842881965636, "data_wait_seconds": 2.0951032638549805e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20768, "cosmopedia_v2": 0.20043, "finemath": 0.08232}}
{"step": 23200, "total_steps": 72479, "tokens": 12163481600, "loss": 4.851100027561188, "l_mix": 3.005877673625946, "l_a": 3.0908222794532776, "l_c": 3.0542157888412476, "z_loss": 17.107404470443726, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2074642926454544, "tokens_per_second": 1461848.358769824, "eta_seconds": 17673.78141310215, "data_wait_seconds": 2.5212764739990234e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.31216961, "gate_a2b/L1_absmax": 0.65108138, "gate_a2b/L1_rms": 0.32783458, "gate_a2b/L3_absmean": 0.33686692, "gate_a2b/L3_absmax": 0.52837098, "gate_a2b/L3_rms": 0.34143746, "gate_a2b/L5_absmean": 0.51556194, "gate_a2b/L5_absmax": 0.67916763, "gate_a2b/L5_rms": 0.52129626, "gate_c2b/L1_absmean": 0.01521227, "gate_c2b/L1_absmax": 0.43187377, "gate_c2b/L1_rms": 0.05309559, "gate_c2b/L3_absmean": 0.2093869, "gate_c2b/L3_absmax": 0.43641031, "gate_c2b/L3_rms": 0.21341914, "gate_c2b/L5_absmean": 0.5293048, "gate_c2b/L5_absmax": 0.71067351, "gate_c2b/L5_rms": 0.54086643, "gate_b2a/L7_absmean": 0.08768034, "gate_b2a/L7_absmax": 0.27060431, "gate_b2a/L7_rms": 0.09073922, "gate_b2a/L11_absmean": 0.13435572, "gate_b2a/L11_absmax": 0.45850375, "gate_b2a/L11_rms": 0.1372215, "gate_b2a/L14_absmean": 0.32285911, "gate_b2a/L14_absmax": 0.55618364, "gate_b2a/L14_rms": 0.32433426, "gate_b2c/L3_absmean": 0.10186176, "gate_b2c/L3_absmax": 0.62102032, "gate_b2c/L3_rms": 0.11797737, "gate_b2c/L5_absmean": 0.12578422, "gate_b2c/L5_absmax": 0.76356697, "gate_b2c/L5_rms": 0.13368163, "gate_b2c/L6_absmean": 0.18429586, "gate_b2c/L6_absmax": 0.52791446, "gate_b2c/L6_rms": 0.18888642}, "streams": {"cos_a2b/round0": 0.37937, "cos_c2b/round0": 0.293711, "cos_b2a/round0": 0.117299, "cos_b2c/round0": 0.154042, "rms_a/round0": 0.820276, "rms_b/round0": 1.55798, "rms_c/round0": 1.213889, "cos_a2b/round1": 0.411356, "cos_c2b/round1": 0.273848, "cos_b2a/round1": 0.229555, "cos_b2c/round1": 0.134314, "rms_a/round1": 1.45583, "rms_b/round1": 3.061328, "rms_c/round1": 2.042037, "cos_a2b/round2": 0.556349, "cos_c2b/round2": 0.483752, "cos_b2a/round2": 0.073922, "cos_b2c/round2": 0.145044, "rms_a/round2": 3.314129, "rms_b/round2": 6.728058, "rms_c/round2": 3.242651}, "alpha_mean": 0.45827185502592727, "alpha_hist": [0.0119, 0.05803, 0.13919, 0.17485, 0.18289, 0.20473, 0.14433, 0.07323, 0.01056, 0.00029], "alpha_by_type": {"special": {"share": 0.00046, "mean": 0.59051, "std": 0.15714}, "whitespace": {"share": 0.04054, "mean": 0.56313, "std": 0.12487}, "digit": {"share": 0.0298, "mean": 0.49929, "std": 0.14467}, "word_start": {"share": 0.5585, "mean": 0.48078, "std": 0.15316}, "word_cont": {"share": 0.25238, "mean": 0.32489, "std": 0.14292}, "punct": {"share": 0.11832, "mean": 0.58977, "std": 0.15549}}, "alpha_p05": 0.18011, "alpha_p50": 0.45217, "alpha_p95": 0.72894}
{"step": 23210, "total_steps": 72479, "tokens": 12168724480, "loss": 5.008806467056274, "l_mix": 3.10489284992218, "l_a": 3.1902084946632385, "l_c": 3.1505550742149353, "z_loss": 16.84462571144104, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2078678160905838, "tokens_per_second": 1430954.2623648692, "eta_seconds": 18051.691903352737, "data_wait_seconds": 1.8805265426635742e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 23220, "total_steps": 72479, "tokens": 12173967360, "loss": 4.874765038490295, "l_mix": 3.0213570296764374, "l_a": 3.106961965560913, "l_c": 3.0654995441436768, "z_loss": 16.6936457157135, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18804845213890076, "tokens_per_second": 1406436.5091143989, "eta_seconds": 18362.650873064995, "data_wait_seconds": 3.1322240829467773e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 23230, "total_steps": 72479, "tokens": 12179210240, "loss": 4.829257309436798, "l_mix": 2.992402493953705, "l_a": 3.077589750289917, "l_c": 3.039657413959503, "z_loss": 16.805562257766724, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19120799005031586, "tokens_per_second": 1344483.5978416344, "eta_seconds": 19204.890080809593, "data_wait_seconds": 2.047419548034668e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 23240, "total_steps": 72479, "tokens": 12184453120, "loss": 3.3051176965236664, "l_mix": 2.0422940850257874, "l_a": 2.1226805299520493, "l_c": 2.0805278569459915, "z_loss": 18.610379695892334, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2132604718208313, "tokens_per_second": 1407695.0722436323, "eta_seconds": 18338.784684991835, "data_wait_seconds": 2.6434659957885742e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08236}}
{"step": 23250, "total_steps": 72479, "tokens": 12189696000, "loss": 4.624377429485321, "l_mix": 2.8631542325019836, "l_a": 2.9587570428848267, "l_c": 2.906449466943741, "z_loss": 16.611803650856018, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16934292018413544, "tokens_per_second": 1355671.4402269267, "eta_seconds": 19038.664669132235, "data_wait_seconds": 2.6971101760864258e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08232}}
{"step": 23260, "total_steps": 72479, "tokens": 12194938880, "loss": 4.64667409658432, "l_mix": 2.8769001960754395, "l_a": 2.969662308692932, "l_c": 2.923983782529831, "z_loss": 16.798882484436035, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18014246225357056, "tokens_per_second": 1461075.218766625, "eta_seconds": 17661.60341408253, "data_wait_seconds": 2.1964311599731445e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 23270, "total_steps": 72479, "tokens": 12200181760, "loss": 4.651357114315033, "l_mix": 2.8797742128372192, "l_a": 2.972939372062683, "l_c": 2.9265396296977997, "z_loss": 17.392192363739014, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.25440794229507446, "tokens_per_second": 1456550.6434136787, "eta_seconds": 17712.867251586915, "data_wait_seconds": 1.6480684280395508e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 23280, "total_steps": 72479, "tokens": 12205424640, "loss": 4.699624240398407, "l_mix": 2.90888649225235, "l_a": 3.005466550588608, "l_c": 2.95792880654335, "z_loss": 17.191314458847046, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1978779286146164, "tokens_per_second": 1334518.8652497935, "eta_seconds": 19328.6479372263, "data_wait_seconds": 2.16066837310791e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 23290, "total_steps": 72479, "tokens": 12210667520, "loss": 4.587071239948273, "l_mix": 2.8386387825012207, "l_a": 2.9357330203056335, "l_c": 2.886756718158722, "z_loss": 16.854907512664795, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17650587856769562, "tokens_per_second": 1321056.0259714825, "eta_seconds": 19521.656860113144, "data_wait_seconds": 1.8835067749023438e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20764, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 23300, "total_steps": 72479, "tokens": 12215910400, "loss": 3.138829305768013, "l_mix": 1.9407482147216797, "l_a": 2.015014201402664, "l_c": 1.9724124893546104, "z_loss": 18.530937433242798, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2460252195596695, "tokens_per_second": 1432043.7037351069, "eta_seconds": 18005.00884487629, "data_wait_seconds": 3.17990779876709e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20764, "cosmopedia_v2": 0.20043, "finemath": 0.08236}, "gates": {"gate_a2b/L1_absmean": 0.3132579, "gate_a2b/L1_absmax": 0.65013206, "gate_a2b/L1_rms": 0.32878464, "gate_a2b/L3_absmean": 0.33698404, "gate_a2b/L3_absmax": 0.53535575, "gate_a2b/L3_rms": 0.34160048, "gate_a2b/L5_absmean": 0.51584834, "gate_a2b/L5_absmax": 0.68619835, "gate_a2b/L5_rms": 0.52157706, "gate_c2b/L1_absmean": 0.01464672, "gate_c2b/L1_absmax": 0.43119925, "gate_c2b/L1_rms": 0.05307349, "gate_c2b/L3_absmean": 0.20943156, "gate_c2b/L3_absmax": 0.43245539, "gate_c2b/L3_rms": 0.21347614, "gate_c2b/L5_absmean": 0.52885258, "gate_c2b/L5_absmax": 0.70726889, "gate_c2b/L5_rms": 0.54048049, "gate_b2a/L7_absmean": 0.08720294, "gate_b2a/L7_absmax": 0.27409539, "gate_b2a/L7_rms": 0.0903989, "gate_b2a/L11_absmean": 0.13444918, "gate_b2a/L11_absmax": 0.45921978, "gate_b2a/L11_rms": 0.13730861, "gate_b2a/L14_absmean": 0.32206678, "gate_b2a/L14_absmax": 0.55013067, "gate_b2a/L14_rms": 0.32354996, "gate_b2c/L3_absmean": 0.10253634, "gate_b2c/L3_absmax": 0.62223887, "gate_b2c/L3_rms": 0.11861268, "gate_b2c/L5_absmean": 0.12599859, "gate_b2c/L5_absmax": 0.7728892, "gate_b2c/L5_rms": 0.13402133, "gate_b2c/L6_absmean": 0.18454397, "gate_b2c/L6_absmax": 0.52605045, "gate_b2c/L6_rms": 0.18911348}, "streams": {"cos_a2b/round0": 0.414693, "cos_c2b/round0": 0.320257, "cos_b2a/round0": 0.107339, "cos_b2c/round0": 0.163433, "rms_a/round0": 0.845251, "rms_b/round0": 1.511057, "rms_c/round0": 1.218107, "cos_a2b/round1": 0.381679, "cos_c2b/round1": 0.238882, "cos_b2a/round1": 0.20429, "cos_b2c/round1": 0.100611, "rms_a/round1": 1.477862, "rms_b/round1": 3.077245, "rms_c/round1": 2.066935, "cos_a2b/round2": 0.479437, "cos_c2b/round2": 0.523924, "cos_b2a/round2": 0.051833, "cos_b2c/round2": 0.116186, "rms_a/round2": 3.154765, "rms_b/round2": 7.292884, "rms_c/round2": 3.310615}, "alpha_mean": 0.4305186286635525, "alpha_hist": [0.03084, 0.04454, 0.11357, 0.22601, 0.25758, 0.18954, 0.09634, 0.03198, 0.00784, 0.00174], "alpha_by_type": {"special": {"share": 0.0005, "mean": 0.56628, "std": 0.19159}, "whitespace": {"share": 0.09311, "mean": 0.52138, "std": 0.15688}, "digit": {"share": 0.12009, "mean": 0.45076, "std": 0.14037}, "word_start": {"share": 0.36043, "mean": 0.41866, "std": 0.15216}, "word_cont": {"share": 0.25761, "mean": 0.34423, "std": 0.1274}, "punct": {"share": 0.16826, "mean": 0.5229, "std": 0.14501}}, "alpha_p05": 0.15432, "alpha_p50": 0.41822, "alpha_p95": 0.6617}
{"step": 23310, "total_steps": 72479, "tokens": 12221153280, "loss": 4.458264097571373, "l_mix": 2.7636915296316147, "l_a": 2.8380486518144608, "l_c": 2.8048703521490097, "z_loss": 16.967973709106445, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2302197515964508, "tokens_per_second": 1388065.1867098361, "eta_seconds": 18571.69023387432, "data_wait_seconds": 2.580881118774414e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20768, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 23320, "total_steps": 72479, "tokens": 12226396160, "loss": 4.898817718029022, "l_mix": 3.034281402826309, "l_a": 3.1201492846012115, "l_c": 3.0893032252788544, "z_loss": 17.005534648895264, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.25028812885284424, "tokens_per_second": 1446805.876267233, "eta_seconds": 17814.051086449625, "data_wait_seconds": 2.536177635192871e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 23330, "total_steps": 72479, "tokens": 12231639040, "loss": 4.863442122936249, "l_mix": 3.0129135847091675, "l_a": 3.105656147003174, "l_c": 3.0569500625133514, "z_loss": 17.465625047683716, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2096688598394394, "tokens_per_second": 1454869.36155728, "eta_seconds": 17711.714599871637, "data_wait_seconds": 2.613663673400879e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 23340, "total_steps": 72479, "tokens": 12236881920, "loss": 4.8885380029678345, "l_mix": 3.029039740562439, "l_a": 3.109114110469818, "l_c": 3.0837349891662598, "z_loss": 16.435410261154175, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2563619613647461, "tokens_per_second": 1439163.8720101425, "eta_seconds": 17901.3582351923, "data_wait_seconds": 2.187490463256836e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 23350, "total_steps": 72479, "tokens": 12242124800, "loss": 4.776430785655975, "l_mix": 2.960264563560486, "l_a": 3.045407295227051, "l_c": 3.002814531326294, "z_loss": 16.996589183807373, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18587230145931244, "tokens_per_second": 1444543.0454507952, "eta_seconds": 17831.06791667938, "data_wait_seconds": 2.8461217880249023e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08236}}
{"step": 23360, "total_steps": 72479, "tokens": 12247367680, "loss": 4.71466338634491, "l_mix": 2.9194107353687286, "l_a": 3.014147073030472, "l_c": 2.96432688832283, "z_loss": 17.103293657302856, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15982432663440704, "tokens_per_second": 1332090.1767366328, "eta_seconds": 19332.401605939867, "data_wait_seconds": 1.84476375579834e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08232}}
{"step": 23370, "total_steps": 72479, "tokens": 12252610560, "loss": 4.7474082708358765, "l_mix": 2.9390751123428345, "l_a": 3.032635599374771, "l_c": 2.9895163774490356, "z_loss": 16.873984575271606, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15785415470600128, "tokens_per_second": 1445642.735266499, "eta_seconds": 17810.250599193572, "data_wait_seconds": 1.615285873413086e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 23380, "total_steps": 72479, "tokens": 12257853440, "loss": 4.600976884365082, "l_mix": 2.846878081560135, "l_a": 2.949151247739792, "l_c": 2.8921462893486023, "z_loss": 17.094572067260742, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19414517283439636, "tokens_per_second": 1467273.8377445831, "eta_seconds": 17544.111978149413, "data_wait_seconds": 1.6033649444580078e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 23390, "total_steps": 72479, "tokens": 12263096320, "loss": 4.607084274291992, "l_mix": 2.85258212685585, "l_a": 2.9474171102046967, "l_c": 2.8953204452991486, "z_loss": 16.808098793029785, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19644272327423096, "tokens_per_second": 1380375.6198690382, "eta_seconds": 18644.76107919216, "data_wait_seconds": 2.187490463256836e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 23400, "total_steps": 72479, "tokens": 12268339200, "loss": 4.606629908084869, "l_mix": 2.851376235485077, "l_a": 2.945454478263855, "l_c": 2.899744600057602, "z_loss": 16.938143968582153, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20223921537399292, "tokens_per_second": 1431131.8555401235, "eta_seconds": 17979.846268105506, "data_wait_seconds": 1.7017126083374023e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.31299925, "gate_a2b/L1_absmax": 0.65933001, "gate_a2b/L1_rms": 0.32869944, "gate_a2b/L3_absmean": 0.33694622, "gate_a2b/L3_absmax": 0.53311509, "gate_a2b/L3_rms": 0.34153795, "gate_a2b/L5_absmean": 0.51553005, "gate_a2b/L5_absmax": 0.67667508, "gate_a2b/L5_rms": 0.52121431, "gate_c2b/L1_absmean": 0.0144389, "gate_c2b/L1_absmax": 0.43443838, "gate_c2b/L1_rms": 0.05308404, "gate_c2b/L3_absmean": 0.20960215, "gate_c2b/L3_absmax": 0.43171352, "gate_c2b/L3_rms": 0.21356013, "gate_c2b/L5_absmean": 0.52919251, "gate_c2b/L5_absmax": 0.70939982, "gate_c2b/L5_rms": 0.54081959, "gate_b2a/L7_absmean": 0.08778818, "gate_b2a/L7_absmax": 0.27321857, "gate_b2a/L7_rms": 0.09082967, "gate_b2a/L11_absmean": 0.13417268, "gate_b2a/L11_absmax": 0.46025735, "gate_b2a/L11_rms": 0.13705276, "gate_b2a/L14_absmean": 0.32245126, "gate_b2a/L14_absmax": 0.54997718, "gate_b2a/L14_rms": 0.32390431, "gate_b2c/L3_absmean": 0.10318878, "gate_b2c/L3_absmax": 0.62390065, "gate_b2c/L3_rms": 0.11912978, "gate_b2c/L5_absmean": 0.12628506, "gate_b2c/L5_absmax": 0.77042103, "gate_b2c/L5_rms": 0.13422008, "gate_b2c/L6_absmean": 0.18502304, "gate_b2c/L6_absmax": 0.51813942, "gate_b2c/L6_rms": 0.18962008}, "streams": {"cos_a2b/round0": 0.389795, "cos_c2b/round0": 0.306531, "cos_b2a/round0": 0.11452, "cos_b2c/round0": 0.157425, "rms_a/round0": 0.82708, "rms_b/round0": 1.551204, "rms_c/round0": 1.23544, "cos_a2b/round1": 0.416555, "cos_c2b/round1": 0.268115, "cos_b2a/round1": 0.225634, "cos_b2c/round1": 0.116291, "rms_a/round1": 1.457654, "rms_b/round1": 3.063946, "rms_c/round1": 2.035697, "cos_a2b/round2": 0.532549, "cos_c2b/round2": 0.503807, "cos_b2a/round2": 0.069264, "cos_b2c/round2": 0.1241, "rms_a/round2": 3.284094, "rms_b/round2": 6.90691, "rms_c/round2": 3.307191}, "alpha_mean": 0.44197085452601925, "alpha_hist": [0.01582, 0.06001, 0.14781, 0.19846, 0.20009, 0.18144, 0.12607, 0.05862, 0.01112, 0.00055], "alpha_by_type": {"special": {"share": 0.00082, "mean": 0.57296, "std": 0.12992}, "whitespace": {"share": 0.03569, "mean": 0.58729, "std": 0.13189}, "digit": {"share": 0.04027, "mean": 0.50195, "std": 0.11516}, "word_start": {"share": 0.55113, "mean": 0.45572, "std": 0.15807}, "word_cont": {"share": 0.26517, "mean": 0.33545, "std": 0.14226}, "punct": {"share": 0.10692, "mean": 0.56319, "std": 0.16864}}, "alpha_p05": 0.1574, "alpha_p50": 0.44324, "alpha_p95": 0.73564}
{"step": 23410, "total_steps": 72479, "tokens": 12273582080, "loss": 2.9615881145000458, "l_mix": 1.8304166793823242, "l_a": 1.9023076742887497, "l_c": 1.861986443400383, "z_loss": 18.83183741569519, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.23680239915847778, "tokens_per_second": 1431574.4936563498, "eta_seconds": 17970.62464160919, "data_wait_seconds": 2.6345252990722656e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08236}}
{"step": 23420, "total_steps": 72479, "tokens": 12278824960, "loss": 3.063100039958954, "l_mix": 1.891637697815895, "l_a": 1.974120020866394, "l_c": 1.9243680834770203, "z_loss": 19.158849477767944, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2906326949596405, "tokens_per_second": 1330834.4828289065, "eta_seconds": 19327.005216550828, "data_wait_seconds": 1.8298625946044922e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20764, "cosmopedia_v2": 0.20043, "finemath": 0.08237}}
{"step": 23430, "total_steps": 72479, "tokens": 12284067840, "loss": 4.88900762796402, "l_mix": 3.030013859272003, "l_a": 3.1140123903751373, "l_c": 3.0772054195404053, "z_loss": 16.28370451927185, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17616359889507294, "tokens_per_second": 1470504.9553299558, "eta_seconds": 17487.735773205757, "data_wait_seconds": 2.6971101760864258e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 23440, "total_steps": 72479, "tokens": 12289310720, "loss": 4.7866082191467285, "l_mix": 2.9667764008045197, "l_a": 3.047584444284439, "l_c": 3.012888252735138, "z_loss": 16.899337768554688, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18096908926963806, "tokens_per_second": 1305091.6169089181, "eta_seconds": 19700.194912672043, "data_wait_seconds": 1.7583370208740234e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 23450, "total_steps": 72479, "tokens": 12294553600, "loss": 4.800141274929047, "l_mix": 2.973854660987854, "l_a": 3.0603807270526886, "l_c": 3.0216603875160217, "z_loss": 16.740721702575684, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16880396008491516, "tokens_per_second": 1481721.1552586125, "eta_seconds": 17348.28193602562, "data_wait_seconds": 5.3763389587402344e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20768, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 23460, "total_steps": 72479, "tokens": 12299796480, "loss": 4.849075734615326, "l_mix": 3.005196511745453, "l_a": 3.0920936167240143, "l_c": 3.0487192273139954, "z_loss": 16.35415470600128, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2151363492012024, "tokens_per_second": 1447726.5670104986, "eta_seconds": 17752.02172677517, "data_wait_seconds": 2.905726432800293e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 23470, "total_steps": 72479, "tokens": 12305039360, "loss": 4.5929282903671265, "l_mix": 2.843204617500305, "l_a": 2.937050014734268, "l_c": 2.889602243900299, "z_loss": 17.27891993522644, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15886405110359192, "tokens_per_second": 1381938.7204652214, "eta_seconds": 18593.321260547636, "data_wait_seconds": 0.016592711210250854, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08232}}
{"step": 23480, "total_steps": 72479, "tokens": 12310282240, "loss": 4.617356657981873, "l_mix": 2.858196586370468, "l_a": 2.9554527699947357, "l_c": 2.902902752161026, "z_loss": 16.53410530090332, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17369453608989716, "tokens_per_second": 1427987.3439398098, "eta_seconds": 17990.066803479196, "data_wait_seconds": 2.142786979675293e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 23490, "total_steps": 72479, "tokens": 12315525120, "loss": 4.639773488044739, "l_mix": 2.871820032596588, "l_a": 2.969254285097122, "l_c": 2.9182597398757935, "z_loss": 16.990904569625854, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18463873863220215, "tokens_per_second": 1344271.7972387641, "eta_seconds": 19106.511707496644, "data_wait_seconds": 1.7255544662475586e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 23500, "total_steps": 72479, "tokens": 12320768000, "loss": 4.6045375764369965, "l_mix": 2.850018411874771, "l_a": 2.9468336403369904, "l_c": 2.895966202020645, "z_loss": 16.791898012161255, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16500259935855865, "tokens_per_second": 1377665.9688103232, "eta_seconds": 18639.570500659942, "data_wait_seconds": 2.1129846572875977e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.31449339, "gate_a2b/L1_absmax": 0.6679405, "gate_a2b/L1_rms": 0.33010399, "gate_a2b/L3_absmean": 0.33738622, "gate_a2b/L3_absmax": 0.53205264, "gate_a2b/L3_rms": 0.34194902, "gate_a2b/L5_absmean": 0.51622766, "gate_a2b/L5_absmax": 0.6794616, "gate_a2b/L5_rms": 0.52187067, "gate_c2b/L1_absmean": 0.01494776, "gate_c2b/L1_absmax": 0.43330216, "gate_c2b/L1_rms": 0.05314623, "gate_c2b/L3_absmean": 0.20994692, "gate_c2b/L3_absmax": 0.42926297, "gate_c2b/L3_rms": 0.21393207, "gate_c2b/L5_absmean": 0.52953953, "gate_c2b/L5_absmax": 0.7125873, "gate_c2b/L5_rms": 0.54116285, "gate_b2a/L7_absmean": 0.08761658, "gate_b2a/L7_absmax": 0.27276772, "gate_b2a/L7_rms": 0.09065803, "gate_b2a/L11_absmean": 0.13453528, "gate_b2a/L11_absmax": 0.46163115, "gate_b2a/L11_rms": 0.1373836, "gate_b2a/L14_absmean": 0.32291812, "gate_b2a/L14_absmax": 0.5510813, "gate_b2a/L14_rms": 0.32440755, "gate_b2c/L3_absmean": 0.10354688, "gate_b2c/L3_absmax": 0.62699807, "gate_b2c/L3_rms": 0.11952661, "gate_b2c/L5_absmean": 0.12610132, "gate_b2c/L5_absmax": 0.77341825, "gate_b2c/L5_rms": 0.13411595, "gate_b2c/L6_absmean": 0.18536489, "gate_b2c/L6_absmax": 0.52090389, "gate_b2c/L6_rms": 0.18995239}, "streams": {"cos_a2b/round0": 0.388049, "cos_c2b/round0": 0.300347, "cos_b2a/round0": 0.11581, "cos_b2c/round0": 0.151707, "rms_a/round0": 0.82445, "rms_b/round0": 1.563324, "rms_c/round0": 1.253711, "cos_a2b/round1": 0.414219, "cos_c2b/round1": 0.271308, "cos_b2a/round1": 0.230998, "cos_b2c/round1": 0.123221, "rms_a/round1": 1.463731, "rms_b/round1": 3.067989, "rms_c/round1": 2.066089, "cos_a2b/round2": 0.536386, "cos_c2b/round2": 0.495933, "cos_b2a/round2": 0.063507, "cos_b2c/round2": 0.137136, "rms_a/round2": 3.305802, "rms_b/round2": 6.872859, "rms_c/round2": 3.34376}, "alpha_mean": 0.4568741074479732, "alpha_hist": [0.01144, 0.05969, 0.14061, 0.18752, 0.19449, 0.17184, 0.13361, 0.08133, 0.0188, 0.00067], "alpha_by_type": {"special": {"share": 0.00137, "mean": 0.5743, "std": 0.09478}, "whitespace": {"share": 0.02725, "mean": 0.55553, "std": 0.12613}, "digit": {"share": 0.03366, "mean": 0.53179, "std": 0.1232}, "word_start": {"share": 0.56126, "mean": 0.47646, "std": 0.16675}, "word_cont": {"share": 0.26532, "mean": 0.33904, "std": 0.14043}, "punct": {"share": 0.11113, "mean": 0.59093, "std": 0.16829}}, "alpha_p05": 0.16968, "alpha_p50": 0.4477, "alpha_p95": 0.75923}
{"step": 23510, "total_steps": 72479, "tokens": 12326010880, "loss": 4.742249250411987, "l_mix": 2.936103969812393, "l_a": 3.032769352197647, "l_c": 2.98207551240921, "z_loss": 16.917685747146606, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.24123597145080566, "tokens_per_second": 1331593.0495768224, "eta_seconds": 19280.55953743458, "data_wait_seconds": 2.664327621459961e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 23520, "total_steps": 72479, "tokens": 12331253760, "loss": 3.253075361251831, "l_mix": 2.0109432339668274, "l_a": 2.0891485810279846, "l_c": 2.0449557155370712, "z_loss": 19.00738215446472, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2524022161960602, "tokens_per_second": 1468608.3807511623, "eta_seconds": 17478.18991668224, "data_wait_seconds": 3.7044286727905273e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08236}}
{"step": 23530, "total_steps": 72479, "tokens": 12336496640, "loss": 3.236574411392212, "l_mix": 1.999960035085678, "l_a": 2.072238862514496, "l_c": 2.0433047115802765, "z_loss": 19.512176036834717, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.25554314255714417, "tokens_per_second": 1458161.3605595727, "eta_seconds": 17599.817143797874, "data_wait_seconds": 2.2739171981811523e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08236}}
{"step": 23540, "total_steps": 72479, "tokens": 12341739520, "loss": 4.92111074924469, "l_mix": 3.0496445894241333, "l_a": 3.1384256780147552, "l_c": 3.0941830575466156, "z_loss": 16.83635401725769, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16852949559688568, "tokens_per_second": 1424699.5953559042, "eta_seconds": 18009.502154445647, "data_wait_seconds": 1.9997358322143555e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20769, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 23550, "total_steps": 72479, "tokens": 12346982400, "loss": 5.0056726932525635, "l_mix": 3.1024989187717438, "l_a": 3.1884500682353973, "l_c": 3.1498174369335175, "z_loss": 16.933189511299133, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20240822434425354, "tokens_per_second": 1383910.2316530466, "eta_seconds": 18536.525682997704, "data_wait_seconds": 2.0295381546020508e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20769, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 23560, "total_steps": 72479, "tokens": 12352225280, "loss": 4.853035986423492, "l_mix": 3.0082502961158752, "l_a": 3.092468023300171, "l_c": 3.0510933697223663, "z_loss": 17.172202825546265, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22342880070209503, "tokens_per_second": 1436184.6089347894, "eta_seconds": 17858.18098344803, "data_wait_seconds": 2.8759241104125977e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20768, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 23570, "total_steps": 72479, "tokens": 12357468160, "loss": 4.795689642429352, "l_mix": 2.969252496957779, "l_a": 3.065255105495453, "l_c": 3.0171614289283752, "z_loss": 17.121184825897217, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17869728803634644, "tokens_per_second": 1445229.538044723, "eta_seconds": 17742.78833706379, "data_wait_seconds": 1.7136335372924805e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20764, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 23580, "total_steps": 72479, "tokens": 12362711040, "loss": 4.710142910480499, "l_mix": 2.9160578548908234, "l_a": 3.0117178559303284, "l_c": 2.9629200994968414, "z_loss": 16.937013864517212, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19701331853866577, "tokens_per_second": 1366117.776962851, "eta_seconds": 18766.433864140512, "data_wait_seconds": 2.3156404495239258e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20763, "cosmopedia_v2": 0.20042, "finemath": 0.08236}}
{"step": 23590, "total_steps": 72479, "tokens": 12367953920, "loss": 4.651462435722351, "l_mix": 2.879048705101013, "l_a": 2.9770977795124054, "l_c": 2.9252881705760956, "z_loss": 16.979921579360962, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16284285485744476, "tokens_per_second": 1404641.6531646198, "eta_seconds": 18248.01078214645, "data_wait_seconds": 2.396106719970703e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08232}}
{"step": 23600, "total_steps": 72479, "tokens": 12373196800, "loss": 4.692116618156433, "l_mix": 2.904379367828369, "l_a": 3.00200954079628, "l_c": 2.951430529356003, "z_loss": 17.052436113357544, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.14963200688362122, "tokens_per_second": 1460811.0235739457, "eta_seconds": 17542.770925498007, "data_wait_seconds": 1.4990568161010742e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.31508201, "gate_a2b/L1_absmax": 0.66791958, "gate_a2b/L1_rms": 0.33053625, "gate_a2b/L3_absmean": 0.33739743, "gate_a2b/L3_absmax": 0.53164643, "gate_a2b/L3_rms": 0.34197795, "gate_a2b/L5_absmean": 0.51568252, "gate_a2b/L5_absmax": 0.68769848, "gate_a2b/L5_rms": 0.5213514, "gate_c2b/L1_absmean": 0.01433587, "gate_c2b/L1_absmax": 0.43297711, "gate_c2b/L1_rms": 0.05308053, "gate_c2b/L3_absmean": 0.20924191, "gate_c2b/L3_absmax": 0.42471468, "gate_c2b/L3_rms": 0.2132218, "gate_c2b/L5_absmean": 0.52899045, "gate_c2b/L5_absmax": 0.70802301, "gate_c2b/L5_rms": 0.5404923, "gate_b2a/L7_absmean": 0.08763335, "gate_b2a/L7_absmax": 0.2759847, "gate_b2a/L7_rms": 0.09068561, "gate_b2a/L11_absmean": 0.13420406, "gate_b2a/L11_absmax": 0.46200806, "gate_b2a/L11_rms": 0.13708779, "gate_b2a/L14_absmean": 0.32222521, "gate_b2a/L14_absmax": 0.55212629, "gate_b2a/L14_rms": 0.32370436, "gate_b2c/L3_absmean": 0.10344275, "gate_b2c/L3_absmax": 0.62512904, "gate_b2c/L3_rms": 0.11942829, "gate_b2c/L5_absmean": 0.12630999, "gate_b2c/L5_absmax": 0.77848715, "gate_b2c/L5_rms": 0.134426, "gate_b2c/L6_absmean": 0.18530767, "gate_b2c/L6_absmax": 0.52335197, "gate_b2c/L6_rms": 0.18997508}, "streams": {"cos_a2b/round0": 0.385453, "cos_c2b/round0": 0.300747, "cos_b2a/round0": 0.120233, "cos_b2c/round0": 0.149362, "rms_a/round0": 0.819984, "rms_b/round0": 1.546831, "rms_c/round0": 1.250221, "cos_a2b/round1": 0.41346, "cos_c2b/round1": 0.27318, "cos_b2a/round1": 0.23706, "cos_b2c/round1": 0.125924, "rms_a/round1": 1.443424, "rms_b/round1": 3.073355, "rms_c/round1": 2.047943, "cos_a2b/round2": 0.540507, "cos_c2b/round2": 0.495629, "cos_b2a/round2": 0.068259, "cos_b2c/round2": 0.138096, "rms_a/round2": 3.283418, "rms_b/round2": 6.843544, "rms_c/round2": 3.31189}, "alpha_mean": 0.448450233065266, "alpha_hist": [0.01511, 0.06476, 0.14662, 0.18028, 0.19566, 0.18266, 0.12766, 0.07497, 0.01219, 9e-05], "alpha_by_type": {"special": {"share": 0.0014, "mean": 0.58205, "std": 0.09059}, "whitespace": {"share": 0.02634, "mean": 0.54638, "std": 0.11303}, "digit": {"share": 0.03416, "mean": 0.46523, "std": 0.11114}, "word_start": {"share": 0.56203, "mean": 0.47168, "std": 0.16528}, "word_cont": {"share": 0.24939, "mean": 0.32583, "std": 0.14869}, "punct": {"share": 0.12668, "mean": 0.56041, "std": 0.15676}}, "alpha_p05": 0.14223, "alpha_p50": 0.4708, "alpha_p95": 0.7345}
{"step": 23610, "total_steps": 72479, "tokens": 12378439680, "loss": 4.700912833213806, "l_mix": 2.910481870174408, "l_a": 3.0071310997009277, "l_c": 2.9552113711833954, "z_loss": 17.281811952590942, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15989768505096436, "tokens_per_second": 1363896.2285982508, "eta_seconds": 18785.46896367073, "data_wait_seconds": 1.6301870346069336e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 23620, "total_steps": 72479, "tokens": 12383682560, "loss": 4.727390289306641, "l_mix": 2.9275461435317993, "l_a": 3.0217044055461884, "l_c": 2.9721377789974213, "z_loss": 16.915220022201538, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16643261909484863, "tokens_per_second": 1352366.5830073708, "eta_seconds": 18941.748275852202, "data_wait_seconds": 1.6063451766967773e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 23630, "total_steps": 72479, "tokens": 12388925440, "loss": 4.7277631759643555, "l_mix": 2.9267870485782623, "l_a": 3.0218270421028137, "l_c": 2.9757271111011505, "z_loss": 17.09789228439331, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16893208026885986, "tokens_per_second": 1338674.8607750256, "eta_seconds": 19131.564551210402, "data_wait_seconds": 2.5391578674316406e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 23640, "total_steps": 72479, "tokens": 12394168320, "loss": 3.117409408092499, "l_mix": 1.927014172077179, "l_a": 2.0027441531419754, "l_c": 1.9587887972593307, "z_loss": 19.353315830230713, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.302063524723053, "tokens_per_second": 1364029.0520551153, "eta_seconds": 18772.1086977005, "data_wait_seconds": 2.3126602172851562e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08236}}
{"step": 23650, "total_steps": 72479, "tokens": 12399411200, "loss": 3.228768527507782, "l_mix": 1.9956516027450562, "l_a": 2.0781775414943695, "l_c": 2.0257673412561417, "z_loss": 19.333823919296265, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2647196352481842, "tokens_per_second": 1448461.1205608824, "eta_seconds": 17674.24640440941, "data_wait_seconds": 2.22623348236084e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08237}}
{"step": 23660, "total_steps": 72479, "tokens": 12404654080, "loss": 4.734680354595184, "l_mix": 2.93140771985054, "l_a": 3.022948831319809, "l_c": 2.9824966192245483, "z_loss": 16.38903260231018, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22852079570293427, "tokens_per_second": 1438867.8091980696, "eta_seconds": 17788.44151518345, "data_wait_seconds": 3.0428171157836914e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 23670, "total_steps": 72479, "tokens": 12409896960, "loss": 4.678479075431824, "l_mix": 2.89654803276062, "l_a": 2.992023140192032, "l_c": 2.9421301782131195, "z_loss": 16.850014448165894, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18679845333099365, "tokens_per_second": 1401197.865303044, "eta_seconds": 18262.92604753971, "data_wait_seconds": 2.7865171432495117e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 23680, "total_steps": 72479, "tokens": 12415139840, "loss": 4.753365933895111, "l_mix": 2.9431203603744507, "l_a": 3.036874830722809, "l_c": 2.991732954978943, "z_loss": 16.63243865966797, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19014835357666016, "tokens_per_second": 1427014.7158456985, "eta_seconds": 17928.8481246233, "data_wait_seconds": 2.2530555725097656e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20764, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 23690, "total_steps": 72479, "tokens": 12420382720, "loss": 4.674144268035889, "l_mix": 2.8939669132232666, "l_a": 2.9909302294254303, "l_c": 2.937401056289673, "z_loss": 16.778313875198364, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17501530051231384, "tokens_per_second": 1396486.1348314378, "eta_seconds": 18317.036305618287, "data_wait_seconds": 2.6345252990722656e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20042, "finemath": 0.08236}}
{"step": 23700, "total_steps": 72479, "tokens": 12425625600, "loss": 4.754718780517578, "l_mix": 2.9438014030456543, "l_a": 3.0395489037036896, "l_c": 2.991130083799362, "z_loss": 17.13707661628723, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18084610998630524, "tokens_per_second": 1445155.0755603071, "eta_seconds": 17696.539827799796, "data_wait_seconds": 1.677870750427246e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20042, "finemath": 0.08236}, "gates": {"gate_a2b/L1_absmean": 0.31474054, "gate_a2b/L1_absmax": 0.66217965, "gate_a2b/L1_rms": 0.33022594, "gate_a2b/L3_absmean": 0.33765569, "gate_a2b/L3_absmax": 0.5394612, "gate_a2b/L3_rms": 0.34231547, "gate_a2b/L5_absmean": 0.51548874, "gate_a2b/L5_absmax": 0.68609571, "gate_a2b/L5_rms": 0.5211423, "gate_c2b/L1_absmean": 0.01494076, "gate_c2b/L1_absmax": 0.43244821, "gate_c2b/L1_rms": 0.05316941, "gate_c2b/L3_absmean": 0.20874694, "gate_c2b/L3_absmax": 0.43257797, "gate_c2b/L3_rms": 0.21272509, "gate_c2b/L5_absmean": 0.5292716, "gate_c2b/L5_absmax": 0.71405476, "gate_c2b/L5_rms": 0.54079002, "gate_b2a/L7_absmean": 0.08710919, "gate_b2a/L7_absmax": 0.2796281, "gate_b2a/L7_rms": 0.09031622, "gate_b2a/L11_absmean": 0.13394389, "gate_b2a/L11_absmax": 0.46356249, "gate_b2a/L11_rms": 0.13689068, "gate_b2a/L14_absmean": 0.32260221, "gate_b2a/L14_absmax": 0.54534823, "gate_b2a/L14_rms": 0.32405713, "gate_b2c/L3_absmean": 0.10364972, "gate_b2c/L3_absmax": 0.62429649, "gate_b2c/L3_rms": 0.11966658, "gate_b2c/L5_absmean": 0.12604645, "gate_b2c/L5_absmax": 0.7833696, "gate_b2c/L5_rms": 0.13432571, "gate_b2c/L6_absmean": 0.18498659, "gate_b2c/L6_absmax": 0.52290553, "gate_b2c/L6_rms": 0.18967853}, "streams": {"cos_a2b/round0": 0.387378, "cos_c2b/round0": 0.302366, "cos_b2a/round0": 0.117168, "cos_b2c/round0": 0.153297, "rms_a/round0": 0.824262, "rms_b/round0": 1.548855, "rms_c/round0": 1.243353, "cos_a2b/round1": 0.411915, "cos_c2b/round1": 0.268503, "cos_b2a/round1": 0.226816, "cos_b2c/round1": 0.125684, "rms_a/round1": 1.469, "rms_b/round1": 3.076838, "rms_c/round1": 2.045313, "cos_a2b/round2": 0.538792, "cos_c2b/round2": 0.493271, "cos_b2a/round2": 0.061744, "cos_b2c/round2": 0.136994, "rms_a/round2": 3.295609, "rms_b/round2": 6.844714, "rms_c/round2": 3.303019}, "alpha_mean": 0.42288262237064345, "alpha_hist": [0.02313, 0.08311, 0.16066, 0.19247, 0.19804, 0.17, 0.11061, 0.05443, 0.00743, 0.00011], "alpha_by_type": {"special": {"share": 0.00148, "mean": 0.53798, "std": 0.10182}, "whitespace": {"share": 0.02786, "mean": 0.51224, "std": 0.12426}, "digit": {"share": 0.03639, "mean": 0.43541, "std": 0.12162}, "word_start": {"share": 0.56717, "mean": 0.44733, "std": 0.1612}, "word_cont": {"share": 0.26138, "mean": 0.30286, "std": 0.14371}, "punct": {"share": 0.10571, "mean": 0.55901, "std": 0.1622}}, "alpha_p05": 0.13455, "alpha_p50": 0.41656, "alpha_p95": 0.7272}
{"step": 23710, "total_steps": 72479, "tokens": 12430868480, "loss": 4.67094224691391, "l_mix": 2.8915816247463226, "l_a": 2.9878081679344177, "l_c": 2.937691479921341, "z_loss": 17.106140851974487, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.13954579830169678, "tokens_per_second": 1444315.2448732113, "eta_seconds": 17703.199881577493, "data_wait_seconds": 2.047419548034668e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20768, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 23720, "total_steps": 72479, "tokens": 12436111360, "loss": 4.94830584526062, "l_mix": 3.066295564174652, "l_a": 3.1526558697223663, "l_c": 3.11507910490036, "z_loss": 16.897804975509644, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15851333737373352, "tokens_per_second": 1464316.408561932, "eta_seconds": 17457.810649752617, "data_wait_seconds": 2.753734588623047e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 23730, "total_steps": 72479, "tokens": 12441354240, "loss": 4.882037281990051, "l_mix": 3.0251512825489044, "l_a": 3.113360643386841, "l_c": 3.070632725954056, "z_loss": 16.879135608673096, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.14799198508262634, "tokens_per_second": 1378621.7311067814, "eta_seconds": 18539.179482889176, "data_wait_seconds": 2.4706125259399414e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 23740, "total_steps": 72479, "tokens": 12446597120, "loss": 4.93117755651474, "l_mix": 3.0556361973285675, "l_a": 3.143261432647705, "l_c": 3.1027763187885284, "z_loss": 17.299892902374268, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19301632046699524, "tokens_per_second": 1281646.0985053799, "eta_seconds": 19937.854031467436, "data_wait_seconds": 1.7702579498291016e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 23750, "total_steps": 72479, "tokens": 12451840000, "loss": 3.297794848680496, "l_mix": 2.038234233856201, "l_a": 2.1178672909736633, "l_c": 2.0741567462682724, "z_loss": 19.53384566307068, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.41474393010139465, "tokens_per_second": 1471113.6972562983, "eta_seconds": 17366.45508749485, "data_wait_seconds": 2.6434659957885742e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08236}}
{"step": 23760, "total_steps": 72479, "tokens": 12457082880, "loss": 3.482883334159851, "l_mix": 2.152157351374626, "l_a": 2.2458525598049164, "l_c": 2.18359711766243, "z_loss": 18.910660982131958, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.3950226604938507, "tokens_per_second": 1354877.6871083858, "eta_seconds": 18852.467137837408, "data_wait_seconds": 1.767277717590332e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08237}}
{"step": 23770, "total_steps": 72479, "tokens": 12462325760, "loss": 4.579103708267212, "l_mix": 2.8336125910282135, "l_a": 2.9322232604026794, "l_c": 2.8803919553756714, "z_loss": 17.064262866973877, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15483680367469788, "tokens_per_second": 1383558.114930498, "eta_seconds": 18457.876049017905, "data_wait_seconds": 2.276897430419922e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 23780, "total_steps": 72479, "tokens": 12467568640, "loss": 4.727229595184326, "l_mix": 2.9269212186336517, "l_a": 3.022583246231079, "l_c": 2.9727157056331635, "z_loss": 17.185983657836914, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1458309292793274, "tokens_per_second": 1454727.7859012634, "eta_seconds": 17551.25705265999, "data_wait_seconds": 1.6123056411743164e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 23790, "total_steps": 72479, "tokens": 12472811520, "loss": 4.717580020427704, "l_mix": 2.920579195022583, "l_a": 3.0159355103969574, "l_c": 2.968356639146805, "z_loss": 17.131287336349487, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16471117734909058, "tokens_per_second": 1412228.1581393671, "eta_seconds": 18075.732511687278, "data_wait_seconds": 4.309415817260742e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 23800, "total_steps": 72479, "tokens": 12478054400, "loss": 4.711838781833649, "l_mix": 2.9167939722537994, "l_a": 3.0144107937812805, "l_c": 2.9632843136787415, "z_loss": 17.36317777633667, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17809787392616272, "tokens_per_second": 1409812.5735564656, "eta_seconds": 18102.9847730875, "data_wait_seconds": 1.576542854309082e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.31462795, "gate_a2b/L1_absmax": 0.6608488, "gate_a2b/L1_rms": 0.3301315, "gate_a2b/L3_absmean": 0.33781233, "gate_a2b/L3_absmax": 0.54506868, "gate_a2b/L3_rms": 0.3423616, "gate_a2b/L5_absmean": 0.51530308, "gate_a2b/L5_absmax": 0.68569398, "gate_a2b/L5_rms": 0.52092063, "gate_c2b/L1_absmean": 0.01410503, "gate_c2b/L1_absmax": 0.43326896, "gate_c2b/L1_rms": 0.05306534, "gate_c2b/L3_absmean": 0.20966621, "gate_c2b/L3_absmax": 0.43285277, "gate_c2b/L3_rms": 0.21358204, "gate_c2b/L5_absmean": 0.5292992, "gate_c2b/L5_absmax": 0.70919824, "gate_c2b/L5_rms": 0.54084647, "gate_b2a/L7_absmean": 0.08796827, "gate_b2a/L7_absmax": 0.27450386, "gate_b2a/L7_rms": 0.09108309, "gate_b2a/L11_absmean": 0.13447015, "gate_b2a/L11_absmax": 0.46609703, "gate_b2a/L11_rms": 0.13746804, "gate_b2a/L14_absmean": 0.3232654, "gate_b2a/L14_absmax": 0.55047226, "gate_b2a/L14_rms": 0.3247408, "gate_b2c/L3_absmean": 0.10348859, "gate_b2c/L3_absmax": 0.62612432, "gate_b2c/L3_rms": 0.11944374, "gate_b2c/L5_absmean": 0.12555562, "gate_b2c/L5_absmax": 0.78281057, "gate_b2c/L5_rms": 0.13383451, "gate_b2c/L6_absmean": 0.18489705, "gate_b2c/L6_absmax": 0.52590621, "gate_b2c/L6_rms": 0.1895441}, "streams": {"cos_a2b/round0": 0.389535, "cos_c2b/round0": 0.30091, "cos_b2a/round0": 0.11574, "cos_b2c/round0": 0.153517, "rms_a/round0": 0.827573, "rms_b/round0": 1.558746, "rms_c/round0": 1.246871, "cos_a2b/round1": 0.415231, "cos_c2b/round1": 0.26893, "cos_b2a/round1": 0.229738, "cos_b2c/round1": 0.125901, "rms_a/round1": 1.476233, "rms_b/round1": 3.111068, "rms_c/round1": 2.03596, "cos_a2b/round2": 0.541068, "cos_c2b/round2": 0.491864, "cos_b2a/round2": 0.060284, "cos_b2c/round2": 0.134093, "rms_a/round2": 3.34199, "rms_b/round2": 6.905273, "rms_c/round2": 3.297994}, "alpha_mean": 0.42656695953257895, "alpha_hist": [0.02124, 0.08611, 0.16725, 0.19278, 0.18301, 0.15509, 0.11601, 0.06589, 0.01227, 0.00035], "alpha_by_type": {"special": {"share": 0.00127, "mean": 0.58619, "std": 0.10257}, "whitespace": {"share": 0.02751, "mean": 0.5357, "std": 0.13255}, "digit": {"share": 0.04526, "mean": 0.44905, "std": 0.11568}, "word_start": {"share": 0.56236, "mean": 0.44675, "std": 0.1693}, "word_cont": {"share": 0.25475, "mean": 0.30415, "std": 0.14768}, "punct": {"share": 0.10886, "mean": 0.56997, "std": 0.1691}}, "alpha_p05": 0.14295, "alpha_p50": 0.40851, "alpha_p95": 0.71582}
{"step": 23810, "total_steps": 72479, "tokens": 12483297280, "loss": 4.8578179478645325, "l_mix": 3.0076132118701935, "l_a": 3.114822596311569, "l_c": 3.0467520654201508, "z_loss": 17.321725368499756, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.7433143854141235, "tokens_per_second": 1436928.338124024, "eta_seconds": 17757.721102023126, "data_wait_seconds": 2.104043960571289e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20042, "finemath": 0.08236}}
{"step": 23820, "total_steps": 72479, "tokens": 12488540160, "loss": 5.028750777244568, "l_mix": 3.1157782077789307, "l_a": 3.208262324333191, "l_c": 3.162665009498596, "z_loss": 16.94275689125061, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1704411506652832, "tokens_per_second": 1462287.839540776, "eta_seconds": 17446.17516617775, "data_wait_seconds": 1.862645149230957e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20768, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 23830, "total_steps": 72479, "tokens": 12493783040, "loss": 4.913960576057434, "l_mix": 3.0447411835193634, "l_a": 3.1345673203468323, "l_c": 3.090489447116852, "z_loss": 17.022067546844482, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1716526299715042, "tokens_per_second": 1335147.950953291, "eta_seconds": 19103.5659334898, "data_wait_seconds": 2.1129846572875977e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20768, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 23840, "total_steps": 72479, "tokens": 12499025920, "loss": 5.047234356403351, "l_mix": 3.128564774990082, "l_a": 3.213632047176361, "l_c": 3.176428973674774, "z_loss": 16.512446880340576, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17824886739253998, "tokens_per_second": 1477912.0056545835, "eta_seconds": 17254.642992568017, "data_wait_seconds": 2.7745962142944336e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 23850, "total_steps": 72479, "tokens": 12504268800, "loss": 4.912919700145721, "l_mix": 3.0443321764469147, "l_a": 3.134767323732376, "l_c": 3.088209092617035, "z_loss": 16.945727586746216, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17298036813735962, "tokens_per_second": 1293296.604383635, "eta_seconds": 19713.653515815735, "data_wait_seconds": 2.2232532501220703e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 23860, "total_steps": 72479, "tokens": 12509511680, "loss": 4.985323250293732, "l_mix": 3.0892122387886047, "l_a": 3.1779642403125763, "l_c": 3.1368209421634674, "z_loss": 16.753053307533264, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17039932310581207, "tokens_per_second": 1467413.4595905032, "eta_seconds": 17370.944845438004, "data_wait_seconds": 3.331899642944336e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08236}}
{"step": 23870, "total_steps": 72479, "tokens": 12514754560, "loss": 3.1154324412345886, "l_mix": 1.92483651638031, "l_a": 1.991088718175888, "l_c": 1.9710837751626968, "z_loss": 19.441211938858032, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.5750831365585327, "tokens_per_second": 1410549.1381963044, "eta_seconds": 18067.513354825976, "data_wait_seconds": 1.9848346710205078e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08236}}
{"step": 23880, "total_steps": 72479, "tokens": 12519997440, "loss": 4.633104681968689, "l_mix": 2.8673543334007263, "l_a": 2.966060608625412, "l_c": 2.914116770029068, "z_loss": 16.969828605651855, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16537025570869446, "tokens_per_second": 1420733.8736794228, "eta_seconds": 17934.303520202637, "data_wait_seconds": 2.8759241104125977e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 23890, "total_steps": 72479, "tokens": 12525240320, "loss": 4.761624455451965, "l_mix": 2.9477895200252533, "l_a": 3.0470166504383087, "l_c": 2.9933705627918243, "z_loss": 17.1864972114563, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1842217892408371, "tokens_per_second": 1429930.6322248855, "eta_seconds": 17815.29051682949, "data_wait_seconds": 2.0325183868408203e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 23900, "total_steps": 72479, "tokens": 12530483200, "loss": 4.73262757062912, "l_mix": 2.9300499856472015, "l_a": 3.026888459920883, "l_c": 2.9759649634361267, "z_loss": 17.214890718460083, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.14891237020492554, "tokens_per_second": 1463904.1635230894, "eta_seconds": 17398.26102461815, "data_wait_seconds": 1.4781951904296875e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.31437716, "gate_a2b/L1_absmax": 0.6593495, "gate_a2b/L1_rms": 0.32989532, "gate_a2b/L3_absmean": 0.33723539, "gate_a2b/L3_absmax": 0.54331774, "gate_a2b/L3_rms": 0.3418166, "gate_a2b/L5_absmean": 0.51540607, "gate_a2b/L5_absmax": 0.67829663, "gate_a2b/L5_rms": 0.52101541, "gate_c2b/L1_absmean": 0.01458828, "gate_c2b/L1_absmax": 0.43258765, "gate_c2b/L1_rms": 0.05321644, "gate_c2b/L3_absmean": 0.21001993, "gate_c2b/L3_absmax": 0.43340436, "gate_c2b/L3_rms": 0.21391994, "gate_c2b/L5_absmean": 0.52910084, "gate_c2b/L5_absmax": 0.71206552, "gate_c2b/L5_rms": 0.54061735, "gate_b2a/L7_absmean": 0.08826376, "gate_b2a/L7_absmax": 0.27504116, "gate_b2a/L7_rms": 0.09144013, "gate_b2a/L11_absmean": 0.13435915, "gate_b2a/L11_absmax": 0.4659566, "gate_b2a/L11_rms": 0.13732916, "gate_b2a/L14_absmean": 0.32315999, "gate_b2a/L14_absmax": 0.54327971, "gate_b2a/L14_rms": 0.32468081, "gate_b2c/L3_absmean": 0.10342152, "gate_b2c/L3_absmax": 0.6274268, "gate_b2c/L3_rms": 0.11913399, "gate_b2c/L5_absmean": 0.12552665, "gate_b2c/L5_absmax": 0.77507335, "gate_b2c/L5_rms": 0.13364989, "gate_b2c/L6_absmean": 0.1852119, "gate_b2c/L6_absmax": 0.52681762, "gate_b2c/L6_rms": 0.18987243}, "streams": {"cos_a2b/round0": 0.383047, "cos_c2b/round0": 0.296288, "cos_b2a/round0": 0.121483, "cos_b2c/round0": 0.149587, "rms_a/round0": 0.827814, "rms_b/round0": 1.560044, "rms_c/round0": 1.238463, "cos_a2b/round1": 0.420651, "cos_c2b/round1": 0.277381, "cos_b2a/round1": 0.232412, "cos_b2c/round1": 0.131161, "rms_a/round1": 1.479347, "rms_b/round1": 3.095081, "rms_c/round1": 2.041214, "cos_a2b/round2": 0.548655, "cos_c2b/round2": 0.484218, "cos_b2a/round2": 0.072136, "cos_b2c/round2": 0.143271, "rms_a/round2": 3.352476, "rms_b/round2": 6.8665, "rms_c/round2": 3.320967}, "alpha_mean": 0.4473306119796092, "alpha_hist": [0.02367, 0.08792, 0.1485, 0.16104, 0.15915, 0.17633, 0.13799, 0.08527, 0.02005, 9e-05], "alpha_by_type": {"special": {"share": 0.00136, "mean": 0.56402, "std": 0.09208}, "whitespace": {"share": 0.02339, "mean": 0.52728, "std": 0.12329}, "digit": {"share": 0.02315, "mean": 0.47954, "std": 0.12294}, "word_start": {"share": 0.5874, "mean": 0.47548, "std": 0.17338}, "word_cont": {"share": 0.25342, "mean": 0.30261, "std": 0.1545}, "punct": {"share": 0.11128, "mean": 0.6034, "std": 0.17416}}, "alpha_p05": 0.1538, "alpha_p50": 0.45617, "alpha_p95": 0.75092}
{"step": 23910, "total_steps": 72479, "tokens": 12535726080, "loss": 4.696929454803467, "l_mix": 2.90693998336792, "l_a": 3.00616854429245, "l_c": 2.9547722041606903, "z_loss": 17.072319984436035, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17696158587932587, "tokens_per_second": 1462062.671063845, "eta_seconds": 17416.58847870827, "data_wait_seconds": 1.5854835510253906e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 23920, "total_steps": 72479, "tokens": 12540968960, "loss": 4.61551570892334, "l_mix": 2.8582955598831177, "l_a": 2.952865242958069, "l_c": 2.8988939225673676, "z_loss": 16.923792123794556, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21550823748111725, "tokens_per_second": 1437971.1360930796, "eta_seconds": 17704.73714873791, "data_wait_seconds": 2.1010637283325195e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08236}}
{"step": 23930, "total_steps": 72479, "tokens": 12546211840, "loss": 3.4391757249832153, "l_mix": 2.1271146535873413, "l_a": 2.2035473436117172, "l_c": 2.1637245267629623, "z_loss": 18.79559898376465, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2090020775794983, "tokens_per_second": 1441699.1333257325, "eta_seconds": 17655.319007706643, "data_wait_seconds": 1.7434358596801758e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08237}}
{"step": 23940, "total_steps": 72479, "tokens": 12551454720, "loss": 4.870876252651215, "l_mix": 3.018953800201416, "l_a": 3.106408715248108, "l_c": 3.0608659386634827, "z_loss": 17.40044379234314, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19049200415611267, "tokens_per_second": 1395441.2486116332, "eta_seconds": 18236.823124814033, "data_wait_seconds": 2.2530555725097656e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20769, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 23950, "total_steps": 72479, "tokens": 12556697600, "loss": 4.834534466266632, "l_mix": 2.995457351207733, "l_a": 3.0822207927703857, "l_c": 3.0423443615436554, "z_loss": 17.07498049736023, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1855880320072174, "tokens_per_second": 1402965.486490204, "eta_seconds": 18135.280302333835, "data_wait_seconds": 1.6361474990844727e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 23960, "total_steps": 72479, "tokens": 12561940480, "loss": 4.863046050071716, "l_mix": 3.013080894947052, "l_a": 3.1009328961372375, "l_c": 3.059988170862198, "z_loss": 16.88841414451599, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1930239349603653, "tokens_per_second": 1321582.1674667893, "eta_seconds": 19248.08770744801, "data_wait_seconds": 3.4242868423461914e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 23970, "total_steps": 72479, "tokens": 12567183360, "loss": 5.108488857746124, "l_mix": 3.1662015318870544, "l_a": 3.244708836078644, "l_c": 3.223814070224762, "z_loss": 17.30314040184021, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.46887698769569397, "tokens_per_second": 1346936.5269777386, "eta_seconds": 18881.874596619604, "data_wait_seconds": 3.87728214263916e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20768, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 23980, "total_steps": 72479, "tokens": 12572426240, "loss": 3.001444637775421, "l_mix": 1.853771910071373, "l_a": 1.9254881888628006, "l_c": 1.893946960568428, "z_loss": 18.42063021659851, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2630053162574768, "tokens_per_second": 1340273.8686001839, "eta_seconds": 18971.826809215545, "data_wait_seconds": 2.0176172256469727e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08236}}
{"step": 23990, "total_steps": 72479, "tokens": 12577669120, "loss": 4.690703094005585, "l_mix": 2.9041015207767487, "l_a": 3.001024544239044, "l_c": 2.948683440685272, "z_loss": 16.892456769943237, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.26519519090652466, "tokens_per_second": 1343748.786988983, "eta_seconds": 18918.86420896053, "data_wait_seconds": 2.0176172256469727e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 24000, "total_steps": 72479, "tokens": 12582912000, "loss": 4.684343159198761, "l_mix": 2.8988043069839478, "l_a": 3.000570237636566, "l_c": 2.945624977350235, "z_loss": 16.80324101448059, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16908062994480133, "tokens_per_second": 1426892.0268478673, "eta_seconds": 17812.810972213745, "data_wait_seconds": 1.6897916793823242e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.31383958, "gate_a2b/L1_absmax": 0.66172749, "gate_a2b/L1_rms": 0.32921991, "gate_a2b/L3_absmean": 0.33722737, "gate_a2b/L3_absmax": 0.5368858, "gate_a2b/L3_rms": 0.34183684, "gate_a2b/L5_absmean": 0.51517653, "gate_a2b/L5_absmax": 0.67725968, "gate_a2b/L5_rms": 0.52084154, "gate_c2b/L1_absmean": 0.0149981, "gate_c2b/L1_absmax": 0.43390033, "gate_c2b/L1_rms": 0.05319713, "gate_c2b/L3_absmean": 0.20920324, "gate_c2b/L3_absmax": 0.43096569, "gate_c2b/L3_rms": 0.21315603, "gate_c2b/L5_absmean": 0.52930182, "gate_c2b/L5_absmax": 0.72267467, "gate_c2b/L5_rms": 0.54095811, "gate_b2a/L7_absmean": 0.08790972, "gate_b2a/L7_absmax": 0.27460951, "gate_b2a/L7_rms": 0.09097778, "gate_b2a/L11_absmean": 0.13435513, "gate_b2a/L11_absmax": 0.46385509, "gate_b2a/L11_rms": 0.13730308, "gate_b2a/L14_absmean": 0.32453185, "gate_b2a/L14_absmax": 0.55262268, "gate_b2a/L14_rms": 0.32602921, "gate_b2c/L3_absmean": 0.10343108, "gate_b2c/L3_absmax": 0.62929034, "gate_b2c/L3_rms": 0.11921558, "gate_b2c/L5_absmean": 0.12569833, "gate_b2c/L5_absmax": 0.78129995, "gate_b2c/L5_rms": 0.13402008, "gate_b2c/L6_absmean": 0.18469232, "gate_b2c/L6_absmax": 0.52083349, "gate_b2c/L6_rms": 0.18931267}, "streams": {"cos_a2b/round0": 0.380168, "cos_c2b/round0": 0.29179, "cos_b2a/round0": 0.114682, "cos_b2c/round0": 0.145018, "rms_a/round0": 0.823856, "rms_b/round0": 1.568574, "rms_c/round0": 1.261207, "cos_a2b/round1": 0.404638, "cos_c2b/round1": 0.268278, "cos_b2a/round1": 0.225845, "cos_b2c/round1": 0.126948, "rms_a/round1": 1.464764, "rms_b/round1": 3.130349, "rms_c/round1": 2.066398, "cos_a2b/round2": 0.532194, "cos_c2b/round2": 0.492663, "cos_b2a/round2": 0.063089, "cos_b2c/round2": 0.13408, "rms_a/round2": 3.323191, "rms_b/round2": 6.991959, "rms_c/round2": 3.352501}, "alpha_mean": 0.42225734489775135, "alpha_hist": [0.02681, 0.09058, 0.17697, 0.19475, 0.17003, 0.13976, 0.1113, 0.07065, 0.01889, 0.00027], "alpha_by_type": {"special": {"share": 0.00139, "mean": 0.53468, "std": 0.10607}, "whitespace": {"share": 0.02631, "mean": 0.49737, "std": 0.12469}, "digit": {"share": 0.0351, "mean": 0.40627, "std": 0.12094}, "word_start": {"share": 0.55122, "mean": 0.4609, "std": 0.17748}, "word_cont": {"share": 0.27235, "mean": 0.28838, "std": 0.1369}, "punct": {"share": 0.11363, "mean": 0.54188, "std": 0.1928}}, "alpha_p05": 0.14609, "alpha_p50": 0.40663, "alpha_p95": 0.74279}
{"step": 24010, "total_steps": 72479, "tokens": 12588154880, "loss": 4.6879332065582275, "l_mix": 2.902168720960617, "l_a": 2.9965603053569794, "l_c": 2.950419306755066, "z_loss": 16.704294443130493, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16402408480644226, "tokens_per_second": 1388952.9670077576, "eta_seconds": 18295.590761971474, "data_wait_seconds": 2.0384788513183594e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 24020, "total_steps": 72479, "tokens": 12593397760, "loss": 4.669840157032013, "l_mix": 2.8905628323554993, "l_a": 2.9890314042568207, "l_c": 2.9362919628620148, "z_loss": 16.80168914794922, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15312311053276062, "tokens_per_second": 1426586.9236701678, "eta_seconds": 17809.270343399046, "data_wait_seconds": 1.6748905181884766e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 24030, "total_steps": 72479, "tokens": 12598640640, "loss": 4.7254467606544495, "l_mix": 2.925193339586258, "l_a": 3.0231493711471558, "l_c": 2.9720101058483124, "z_loss": 17.05554509162903, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18075886368751526, "tokens_per_second": 1474113.2006091329, "eta_seconds": 17231.532355523108, "data_wait_seconds": 1.704692840576172e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08236}}
{"step": 24040, "total_steps": 72479, "tokens": 12603883520, "loss": 3.176744818687439, "l_mix": 1.9626590609550476, "l_a": 2.0544702410697937, "l_c": 1.9862317889928818, "z_loss": 18.75125551223755, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.3513849377632141, "tokens_per_second": 1425779.4376588261, "eta_seconds": 17812.00216612816, "data_wait_seconds": 3.3229589462280273e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08236}}
{"step": 24050, "total_steps": 72479, "tokens": 12609126400, "loss": 3.4607605636119843, "l_mix": 2.1283326148986816, "l_a": 2.2475392818450928, "l_c": 2.1886287331581116, "z_loss": 15.77545154094696, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21471795439720154, "tokens_per_second": 1471634.0061188275, "eta_seconds": 17253.436279964448, "data_wait_seconds": 1.6391277313232422e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20046, "finemath": 0.08233}}
{"step": 24060, "total_steps": 72479, "tokens": 12614369280, "loss": 3.431943714618683, "l_mix": 2.1108696162700653, "l_a": 2.233448773622513, "l_c": 2.1649007499217987, "z_loss": 15.691616296768188, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21368591487407684, "tokens_per_second": 1239069.5838409907, "eta_seconds": 20487.550500035286, "data_wait_seconds": 2.0384788513183594e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20046, "finemath": 0.08234}}
{"step": 24070, "total_steps": 72479, "tokens": 12619612160, "loss": 4.937001168727875, "l_mix": 3.0596983432769775, "l_a": 3.144859164953232, "l_c": 3.1071559488773346, "z_loss": 16.98226761817932, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15854521095752716, "tokens_per_second": 1436252.8966752698, "eta_seconds": 17671.162126636504, "data_wait_seconds": 2.345442771911621e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20769, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 24080, "total_steps": 72479, "tokens": 12624855040, "loss": 4.911023437976837, "l_mix": 3.043287545442581, "l_a": 3.1276717483997345, "l_c": 3.092455893754959, "z_loss": 16.976267099380493, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15957452356815338, "tokens_per_second": 1343456.778264651, "eta_seconds": 18887.853574848174, "data_wait_seconds": 2.0056962966918945e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20768, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 24090, "total_steps": 72479, "tokens": 12630097920, "loss": 4.742637634277344, "l_mix": 2.9358071982860565, "l_a": 3.036636173725128, "l_c": 2.9804666340351105, "z_loss": 16.996618509292603, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19963215291500092, "tokens_per_second": 1193736.3408758142, "eta_seconds": 21252.408227252963, "data_wait_seconds": 2.2202730178833008e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20764, "cosmopedia_v2": 0.20042, "finemath": 0.08236}}
{"step": 24100, "total_steps": 72479, "tokens": 12635340800, "loss": 4.685590505599976, "l_mix": 2.900995522737503, "l_a": 2.9936822056770325, "l_c": 2.9493211209774017, "z_loss": 16.939103603363037, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16146732866764069, "tokens_per_second": 1454706.0370866072, "eta_seconds": 17436.18882808685, "data_wait_seconds": 1.963973045349121e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20768, "cosmopedia_v2": 0.20041, "finemath": 0.08232}, "gates": {"gate_a2b/L1_absmean": 0.31553456, "gate_a2b/L1_absmax": 0.6605593, "gate_a2b/L1_rms": 0.33101159, "gate_a2b/L3_absmean": 0.33793935, "gate_a2b/L3_absmax": 0.5408932, "gate_a2b/L3_rms": 0.34250906, "gate_a2b/L5_absmean": 0.51579875, "gate_a2b/L5_absmax": 0.67863715, "gate_a2b/L5_rms": 0.52143878, "gate_c2b/L1_absmean": 0.01442671, "gate_c2b/L1_absmax": 0.43541443, "gate_c2b/L1_rms": 0.05325691, "gate_c2b/L3_absmean": 0.21009074, "gate_c2b/L3_absmax": 0.43634319, "gate_c2b/L3_rms": 0.21413721, "gate_c2b/L5_absmean": 0.52962095, "gate_c2b/L5_absmax": 0.71226954, "gate_c2b/L5_rms": 0.54123914, "gate_b2a/L7_absmean": 0.08824092, "gate_b2a/L7_absmax": 0.27299964, "gate_b2a/L7_rms": 0.09137399, "gate_b2a/L11_absmean": 0.13461173, "gate_b2a/L11_absmax": 0.46691081, "gate_b2a/L11_rms": 0.13754855, "gate_b2a/L14_absmean": 0.32388353, "gate_b2a/L14_absmax": 0.54837143, "gate_b2a/L14_rms": 0.32537439, "gate_b2c/L3_absmean": 0.10428123, "gate_b2c/L3_absmax": 0.62813604, "gate_b2c/L3_rms": 0.11981668, "gate_b2c/L5_absmean": 0.1261104, "gate_b2c/L5_absmax": 0.77749753, "gate_b2c/L5_rms": 0.13424692, "gate_b2c/L6_absmean": 0.18498468, "gate_b2c/L6_absmax": 0.5258531, "gate_b2c/L6_rms": 0.18966289}, "streams": {"cos_a2b/round0": 0.388823, "cos_c2b/round0": 0.306383, "cos_b2a/round0": 0.113443, "cos_b2c/round0": 0.149443, "rms_a/round0": 0.832475, "rms_b/round0": 1.555247, "rms_c/round0": 1.240073, "cos_a2b/round1": 0.413465, "cos_c2b/round1": 0.270471, "cos_b2a/round1": 0.224059, "cos_b2c/round1": 0.122926, "rms_a/round1": 1.47191, "rms_b/round1": 3.077111, "rms_c/round1": 2.044758, "cos_a2b/round2": 0.543808, "cos_c2b/round2": 0.487116, "cos_b2a/round2": 0.065226, "cos_b2c/round2": 0.132805, "rms_a/round2": 3.314358, "rms_b/round2": 6.839482, "rms_c/round2": 3.271061}, "alpha_mean": 0.4335957842434439, "alpha_hist": [0.00998, 0.06554, 0.16269, 0.20244, 0.20425, 0.1767, 0.11717, 0.05537, 0.00568, 0.00018], "alpha_by_type": {"special": {"share": 0.00121, "mean": 0.55156, "std": 0.08946}, "whitespace": {"share": 0.02757, "mean": 0.50984, "std": 0.12147}, "digit": {"share": 0.04314, "mean": 0.47601, "std": 0.12831}, "word_start": {"share": 0.55225, "mean": 0.44643, "std": 0.15913}, "word_cont": {"share": 0.26135, "mean": 0.33067, "std": 0.13002}, "punct": {"share": 0.11449, "mean": 0.57107, "std": 0.14826}}, "alpha_p05": 0.16968, "alpha_p50": 0.43903, "alpha_p95": 0.71681}
{"step": 24110, "total_steps": 72479, "tokens": 12640583680, "loss": 4.701613128185272, "l_mix": 2.9100714027881622, "l_a": 3.0088558197021484, "l_c": 2.95733642578125, "z_loss": 16.84000873565674, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.14853434264659882, "tokens_per_second": 1371986.6715630912, "eta_seconds": 18483.624365758897, "data_wait_seconds": 3.266334533691406e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08233}}
{"step": 24120, "total_steps": 72479, "tokens": 12645826560, "loss": 4.629099547863007, "l_mix": 2.8654593527317047, "l_a": 2.9626393914222717, "l_c": 2.9104998409748077, "z_loss": 16.984378337860107, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.14864104986190796, "tokens_per_second": 1411563.2287957394, "eta_seconds": 17961.6774330616, "data_wait_seconds": 1.65402889251709e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08234}}
{"step": 24130, "total_steps": 72479, "tokens": 12651069440, "loss": 4.6861990094184875, "l_mix": 2.901081532239914, "l_a": 3.0000685453414917, "l_c": 2.9446407854557037, "z_loss": 17.04707646369934, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20464195311069489, "tokens_per_second": 1187608.2279868287, "eta_seconds": 21344.41300981045, "data_wait_seconds": 2.3305416107177734e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 24140, "total_steps": 72479, "tokens": 12656312320, "loss": 4.667531073093414, "l_mix": 2.8896912336349487, "l_a": 2.9877928495407104, "l_c": 2.9326717853546143, "z_loss": 17.00363326072693, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16437847912311554, "tokens_per_second": 1444081.541708528, "eta_seconds": 17549.949154543876, "data_wait_seconds": 1.9818544387817383e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 24150, "total_steps": 72479, "tokens": 12661555200, "loss": 3.106523245573044, "l_mix": 1.9209486097097397, "l_a": 1.9946281611919403, "l_c": 1.951003298163414, "z_loss": 18.851749897003174, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.226614847779274, "tokens_per_second": 1315011.4265761657, "eta_seconds": 19268.512987732887, "data_wait_seconds": 1.671910285949707e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20041, "finemath": 0.08236}}
{"step": 24160, "total_steps": 72479, "tokens": 12666798080, "loss": 3.428594648838043, "l_mix": 2.1073698103427887, "l_a": 2.237511098384857, "l_c": 2.161362200975418, "z_loss": 15.627907514572144, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.208420068025589, "tokens_per_second": 1449125.6543744681, "eta_seconds": 17481.625417041778, "data_wait_seconds": 2.8908252716064453e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20046, "finemath": 0.08233}}
{"step": 24170, "total_steps": 72479, "tokens": 12672040960, "loss": 3.4073935449123383, "l_mix": 2.094335526227951, "l_a": 2.2191898226737976, "l_c": 2.1524820625782013, "z_loss": 15.563992977142334, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1883295178413391, "tokens_per_second": 1392369.5508221542, "eta_seconds": 18190.45021276474, "data_wait_seconds": 2.8818845748901367e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20046, "finemath": 0.08233}}
{"step": 24180, "total_steps": 72479, "tokens": 12677283840, "loss": 4.638217628002167, "l_mix": 2.8714661598205566, "l_a": 2.9690592885017395, "l_c": 2.914406567811966, "z_loss": 17.116848945617676, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15811802446842194, "tokens_per_second": 1404592.5766834605, "eta_seconds": 18028.420861935614, "data_wait_seconds": 1.5944242477416992e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20765, "cosmopedia_v2": 0.20041, "finemath": 0.08234}}
{"step": 24190, "total_steps": 72479, "tokens": 12682526720, "loss": 4.745029926300049, "l_mix": 2.9385378658771515, "l_a": 3.033424526453018, "l_c": 2.9825886487960815, "z_loss": 16.880423307418823, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17339058220386505, "tokens_per_second": 1437970.2898156291, "eta_seconds": 17606.304811239243, "data_wait_seconds": 1.5854835510253906e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20765, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 24200, "total_steps": 72479, "tokens": 12687769600, "loss": 4.715655267238617, "l_mix": 2.919749617576599, "l_a": 3.019228160381317, "l_c": 2.9615198373794556, "z_loss": 16.810842990875244, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17660173773765564, "tokens_per_second": 1400992.6335911031, "eta_seconds": 18067.261558055878, "data_wait_seconds": 1.531839370727539e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20764, "cosmopedia_v2": 0.20041, "finemath": 0.08236}, "gates": {"gate_a2b/L1_absmean": 0.31634149, "gate_a2b/L1_absmax": 0.65806085, "gate_a2b/L1_rms": 0.33156508, "gate_a2b/L3_absmean": 0.33846781, "gate_a2b/L3_absmax": 0.53941464, "gate_a2b/L3_rms": 0.3430213, "gate_a2b/L5_absmean": 0.51620483, "gate_a2b/L5_absmax": 0.67547297, "gate_a2b/L5_rms": 0.5218063, "gate_c2b/L1_absmean": 0.01457042, "gate_c2b/L1_absmax": 0.43198848, "gate_c2b/L1_rms": 0.05316046, "gate_c2b/L3_absmean": 0.20952547, "gate_c2b/L3_absmax": 0.43406621, "gate_c2b/L3_rms": 0.21355416, "gate_c2b/L5_absmean": 0.53029227, "gate_c2b/L5_absmax": 0.71685946, "gate_c2b/L5_rms": 0.54193109, "gate_b2a/L7_absmean": 0.08854626, "gate_b2a/L7_absmax": 0.27750739, "gate_b2a/L7_rms": 0.0916698, "gate_b2a/L11_absmean": 0.13482079, "gate_b2a/L11_absmax": 0.46935666, "gate_b2a/L11_rms": 0.1377728, "gate_b2a/L14_absmean": 0.32438013, "gate_b2a/L14_absmax": 0.55701733, "gate_b2a/L14_rms": 0.32588714, "gate_b2c/L3_absmean": 0.10479137, "gate_b2c/L3_absmax": 0.62670451, "gate_b2c/L3_rms": 0.12032461, "gate_b2c/L5_absmean": 0.12652579, "gate_b2c/L5_absmax": 0.77966356, "gate_b2c/L5_rms": 0.13466312, "gate_b2c/L6_absmean": 0.18526331, "gate_b2c/L6_absmax": 0.52199626, "gate_b2c/L6_rms": 0.18992877}, "streams": {"cos_a2b/round0": 0.385619, "cos_c2b/round0": 0.301761, "cos_b2a/round0": 0.120904, "cos_b2c/round0": 0.152344, "rms_a/round0": 0.831426, "rms_b/round0": 1.566988, "rms_c/round0": 1.260278, "cos_a2b/round1": 0.417887, "cos_c2b/round1": 0.276588, "cos_b2a/round1": 0.2332, "cos_b2c/round1": 0.122288, "rms_a/round1": 1.462569, "rms_b/round1": 3.096525, "rms_c/round1": 2.067698, "cos_a2b/round2": 0.545931, "cos_c2b/round2": 0.494723, "cos_b2a/round2": 0.073656, "cos_b2c/round2": 0.136877, "rms_a/round2": 3.316476, "rms_b/round2": 6.872348, "rms_c/round2": 3.327884}, "alpha_mean": 0.44027015406027914, "alpha_hist": [0.01057, 0.06509, 0.15359, 0.19659, 0.20262, 0.17934, 0.12027, 0.06357, 0.00827, 8e-05], "alpha_by_type": {"special": {"share": 0.00101, "mean": 0.55818, "std": 0.09427}, "whitespace": {"share": 0.02289, "mean": 0.50935, "std": 0.11264}, "digit": {"share": 0.02991, "mean": 0.48598, "std": 0.11196}, "word_start": {"share": 0.55719, "mean": 0.4688, "std": 0.15391}, "word_cont": {"share": 0.24901, "mean": 0.32569, "std": 0.13828}, "punct": {"share": 0.14, "mean": 0.50861, "std": 0.19163}}, "alpha_p05": 0.16776, "alpha_p50": 0.43111, "alpha_p95": 0.72759}
{"step": 24210, "total_steps": 72479, "tokens": 12693012480, "loss": 4.885557770729065, "l_mix": 3.0256770253181458, "l_a": 3.1212017834186554, "l_c": 3.0727299451828003, "z_loss": 17.010546684265137, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21260474622249603, "tokens_per_second": 1480739.0847163014, "eta_seconds": 17090.69324448109, "data_wait_seconds": 1.570582389831543e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20041, "finemath": 0.08236}}
{"step": 24220, "total_steps": 72479, "tokens": 12698255360, "loss": 4.695120632648468, "l_mix": 2.9065979421138763, "l_a": 3.0023339688777924, "l_c": 2.95378041267395, "z_loss": 16.883787631988525, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18050016462802887, "tokens_per_second": 1369559.9527416506, "eta_seconds": 18474.26579709053, "data_wait_seconds": 2.065300941467285e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20768, "cosmopedia_v2": 0.20041, "finemath": 0.08233}}
{"step": 24230, "total_steps": 72479, "tokens": 12703498240, "loss": 4.734926521778107, "l_mix": 2.9319772720336914, "l_a": 3.027379423379898, "l_c": 2.976789712905884, "z_loss": 16.982964038848877, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16035059094429016, "tokens_per_second": 1404514.0793874825, "eta_seconds": 18010.764066553114, "data_wait_seconds": 1.659989356994629e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20768, "cosmopedia_v2": 0.20041, "finemath": 0.08234}}
{"step": 24240, "total_steps": 72479, "tokens": 12708741120, "loss": 4.601813912391663, "l_mix": 2.8479342460632324, "l_a": 2.9476040303707123, "l_c": 2.893021434545517, "z_loss": 16.918756008148193, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1656302511692047, "tokens_per_second": 1457342.1802933898, "eta_seconds": 17354.28314228058, "data_wait_seconds": 2.014636993408203e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08234}}
{"step": 24250, "total_steps": 72479, "tokens": 12713984000, "loss": 4.953331828117371, "l_mix": 3.069392591714859, "l_a": 3.1542558670043945, "l_c": 3.119884669780731, "z_loss": 16.970521450042725, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1656346172094345, "tokens_per_second": 1456265.3219491101, "eta_seconds": 17363.515817403793, "data_wait_seconds": 1.755356788635254e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 24260, "total_steps": 72479, "tokens": 12719226880, "loss": 2.9909433498978615, "l_mix": 1.8489070236682892, "l_a": 1.9202639013528824, "l_c": 1.880432665348053, "z_loss": 18.273369073867798, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.3797503113746643, "tokens_per_second": 1429491.1471957925, "eta_seconds": 17685.06445219517, "data_wait_seconds": 3.129243850708008e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08236}}
{"step": 24270, "total_steps": 72479, "tokens": 12724469760, "loss": 3.4395836889743805, "l_mix": 2.115094691514969, "l_a": 2.2368733882904053, "l_c": 2.1728789806365967, "z_loss": 15.632109642028809, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.23445312678813934, "tokens_per_second": 1372592.0396570666, "eta_seconds": 18414.35726110935, "data_wait_seconds": 3.36766242980957e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20045, "finemath": 0.08232}}
{"step": 24280, "total_steps": 72479, "tokens": 12729712640, "loss": 3.427980065345764, "l_mix": 2.1073860824108124, "l_a": 2.2334084510803223, "l_c": 2.163431793451309, "z_loss": 15.418187856674194, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.187666118144989, "tokens_per_second": 1385976.2965592353, "eta_seconds": 18232.74855041504, "data_wait_seconds": 2.282857894897461e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20045, "finemath": 0.08233}}
{"step": 24290, "total_steps": 72479, "tokens": 12734955520, "loss": 4.68787145614624, "l_mix": 2.9016882181167603, "l_a": 3.0019403994083405, "l_c": 2.9463086426258087, "z_loss": 17.08448886871338, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16111724078655243, "tokens_per_second": 1393457.078840448, "eta_seconds": 18131.103437376023, "data_wait_seconds": 2.8312206268310547e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20766, "cosmopedia_v2": 0.20041, "finemath": 0.08234}}
{"step": 24300, "total_steps": 72479, "tokens": 12740198400, "loss": 4.674400806427002, "l_mix": 2.893245041370392, "l_a": 2.9912708401679993, "l_c": 2.940213054418564, "z_loss": 17.104247570037842, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1747201830148697, "tokens_per_second": 1443131.573916201, "eta_seconds": 17503.37391860485, "data_wait_seconds": 2.562999725341797e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20765, "cosmopedia_v2": 0.20041, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.31597015, "gate_a2b/L1_absmax": 0.66483414, "gate_a2b/L1_rms": 0.33127668, "gate_a2b/L3_absmean": 0.33832288, "gate_a2b/L3_absmax": 0.53724056, "gate_a2b/L3_rms": 0.34290737, "gate_a2b/L5_absmean": 0.51622647, "gate_a2b/L5_absmax": 0.67655444, "gate_a2b/L5_rms": 0.52186114, "gate_c2b/L1_absmean": 0.01730271, "gate_c2b/L1_absmax": 0.43627423, "gate_c2b/L1_rms": 0.05398976, "gate_c2b/L3_absmean": 0.20968142, "gate_c2b/L3_absmax": 0.44295126, "gate_c2b/L3_rms": 0.21375157, "gate_c2b/L5_absmean": 0.52959597, "gate_c2b/L5_absmax": 0.72112918, "gate_c2b/L5_rms": 0.54121166, "gate_b2a/L7_absmean": 0.08790629, "gate_b2a/L7_absmax": 0.27432925, "gate_b2a/L7_rms": 0.09114195, "gate_b2a/L11_absmean": 0.13628346, "gate_b2a/L11_absmax": 0.47041497, "gate_b2a/L11_rms": 0.13924044, "gate_b2a/L14_absmean": 0.32493401, "gate_b2a/L14_absmax": 0.55569571, "gate_b2a/L14_rms": 0.32641217, "gate_b2c/L3_absmean": 0.10493859, "gate_b2c/L3_absmax": 0.62233037, "gate_b2c/L3_rms": 0.12047535, "gate_b2c/L5_absmean": 0.12668979, "gate_b2c/L5_absmax": 0.76782757, "gate_b2c/L5_rms": 0.13473028, "gate_b2c/L6_absmean": 0.18564785, "gate_b2c/L6_absmax": 0.5331769, "gate_b2c/L6_rms": 0.19043694}, "streams": {"cos_a2b/round0": 0.386412, "cos_c2b/round0": 0.307416, "cos_b2a/round0": 0.117523, "cos_b2c/round0": 0.162385, "rms_a/round0": 0.82892, "rms_b/round0": 1.535618, "rms_c/round0": 1.227818, "cos_a2b/round1": 0.429358, "cos_c2b/round1": 0.279799, "cos_b2a/round1": 0.232443, "cos_b2c/round1": 0.127464, "rms_a/round1": 1.462312, "rms_b/round1": 2.982059, "rms_c/round1": 1.990026, "cos_a2b/round2": 0.565197, "cos_c2b/round2": 0.497989, "cos_b2a/round2": 0.08241, "cos_b2c/round2": 0.137161, "rms_a/round2": 3.323748, "rms_b/round2": 6.691116, "rms_c/round2": 3.201349}, "alpha_mean": 0.4585522095668324, "alpha_hist": [0.00906, 0.05623, 0.14102, 0.17648, 0.18993, 0.1972, 0.14989, 0.07082, 0.00926, 0.00011], "alpha_by_type": {"special": {"share": 0.00063, "mean": 0.53485, "std": 0.11919}, "whitespace": {"share": 0.02461, "mean": 0.50802, "std": 0.10894}, "digit": {"share": 0.02058, "mean": 0.35613, "std": 0.11447}, "word_start": {"share": 0.6107, "mean": 0.48255, "std": 0.15852}, "word_cont": {"share": 0.22754, "mean": 0.33842, "std": 0.14347}, "punct": {"share": 0.11594, "mean": 0.57519, "std": 0.15187}}, "alpha_p05": 0.17724, "alpha_p50": 0.47068, "alpha_p95": 0.71263}
{"step": 24310, "total_steps": 72479, "tokens": 12745441280, "loss": 4.667557060718536, "l_mix": 2.8893218636512756, "l_a": 2.9896903932094574, "l_c": 2.9321477115154266, "z_loss": 16.837510585784912, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2043178528547287, "tokens_per_second": 1451101.4697555774, "eta_seconds": 17403.626967763903, "data_wait_seconds": 1.9788742065429688e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 24320, "total_steps": 72479, "tokens": 12750684160, "loss": 4.551799416542053, "l_mix": 2.817099153995514, "l_a": 2.9162611961364746, "l_c": 2.8604137003421783, "z_loss": 16.977848768234253, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1964588165283203, "tokens_per_second": 1429320.185428771, "eta_seconds": 17665.171211743356, "data_wait_seconds": 1.6510486602783203e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20041, "finemath": 0.08236}}
{"step": 24330, "total_steps": 72479, "tokens": 12755927040, "loss": 3.4243004322052, "l_mix": 2.105879634618759, "l_a": 2.222541719675064, "l_c": 2.166869878768921, "z_loss": 15.972407341003418, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22188423573970795, "tokens_per_second": 1447642.2216432006, "eta_seconds": 17437.970884370807, "data_wait_seconds": 3.0368566513061523e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20045, "finemath": 0.08233}}
{"step": 24340, "total_steps": 72479, "tokens": 12761169920, "loss": 3.4363309144973755, "l_mix": 2.1132012009620667, "l_a": 2.2360506653785706, "l_c": 2.169093668460846, "z_loss": 15.863888025283813, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20542490482330322, "tokens_per_second": 1388203.0224199677, "eta_seconds": 18180.842156648636, "data_wait_seconds": 2.9027462005615234e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20764, "cosmopedia_v2": 0.20045, "finemath": 0.08233}}
{"step": 24350, "total_steps": 72479, "tokens": 12766412800, "loss": 4.7643643617630005, "l_mix": 2.9516512155532837, "l_a": 3.0392275750637054, "l_c": 2.9974557757377625, "z_loss": 17.080857276916504, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.14753784239292145, "tokens_per_second": 1271302.4772688434, "eta_seconds": 19848.50781240463, "data_wait_seconds": 2.0682811737060547e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20768, "cosmopedia_v2": 0.20041, "finemath": 0.08234}}
{"step": 24360, "total_steps": 72479, "tokens": 12771655680, "loss": 4.956896722316742, "l_mix": 3.0713863372802734, "l_a": 3.1607436537742615, "l_c": 3.1186729073524475, "z_loss": 16.854098320007324, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17037516832351685, "tokens_per_second": 1357617.5277817838, "eta_seconds": 18582.71107711792, "data_wait_seconds": 2.574920654296875e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 24370, "total_steps": 72479, "tokens": 12776898560, "loss": 4.729571759700775, "l_mix": 2.9312141835689545, "l_a": 3.0106022655963898, "l_c": 2.9782599806785583, "z_loss": 16.98831582069397, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20122426748275757, "tokens_per_second": 1417348.755406284, "eta_seconds": 17795.88213260174, "data_wait_seconds": 1.615285873413086e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08236}}
{"step": 24380, "total_steps": 72479, "tokens": 12782141440, "loss": 3.449817031621933, "l_mix": 2.121879369020462, "l_a": 2.2431987822055817, "l_c": 2.1779537200927734, "z_loss": 15.918458104133606, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20019836723804474, "tokens_per_second": 1408094.476154374, "eta_seconds": 17909.11685192585, "data_wait_seconds": 1.963973045349121e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20045, "finemath": 0.08232}}
{"step": 24390, "total_steps": 72479, "tokens": 12787384320, "loss": 3.494985044002533, "l_mix": 2.1494531333446503, "l_a": 2.2742887437343597, "l_c": 2.205439954996109, "z_loss": 16.13229751586914, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.26624494791030884, "tokens_per_second": 1422911.1610392279, "eta_seconds": 17718.945723629, "data_wait_seconds": 1.6003847122192383e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20045, "finemath": 0.08233}}
{"step": 24400, "total_steps": 72479, "tokens": 12792627200, "loss": 4.703300595283508, "l_mix": 2.9115745425224304, "l_a": 3.005840629339218, "l_c": 2.9610345363616943, "z_loss": 16.634265184402466, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19552142918109894, "tokens_per_second": 1404030.1915986123, "eta_seconds": 17953.490532350537, "data_wait_seconds": 1.531839370727539e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20766, "cosmopedia_v2": 0.20041, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.31717446, "gate_a2b/L1_absmax": 0.66662306, "gate_a2b/L1_rms": 0.33219221, "gate_a2b/L3_absmean": 0.33860907, "gate_a2b/L3_absmax": 0.53758281, "gate_a2b/L3_rms": 0.34311172, "gate_a2b/L5_absmean": 0.51616746, "gate_a2b/L5_absmax": 0.68211246, "gate_a2b/L5_rms": 0.52174973, "gate_c2b/L1_absmean": 0.01370993, "gate_c2b/L1_absmax": 0.43117848, "gate_c2b/L1_rms": 0.05335169, "gate_c2b/L3_absmean": 0.21068974, "gate_c2b/L3_absmax": 0.43947965, "gate_c2b/L3_rms": 0.21469453, "gate_c2b/L5_absmean": 0.52954787, "gate_c2b/L5_absmax": 0.72221214, "gate_c2b/L5_rms": 0.54120386, "gate_b2a/L7_absmean": 0.08832555, "gate_b2a/L7_absmax": 0.27878398, "gate_b2a/L7_rms": 0.09151836, "gate_b2a/L11_absmean": 0.13604093, "gate_b2a/L11_absmax": 0.47188857, "gate_b2a/L11_rms": 0.1389579, "gate_b2a/L14_absmean": 0.32569736, "gate_b2a/L14_absmax": 0.56334335, "gate_b2a/L14_rms": 0.32719579, "gate_b2c/L3_absmean": 0.10525247, "gate_b2c/L3_absmax": 0.62224025, "gate_b2c/L3_rms": 0.12075549, "gate_b2c/L5_absmean": 0.12653677, "gate_b2c/L5_absmax": 0.77316588, "gate_b2c/L5_rms": 0.13461731, "gate_b2c/L6_absmean": 0.18564837, "gate_b2c/L6_absmax": 0.52518141, "gate_b2c/L6_rms": 0.19037224}, "streams": {"cos_a2b/round0": 0.394411, "cos_c2b/round0": 0.307041, "cos_b2a/round0": 0.117303, "cos_b2c/round0": 0.158997, "rms_a/round0": 0.827823, "rms_b/round0": 1.535624, "rms_c/round0": 1.243995, "cos_a2b/round1": 0.427864, "cos_c2b/round1": 0.271339, "cos_b2a/round1": 0.226913, "cos_b2c/round1": 0.124145, "rms_a/round1": 1.465443, "rms_b/round1": 3.061747, "rms_c/round1": 2.037699, "cos_a2b/round2": 0.548926, "cos_c2b/round2": 0.488482, "cos_b2a/round2": 0.064157, "cos_b2c/round2": 0.140819, "rms_a/round2": 3.33045, "rms_b/round2": 6.913439, "rms_c/round2": 3.284581}, "alpha_mean": 0.44687667284996735, "alpha_hist": [0.01512, 0.0808, 0.16115, 0.17459, 0.1629, 0.16821, 0.13277, 0.08334, 0.0208, 0.00032], "alpha_by_type": {"special": {"share": 0.00096, "mean": 0.56787, "std": 0.11832}, "whitespace": {"share": 0.02609, "mean": 0.52238, "std": 0.11827}, "digit": {"share": 0.041, "mean": 0.53623, "std": 0.11637}, "word_start": {"share": 0.59901, "mean": 0.4611, "std": 0.17821}, "word_cont": {"share": 0.23318, "mean": 0.31836, "std": 0.14854}, "punct": {"share": 0.09975, "mean": 0.60422, "std": 0.16609}}, "alpha_p05": 0.16398, "alpha_p50": 0.45811, "alpha_p95": 0.76278}
{"step": 24410, "total_steps": 72479, "tokens": 12797870080, "loss": 4.732660412788391, "l_mix": 2.930072396993637, "l_a": 3.027368724346161, "l_c": 2.9756658375263214, "z_loss": 16.7762770652771, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.195156067609787, "tokens_per_second": 1430411.791328869, "eta_seconds": 17618.702547597884, "data_wait_seconds": 2.3752450942993164e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20766, "cosmopedia_v2": 0.20041, "finemath": 0.08234}}
{"step": 24420, "total_steps": 72479, "tokens": 12803112960, "loss": 4.712805926799774, "l_mix": 2.917885810136795, "l_a": 3.0148650109767914, "l_c": 2.962677240371704, "z_loss": 16.573796033859253, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18216821551322937, "tokens_per_second": 1428986.463275761, "eta_seconds": 17632.607193660737, "data_wait_seconds": 2.0891427993774414e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 24430, "total_steps": 72479, "tokens": 12808355840, "loss": 4.7124828696250916, "l_mix": 2.9180044531822205, "l_a": 3.010352075099945, "l_c": 2.965540289878845, "z_loss": 17.1061429977417, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1740320324897766, "tokens_per_second": 1387680.4781496276, "eta_seconds": 18153.684878230095, "data_wait_seconds": 1.8358230590820312e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20041, "finemath": 0.08236}}
{"step": 24440, "total_steps": 72479, "tokens": 12813598720, "loss": 3.471909314393997, "l_mix": 2.135311245918274, "l_a": 2.2579896450042725, "l_c": 2.192012459039688, "z_loss": 15.973902821540833, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18442505598068237, "tokens_per_second": 1198332.3442075439, "eta_seconds": 21017.768028831484, "data_wait_seconds": 1.9878149032592773e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20045, "finemath": 0.08232}}
{"step": 24450, "total_steps": 72479, "tokens": 12818841600, "loss": 3.4224289059638977, "l_mix": 2.102681577205658, "l_a": 2.2331707775592804, "l_c": 2.1608197391033173, "z_loss": 15.50106418132782, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2349727600812912, "tokens_per_second": 1377354.8937215016, "eta_seconds": 18282.164216923713, "data_wait_seconds": 2.9146671295166016e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20045, "finemath": 0.08233}}
{"step": 24460, "total_steps": 72479, "tokens": 12824084480, "loss": 3.4533623456954956, "l_mix": 2.1225802898406982, "l_a": 2.250584125518799, "l_c": 2.180120348930359, "z_loss": 15.706376433372498, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19687531888484955, "tokens_per_second": 1453889.1954775334, "eta_seconds": 17316.165186667444, "data_wait_seconds": 2.142786979675293e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20045, "finemath": 0.08234}}
{"step": 24470, "total_steps": 72479, "tokens": 12829327360, "loss": 3.4409964084625244, "l_mix": 2.116690158843994, "l_a": 2.2326466143131256, "l_c": 2.1765418648719788, "z_loss": 15.496958613395691, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2504409849643707, "tokens_per_second": 1392541.0461551866, "eta_seconds": 18075.260805773738, "data_wait_seconds": 2.6553869247436523e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20764, "cosmopedia_v2": 0.20045, "finemath": 0.08235}}
{"step": 24480, "total_steps": 72479, "tokens": 12834570240, "loss": 3.4856250882148743, "l_mix": 2.1429100334644318, "l_a": 2.2701070606708527, "l_c": 2.2004737555980682, "z_loss": 15.40735375881195, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.25493770837783813, "tokens_per_second": 1420392.6801990313, "eta_seconds": 17717.142634439468, "data_wait_seconds": 2.676248550415039e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20764, "cosmopedia_v2": 0.20045, "finemath": 0.08235}}
{"step": 24490, "total_steps": 72479, "tokens": 12839813120, "loss": 4.893824577331543, "l_mix": 3.030564993619919, "l_a": 3.127769857645035, "l_c": 3.0775444507598877, "z_loss": 16.651665210723877, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18019723892211914, "tokens_per_second": 1437742.489347317, "eta_seconds": 17499.696237969398, "data_wait_seconds": 3.853440284729004e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20768, "cosmopedia_v2": 0.20041, "finemath": 0.08236}}
{"step": 24500, "total_steps": 72479, "tokens": 12845056000, "loss": 3.4426863193511963, "l_mix": 2.116461604833603, "l_a": 2.2420002818107605, "l_c": 2.1735480427742004, "z_loss": 15.601520538330078, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1839267909526825, "tokens_per_second": 1357314.9369737876, "eta_seconds": 18532.77619421482, "data_wait_seconds": 1.5974044799804688e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20045, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.31414217, "gate_a2b/L1_absmax": 0.66525608, "gate_a2b/L1_rms": 0.3293137, "gate_a2b/L3_absmean": 0.33876893, "gate_a2b/L3_absmax": 0.52805203, "gate_a2b/L3_rms": 0.34326231, "gate_a2b/L5_absmean": 0.51567394, "gate_a2b/L5_absmax": 0.68164647, "gate_a2b/L5_rms": 0.52120638, "gate_c2b/L1_absmean": 0.01702288, "gate_c2b/L1_absmax": 0.42504781, "gate_c2b/L1_rms": 0.0539576, "gate_c2b/L3_absmean": 0.20997424, "gate_c2b/L3_absmax": 0.44813392, "gate_c2b/L3_rms": 0.21407029, "gate_c2b/L5_absmean": 0.52899939, "gate_c2b/L5_absmax": 0.71152467, "gate_c2b/L5_rms": 0.54065692, "gate_b2a/L7_absmean": 0.08788215, "gate_b2a/L7_absmax": 0.28940365, "gate_b2a/L7_rms": 0.09136969, "gate_b2a/L11_absmean": 0.13483492, "gate_b2a/L11_absmax": 0.48214144, "gate_b2a/L11_rms": 0.13797267, "gate_b2a/L14_absmean": 0.3246735, "gate_b2a/L14_absmax": 0.55639815, "gate_b2a/L14_rms": 0.32620591, "gate_b2c/L3_absmean": 0.10300838, "gate_b2c/L3_absmax": 0.62762266, "gate_b2c/L3_rms": 0.11915127, "gate_b2c/L5_absmean": 0.1258539, "gate_b2c/L5_absmax": 0.77752101, "gate_b2c/L5_rms": 0.13418914, "gate_b2c/L6_absmean": 0.18523562, "gate_b2c/L6_absmax": 0.52066553, "gate_b2c/L6_rms": 0.19001782}, "streams": {"cos_a2b/round0": 0.381541, "cos_c2b/round0": 0.260048, "cos_b2a/round0": 0.100786, "cos_b2c/round0": 0.113746, "rms_a/round0": 0.863842, "rms_b/round0": 1.557082, "rms_c/round0": 1.26489, "cos_a2b/round1": 0.414809, "cos_c2b/round1": 0.273702, "cos_b2a/round1": 0.221329, "cos_b2c/round1": 0.126969, "rms_a/round1": 1.511498, "rms_b/round1": 3.179596, "rms_c/round1": 2.094219, "cos_a2b/round2": 0.507946, "cos_c2b/round2": 0.48057, "cos_b2a/round2": 0.038214, "cos_b2c/round2": 0.112221, "rms_a/round2": 3.327903, "rms_b/round2": 7.090105, "rms_c/round2": 3.317824}, "alpha_mean": 0.4185236561723329, "alpha_hist": [0.06865, 0.15134, 0.16022, 0.12984, 0.11272, 0.11275, 0.10864, 0.09546, 0.05547, 0.00493], "alpha_by_type": {"special": {"share": 0.00104, "mean": 0.61221, "std": 0.1406}, "whitespace": {"share": 0.04227, "mean": 0.57867, "std": 0.14061}, "digit": {"share": 0.01482, "mean": 0.37919, "std": 0.17727}, "word_start": {"share": 0.56047, "mean": 0.44764, "std": 0.21728}, "word_cont": {"share": 0.26199, "mean": 0.23347, "std": 0.15362}, "punct": {"share": 0.11942, "mean": 0.63438, "std": 0.19462}}, "alpha_p05": 0.07586, "alpha_p50": 0.36117, "alpha_p95": 0.79975}
{"step": 24510, "total_steps": 72479, "tokens": 12850298880, "loss": 4.653382420539856, "l_mix": 2.87896865606308, "l_a": 2.9843509197235107, "l_c": 2.92463481426239, "z_loss": 17.18045210838318, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17433567345142365, "tokens_per_second": 1411209.670843799, "eta_seconds": 17821.285944676398, "data_wait_seconds": 2.4706125259399414e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08233}}
{"step": 24520, "total_steps": 72479, "tokens": 12855541760, "loss": 4.665691316127777, "l_mix": 2.888237476348877, "l_a": 2.988166183233261, "l_c": 2.9310618340969086, "z_loss": 16.854240655899048, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17853964865207672, "tokens_per_second": 1395347.5679818646, "eta_seconds": 18020.118262267115, "data_wait_seconds": 1.7315149307250977e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08234}}
{"step": 24530, "total_steps": 72479, "tokens": 12860784640, "loss": 4.731326997280121, "l_mix": 2.9288565516471863, "l_a": 3.029526025056839, "l_c": 2.973058819770813, "z_loss": 16.94907522201538, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15690898895263672, "tokens_per_second": 1418629.4040200268, "eta_seconds": 17720.685360646246, "data_wait_seconds": 1.7017126083374023e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 24540, "total_steps": 72479, "tokens": 12866027520, "loss": 4.729424595832825, "l_mix": 2.928002178668976, "l_a": 3.02681502699852, "l_c": 2.9723183512687683, "z_loss": 16.824031591415405, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15923026204109192, "tokens_per_second": 1289991.2145147643, "eta_seconds": 19483.73147754669, "data_wait_seconds": 1.755356788635254e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20041, "finemath": 0.08236}}
{"step": 24550, "total_steps": 72479, "tokens": 12871270400, "loss": 3.4552713334560394, "l_mix": 2.1237195432186127, "l_a": 2.2491333186626434, "l_c": 2.1842609345912933, "z_loss": 15.334721803665161, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2040119469165802, "tokens_per_second": 1379038.5321817093, "eta_seconds": 18221.825544095038, "data_wait_seconds": 3.090500831604004e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20045, "finemath": 0.08232}}
{"step": 24560, "total_steps": 72479, "tokens": 12876513280, "loss": 3.4638882875442505, "l_mix": 2.1297508478164673, "l_a": 2.254737675189972, "l_c": 2.1872409880161285, "z_loss": 15.437846302986145, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2521626055240631, "tokens_per_second": 1310738.281504981, "eta_seconds": 19167.33266015053, "data_wait_seconds": 2.03549861907959e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20045, "finemath": 0.08233}}
{"step": 24570, "total_steps": 72479, "tokens": 12881756160, "loss": 3.4543936252593994, "l_mix": 2.1241844594478607, "l_a": 2.2478102147579193, "l_c": 2.1809907853603363, "z_loss": 15.6887047290802, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18313586711883545, "tokens_per_second": 1436935.5680409567, "eta_seconds": 17480.33408780098, "data_wait_seconds": 2.0116567611694336e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20045, "finemath": 0.08234}}
{"step": 24580, "total_steps": 72479, "tokens": 12886999040, "loss": 3.4767750799655914, "l_mix": 2.1382173001766205, "l_a": 2.258063018321991, "l_c": 2.1984913647174835, "z_loss": 15.913089394569397, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.24355833232402802, "tokens_per_second": 1466356.48046159, "eta_seconds": 17126.033980560303, "data_wait_seconds": 1.9729137420654297e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20045, "finemath": 0.08234}}
{"step": 24590, "total_steps": 72479, "tokens": 12892241920, "loss": 3.473106026649475, "l_mix": 2.136935919523239, "l_a": 2.2558175325393677, "l_c": 2.1927017271518707, "z_loss": 16.142719507217407, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.24920284748077393, "tokens_per_second": 1457666.9591373673, "eta_seconds": 17224.529838323593, "data_wait_seconds": 1.919269561767578e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20045, "finemath": 0.08235}}
{"step": 24600, "total_steps": 72479, "tokens": 12897484800, "loss": 3.464847892522812, "l_mix": 2.130390018224716, "l_a": 2.257388323545456, "l_c": 2.185652434825897, "z_loss": 15.456442832946777, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.23980295658111572, "tokens_per_second": 1453657.092563198, "eta_seconds": 17268.436469936372, "data_wait_seconds": 1.6897916793823242e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20764, "cosmopedia_v2": 0.20045, "finemath": 0.08236}, "gates": {"gate_a2b/L1_absmean": 0.31238678, "gate_a2b/L1_absmax": 0.63930249, "gate_a2b/L1_rms": 0.32696146, "gate_a2b/L3_absmean": 0.33695087, "gate_a2b/L3_absmax": 0.53349447, "gate_a2b/L3_rms": 0.34158051, "gate_a2b/L5_absmean": 0.51553917, "gate_a2b/L5_absmax": 0.6804015, "gate_a2b/L5_rms": 0.52109557, "gate_c2b/L1_absmean": 0.01389088, "gate_c2b/L1_absmax": 0.42929959, "gate_c2b/L1_rms": 0.05333397, "gate_c2b/L3_absmean": 0.2108321, "gate_c2b/L3_absmax": 0.43338448, "gate_c2b/L3_rms": 0.21477978, "gate_c2b/L5_absmean": 0.52971375, "gate_c2b/L5_absmax": 0.71431154, "gate_c2b/L5_rms": 0.54142582, "gate_b2a/L7_absmean": 0.08940361, "gate_b2a/L7_absmax": 0.28490555, "gate_b2a/L7_rms": 0.09258007, "gate_b2a/L11_absmean": 0.13542481, "gate_b2a/L11_absmax": 0.47756279, "gate_b2a/L11_rms": 0.13840106, "gate_b2a/L14_absmean": 0.32536164, "gate_b2a/L14_absmax": 0.5515151, "gate_b2a/L14_rms": 0.3268584, "gate_b2c/L3_absmean": 0.10466196, "gate_b2c/L3_absmax": 0.63016516, "gate_b2c/L3_rms": 0.12020228, "gate_b2c/L5_absmean": 0.12644091, "gate_b2c/L5_absmax": 0.76791978, "gate_b2c/L5_rms": 0.13443337, "gate_b2c/L6_absmean": 0.1853016, "gate_b2c/L6_absmax": 0.51871616, "gate_b2c/L6_rms": 0.19003627}, "streams": {"cos_a2b/round0": 0.377805, "cos_c2b/round0": 0.257634, "cos_b2a/round0": 0.101788, "cos_b2c/round0": 0.113892, "rms_a/round0": 0.855395, "rms_b/round0": 1.536163, "rms_c/round0": 1.250791, "cos_a2b/round1": 0.409749, "cos_c2b/round1": 0.271877, "cos_b2a/round1": 0.215539, "cos_b2c/round1": 0.12924, "rms_a/round1": 1.513586, "rms_b/round1": 3.178012, "rms_c/round1": 2.105133, "cos_a2b/round2": 0.51044, "cos_c2b/round2": 0.471034, "cos_b2a/round2": 0.033798, "cos_b2c/round2": 0.109534, "rms_a/round2": 3.347343, "rms_b/round2": 7.080482, "rms_c/round2": 3.371949}, "alpha_mean": 0.43897562550036895, "alpha_hist": [0.04597, 0.13762, 0.16862, 0.13921, 0.11401, 0.10773, 0.1019, 0.09998, 0.07516, 0.0098], "alpha_by_type": {"special": {"share": 0.00108, "mean": 0.64422, "std": 0.11535}, "whitespace": {"share": 0.03703, "mean": 0.64276, "std": 0.17897}, "digit": {"share": 0.01083, "mean": 0.39476, "std": 0.17658}, "word_start": {"share": 0.57195, "mean": 0.45756, "std": 0.22125}, "word_cont": {"share": 0.26259, "mean": 0.27427, "std": 0.16172}, "punct": {"share": 0.11652, "mean": 0.65638, "std": 0.20007}}, "alpha_p05": 0.10559, "alpha_p50": 0.41324, "alpha_p95": 0.83251}
{"step": 24610, "total_steps": 72479, "tokens": 12902727680, "loss": 4.7055458426475525, "l_mix": 2.913262754678726, "l_a": 3.0104404985904694, "l_c": 2.9582491517066956, "z_loss": 16.761348962783813, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21598811447620392, "tokens_per_second": 1405781.6044215872, "eta_seconds": 17852.803161644933, "data_wait_seconds": 1.6957521438598633e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20764, "cosmopedia_v2": 0.20045, "finemath": 0.08232}}
{"step": 24620, "total_steps": 72479, "tokens": 12907970560, "loss": 4.660714030265808, "l_mix": 2.8855157792568207, "l_a": 2.9792646765708923, "l_c": 2.932437300682068, "z_loss": 16.877334594726562, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21723529696464539, "tokens_per_second": 1422117.2104188676, "eta_seconds": 17644.044533157346, "data_wait_seconds": 1.9341707229614258e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20764, "cosmopedia_v2": 0.20045, "finemath": 0.08233}}
{"step": 24630, "total_steps": 72479, "tokens": 12913213440, "loss": 4.682612895965576, "l_mix": 2.898266553878784, "l_a": 2.998488664627075, "l_c": 2.9437185525894165, "z_loss": 16.840298175811768, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1849188655614853, "tokens_per_second": 1368854.145216895, "eta_seconds": 18326.75643322468, "data_wait_seconds": 3.254413604736328e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08234}}
{"step": 24640, "total_steps": 72479, "tokens": 12918456320, "loss": 4.814543843269348, "l_mix": 2.9819943010807037, "l_a": 3.0767790377140045, "l_c": 3.0262741446495056, "z_loss": 16.335540890693665, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1852133870124817, "tokens_per_second": 1376633.361058941, "eta_seconds": 18219.38530728817, "data_wait_seconds": 1.7821788787841797e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 24650, "total_steps": 72479, "tokens": 12923699200, "loss": 4.656428277492523, "l_mix": 2.8825261294841766, "l_a": 2.9822202622890472, "l_c": 2.9251670241355896, "z_loss": 16.859394073486328, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1577358990907669, "tokens_per_second": 1433537.9371809328, "eta_seconds": 17492.50584976673, "data_wait_seconds": 1.7523765563964844e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 24660, "total_steps": 72479, "tokens": 12928942080, "loss": 3.4787419736385345, "l_mix": 2.1388683915138245, "l_a": 2.259748935699463, "l_c": 2.201445072889328, "z_loss": 15.153581738471985, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.25274327397346497, "tokens_per_second": 1422299.9760960406, "eta_seconds": 17627.03247792721, "data_wait_seconds": 1.7136335372924805e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20045, "finemath": 0.08232}}
{"step": 24670, "total_steps": 72479, "tokens": 12934184960, "loss": 3.434571146965027, "l_mix": 2.1118712574243546, "l_a": 2.237482190132141, "l_c": 2.166221797466278, "z_loss": 15.886847138404846, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2318815439939499, "tokens_per_second": 1270681.145817527, "eta_seconds": 19726.179989767075, "data_wait_seconds": 2.1457672119140625e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20045, "finemath": 0.08233}}
{"step": 24680, "total_steps": 72479, "tokens": 12939427840, "loss": 3.461372673511505, "l_mix": 2.129029154777527, "l_a": 2.252744108438492, "l_c": 2.1831314265727997, "z_loss": 15.808066964149475, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22296373546123505, "tokens_per_second": 1359961.701033733, "eta_seconds": 18427.314602279665, "data_wait_seconds": 1.996755599975586e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20045, "finemath": 0.08233}}
{"step": 24690, "total_steps": 72479, "tokens": 12944670720, "loss": 3.4108594954013824, "l_mix": 2.097385823726654, "l_a": 2.2196940779685974, "l_c": 2.153332829475403, "z_loss": 15.656280636787415, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1746876984834671, "tokens_per_second": 1303288.9328126397, "eta_seconds": 19224.592951869963, "data_wait_seconds": 3.56137752532959e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20045, "finemath": 0.08234}}
{"step": 24700, "total_steps": 72479, "tokens": 12949913600, "loss": 3.425294667482376, "l_mix": 2.1059064865112305, "l_a": 2.2304298281669617, "l_c": 2.1623249650001526, "z_loss": 15.616854906082153, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19428418576717377, "tokens_per_second": 1415886.4224830342, "eta_seconds": 17692.066223835944, "data_wait_seconds": 1.7702579498291016e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20765, "cosmopedia_v2": 0.20045, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.31488806, "gate_a2b/L1_absmax": 0.64889747, "gate_a2b/L1_rms": 0.32957962, "gate_a2b/L3_absmean": 0.33753583, "gate_a2b/L3_absmax": 0.53464121, "gate_a2b/L3_rms": 0.34205627, "gate_a2b/L5_absmean": 0.51535171, "gate_a2b/L5_absmax": 0.6798113, "gate_a2b/L5_rms": 0.52093464, "gate_c2b/L1_absmean": 0.01503453, "gate_c2b/L1_absmax": 0.42718771, "gate_c2b/L1_rms": 0.05362643, "gate_c2b/L3_absmean": 0.21051505, "gate_c2b/L3_absmax": 0.43805936, "gate_c2b/L3_rms": 0.21439445, "gate_c2b/L5_absmean": 0.52995211, "gate_c2b/L5_absmax": 0.70712447, "gate_c2b/L5_rms": 0.54173338, "gate_b2a/L7_absmean": 0.08923542, "gate_b2a/L7_absmax": 0.28163791, "gate_b2a/L7_rms": 0.09239172, "gate_b2a/L11_absmean": 0.13564244, "gate_b2a/L11_absmax": 0.47135696, "gate_b2a/L11_rms": 0.13853964, "gate_b2a/L14_absmean": 0.32552284, "gate_b2a/L14_absmax": 0.56659889, "gate_b2a/L14_rms": 0.32704654, "gate_b2c/L3_absmean": 0.1053593, "gate_b2c/L3_absmax": 0.63295656, "gate_b2c/L3_rms": 0.12094494, "gate_b2c/L5_absmean": 0.12660895, "gate_b2c/L5_absmax": 0.76852196, "gate_b2c/L5_rms": 0.13469058, "gate_b2c/L6_absmean": 0.18567789, "gate_b2c/L6_absmax": 0.52016079, "gate_b2c/L6_rms": 0.19034958}, "streams": {"cos_a2b/round0": 0.379254, "cos_c2b/round0": 0.260639, "cos_b2a/round0": 0.100564, "cos_b2c/round0": 0.11236, "rms_a/round0": 0.850497, "rms_b/round0": 1.541982, "rms_c/round0": 1.256029, "cos_a2b/round1": 0.412628, "cos_c2b/round1": 0.270103, "cos_b2a/round1": 0.214514, "cos_b2c/round1": 0.127461, "rms_a/round1": 1.523821, "rms_b/round1": 3.198966, "rms_c/round1": 2.116488, "cos_a2b/round2": 0.501529, "cos_c2b/round2": 0.480231, "cos_b2a/round2": 0.030344, "cos_b2c/round2": 0.106482, "rms_a/round2": 3.322488, "rms_b/round2": 7.0685, "rms_c/round2": 3.381457}, "alpha_mean": 0.4132835255061451, "alpha_hist": [0.05086, 0.15729, 0.18553, 0.1505, 0.11154, 0.09406, 0.09235, 0.08966, 0.0602, 0.00803], "alpha_by_type": {"special": {"share": 0.00105, "mean": 0.54716, "std": 0.1326}, "whitespace": {"share": 0.03699, "mean": 0.60146, "std": 0.19861}, "digit": {"share": 0.01302, "mean": 0.43297, "std": 0.18569}, "word_start": {"share": 0.55946, "mean": 0.42772, "std": 0.21787}, "word_cont": {"share": 0.26837, "mean": 0.25607, "std": 0.14841}, "punct": {"share": 0.12111, "mean": 0.63419, "std": 0.20236}}, "alpha_p05": 0.09335, "alpha_p50": 0.35443, "alpha_p95": 0.81523}
{"step": 24710, "total_steps": 72479, "tokens": 12955156480, "loss": 3.4810840785503387, "l_mix": 2.1402427554130554, "l_a": 2.2653013169765472, "l_c": 2.1990315914154053, "z_loss": 15.4134920835495, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2684764862060547, "tokens_per_second": 1332934.5213309396, "eta_seconds": 18789.155109429357, "data_wait_seconds": 2.053380012512207e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20765, "cosmopedia_v2": 0.20045, "finemath": 0.08236}}
{"step": 24720, "total_steps": 72479, "tokens": 12960399360, "loss": 4.72160416841507, "l_mix": 2.9236875474452972, "l_a": 3.017475128173828, "l_c": 2.969993978738785, "z_loss": 16.757036685943604, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2056877464056015, "tokens_per_second": 1453887.2729958205, "eta_seconds": 17222.429178023336, "data_wait_seconds": 1.627206802368164e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08232}}
{"step": 24730, "total_steps": 72479, "tokens": 12965642240, "loss": 4.692011892795563, "l_mix": 2.9050232768058777, "l_a": 2.997801661491394, "l_c": 2.9533223807811737, "z_loss": 16.512218713760376, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21600550413131714, "tokens_per_second": 1459732.813671292, "eta_seconds": 17149.86981010437, "data_wait_seconds": 1.5348196029663086e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 24740, "total_steps": 72479, "tokens": 12970885120, "loss": 4.662123262882233, "l_mix": 2.885916590690613, "l_a": 2.9849792420864105, "l_c": 2.930152177810669, "z_loss": 16.670581340789795, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20642131567001343, "tokens_per_second": 1439402.581296094, "eta_seconds": 17388.453485655784, "data_wait_seconds": 1.5079975128173828e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 24750, "total_steps": 72479, "tokens": 12976128000, "loss": 4.6381213665008545, "l_mix": 2.871561586856842, "l_a": 2.9698640406131744, "l_c": 2.913130223751068, "z_loss": 16.614187955856323, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1812487691640854, "tokens_per_second": 1326436.692815108, "eta_seconds": 18865.387309885027, "data_wait_seconds": 1.7017126083374023e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 24760, "total_steps": 72479, "tokens": 12981370880, "loss": 4.7097092270851135, "l_mix": 2.9163995385169983, "l_a": 3.011181026697159, "l_c": 2.960899442434311, "z_loss": 16.854028701782227, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16025781631469727, "tokens_per_second": 1456938.002909605, "eta_seconds": 17171.97232966423, "data_wait_seconds": 2.2798776626586914e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.2004, "finemath": 0.08235}}
{"step": 24770, "total_steps": 72479, "tokens": 12986613760, "loss": 4.829468131065369, "l_mix": 2.992902636528015, "l_a": 3.077876329421997, "l_c": 3.0382450819015503, "z_loss": 17.290127754211426, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1733560711145401, "tokens_per_second": 1451853.44764143, "eta_seconds": 17228.499358963967, "data_wait_seconds": 2.962350845336914e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.2004, "finemath": 0.08236}}
{"step": 24780, "total_steps": 72479, "tokens": 12991856640, "loss": 3.4188972413539886, "l_mix": 2.102316230535507, "l_a": 2.223310202360153, "l_c": 2.1599954068660736, "z_loss": 15.892338514328003, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20509028434753418, "tokens_per_second": 1367586.1355919638, "eta_seconds": 18286.24366769791, "data_wait_seconds": 2.2202730178833008e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08232}}
{"step": 24790, "total_steps": 72479, "tokens": 12997099520, "loss": 3.46771502494812, "l_mix": 2.132865756750107, "l_a": 2.254546672105789, "l_c": 2.189720541238785, "z_loss": 15.690499663352966, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.24898189306259155, "tokens_per_second": 1373534.1663179256, "eta_seconds": 18203.238801860807, "data_wait_seconds": 4.464387893676758e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 24800, "total_steps": 72479, "tokens": 13002342400, "loss": 3.3850430250167847, "l_mix": 2.0810614228248596, "l_a": 2.206065148115158, "l_c": 2.1352821588516235, "z_loss": 15.773761630058289, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1925705373287201, "tokens_per_second": 1327434.6052517442, "eta_seconds": 18831.456896710395, "data_wait_seconds": 2.709031105041504e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.31636786, "gate_a2b/L1_absmax": 0.65600121, "gate_a2b/L1_rms": 0.3309958, "gate_a2b/L3_absmean": 0.33823517, "gate_a2b/L3_absmax": 0.53677523, "gate_a2b/L3_rms": 0.34273905, "gate_a2b/L5_absmean": 0.51571935, "gate_a2b/L5_absmax": 0.67577702, "gate_a2b/L5_rms": 0.52133602, "gate_c2b/L1_absmean": 0.01521142, "gate_c2b/L1_absmax": 0.42962205, "gate_c2b/L1_rms": 0.05362949, "gate_c2b/L3_absmean": 0.20989679, "gate_c2b/L3_absmax": 0.43385115, "gate_c2b/L3_rms": 0.21390279, "gate_c2b/L5_absmean": 0.53002542, "gate_c2b/L5_absmax": 0.71969968, "gate_c2b/L5_rms": 0.54167432, "gate_b2a/L7_absmean": 0.08900615, "gate_b2a/L7_absmax": 0.28071812, "gate_b2a/L7_rms": 0.09222296, "gate_b2a/L11_absmean": 0.13625011, "gate_b2a/L11_absmax": 0.47448173, "gate_b2a/L11_rms": 0.13918126, "gate_b2a/L14_absmean": 0.325452, "gate_b2a/L14_absmax": 0.56879747, "gate_b2a/L14_rms": 0.32703203, "gate_b2c/L3_absmean": 0.10560428, "gate_b2c/L3_absmax": 0.62868631, "gate_b2c/L3_rms": 0.1212621, "gate_b2c/L5_absmean": 0.12713556, "gate_b2c/L5_absmax": 0.76798654, "gate_b2c/L5_rms": 0.13528883, "gate_b2c/L6_absmean": 0.18594182, "gate_b2c/L6_absmax": 0.52767372, "gate_b2c/L6_rms": 0.19073826}, "streams": {"cos_a2b/round0": 0.38295, "cos_c2b/round0": 0.266814, "cos_b2a/round0": 0.103768, "cos_b2c/round0": 0.114611, "rms_a/round0": 0.865119, "rms_b/round0": 1.555964, "rms_c/round0": 1.25324, "cos_a2b/round1": 0.418252, "cos_c2b/round1": 0.266167, "cos_b2a/round1": 0.21603, "cos_b2c/round1": 0.129799, "rms_a/round1": 1.539232, "rms_b/round1": 3.18897, "rms_c/round1": 2.112998, "cos_a2b/round2": 0.5131, "cos_c2b/round2": 0.47263, "cos_b2a/round2": 0.032441, "cos_b2c/round2": 0.11332, "rms_a/round2": 3.36856, "rms_b/round2": 7.079897, "rms_c/round2": 3.405402}, "alpha_mean": 0.43507160208930884, "alpha_hist": [0.05452, 0.13824, 0.158, 0.13992, 0.11499, 0.10652, 0.11165, 0.10466, 0.06126, 0.01022], "alpha_by_type": {"special": {"share": 0.00108, "mean": 0.67557, "std": 0.11084}, "whitespace": {"share": 0.03773, "mean": 0.67516, "std": 0.17503}, "digit": {"share": 0.01404, "mean": 0.48698, "std": 0.18396}, "word_start": {"share": 0.55537, "mean": 0.44994, "std": 0.21469}, "word_cont": {"share": 0.26714, "mean": 0.26308, "std": 0.16308}, "punct": {"share": 0.12463, "mean": 0.65682, "std": 0.18554}}, "alpha_p05": 0.10088, "alpha_p50": 0.42704, "alpha_p95": 0.81903}
{"step": 24810, "total_steps": 72479, "tokens": 13007585280, "loss": 3.4327173829078674, "l_mix": 2.1100058555603027, "l_a": 2.2380325496196747, "l_c": 2.1656993627548218, "z_loss": 15.918787837028503, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2019881159067154, "tokens_per_second": 1411604.1852061595, "eta_seconds": 17704.88139233589, "data_wait_seconds": 3.775954246520996e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 24820, "total_steps": 72479, "tokens": 13012828160, "loss": 3.4220612943172455, "l_mix": 2.104903519153595, "l_a": 2.2233870923519135, "l_c": 2.1618120074272156, "z_loss": 15.979817509651184, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21870875358581543, "tokens_per_second": 1387489.4294500635, "eta_seconds": 18008.81596763134, "data_wait_seconds": 2.0056962966918945e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 24830, "total_steps": 72479, "tokens": 13018071040, "loss": 4.6824856996536255, "l_mix": 2.898706316947937, "l_a": 2.9947050511837006, "l_c": 2.9456635415554047, "z_loss": 16.687901258468628, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1916259080171585, "tokens_per_second": 1301031.830718794, "eta_seconds": 19201.527835178375, "data_wait_seconds": 2.154707908630371e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08232}}
{"step": 24840, "total_steps": 72479, "tokens": 13023313920, "loss": 4.664780974388123, "l_mix": 2.8874564468860626, "l_a": 2.9863284826278687, "l_c": 2.932396739721298, "z_loss": 17.069300651550293, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1878342181444168, "tokens_per_second": 1425457.5307615863, "eta_seconds": 17521.781949305532, "data_wait_seconds": 1.6570091247558594e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 24850, "total_steps": 72479, "tokens": 13028556800, "loss": 4.73799741268158, "l_mix": 2.932895839214325, "l_a": 3.0337887704372406, "l_c": 2.977414131164551, "z_loss": 17.407392263412476, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19801416993141174, "tokens_per_second": 1185114.140564476, "eta_seconds": 21070.808538413046, "data_wait_seconds": 1.6450881958007812e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 24860, "total_steps": 72479, "tokens": 13033799680, "loss": 4.75379079580307, "l_mix": 2.943666994571686, "l_a": 3.0399325489997864, "l_c": 2.9882075786590576, "z_loss": 16.81769585609436, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22858057916164398, "tokens_per_second": 1436676.2754741113, "eta_seconds": 17377.658904933927, "data_wait_seconds": 2.104043960571289e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 24870, "total_steps": 72479, "tokens": 13039042560, "loss": 3.425077050924301, "l_mix": 2.1058929562568665, "l_a": 2.2300392985343933, "l_c": 2.162065476179123, "z_loss": 15.526515126228333, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2426246851682663, "tokens_per_second": 1409174.658880808, "eta_seconds": 17713.08278551102, "data_wait_seconds": 3.197789192199707e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 24880, "total_steps": 72479, "tokens": 13044285440, "loss": 3.475344955921173, "l_mix": 2.1376506686210632, "l_a": 2.2658622562885284, "l_c": 2.1877499520778656, "z_loss": 16.105836629867554, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2986215651035309, "tokens_per_second": 1306040.2061039123, "eta_seconds": 19107.822558116914, "data_wait_seconds": 3.4809112548828125e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08236}}
{"step": 24890, "total_steps": 72479, "tokens": 13049528320, "loss": 4.94689404964447, "l_mix": 3.0654601752758026, "l_a": 3.1553662419319153, "l_c": 3.110434502363205, "z_loss": 16.93496561050415, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19311025738716125, "tokens_per_second": 1286263.2952254808, "eta_seconds": 19397.53837695122, "data_wait_seconds": 1.895427703857422e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08232}}
{"step": 24900, "total_steps": 72479, "tokens": 13054771200, "loss": 3.445999890565872, "l_mix": 2.118877112865448, "l_a": 2.238394170999527, "l_c": 2.1802621483802795, "z_loss": 15.257874727249146, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2415660172700882, "tokens_per_second": 1466629.826047804, "eta_seconds": 17008.449104857445, "data_wait_seconds": 2.9921531677246094e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.31553221, "gate_a2b/L1_absmax": 0.65966105, "gate_a2b/L1_rms": 0.33028537, "gate_a2b/L3_absmean": 0.33812481, "gate_a2b/L3_absmax": 0.53571081, "gate_a2b/L3_rms": 0.34263316, "gate_a2b/L5_absmean": 0.51585048, "gate_a2b/L5_absmax": 0.67261851, "gate_a2b/L5_rms": 0.52149588, "gate_c2b/L1_absmean": 0.01459071, "gate_c2b/L1_absmax": 0.42995974, "gate_c2b/L1_rms": 0.05339657, "gate_c2b/L3_absmean": 0.20995429, "gate_c2b/L3_absmax": 0.44016811, "gate_c2b/L3_rms": 0.21390837, "gate_c2b/L5_absmean": 0.53001374, "gate_c2b/L5_absmax": 0.71589267, "gate_c2b/L5_rms": 0.54177737, "gate_b2a/L7_absmean": 0.08918849, "gate_b2a/L7_absmax": 0.2790888, "gate_b2a/L7_rms": 0.0923558, "gate_b2a/L11_absmean": 0.13526416, "gate_b2a/L11_absmax": 0.47402292, "gate_b2a/L11_rms": 0.13823715, "gate_b2a/L14_absmean": 0.32590339, "gate_b2a/L14_absmax": 0.57296306, "gate_b2a/L14_rms": 0.32749212, "gate_b2c/L3_absmean": 0.10445924, "gate_b2c/L3_absmax": 0.63384348, "gate_b2c/L3_rms": 0.12015975, "gate_b2c/L5_absmean": 0.1260421, "gate_b2c/L5_absmax": 0.77235556, "gate_b2c/L5_rms": 0.13426015, "gate_b2c/L6_absmean": 0.18567948, "gate_b2c/L6_absmax": 0.52869934, "gate_b2c/L6_rms": 0.19049083}, "streams": {"cos_a2b/round0": 0.378403, "cos_c2b/round0": 0.263535, "cos_b2a/round0": 0.104585, "cos_b2c/round0": 0.112065, "rms_a/round0": 0.862104, "rms_b/round0": 1.542085, "rms_c/round0": 1.259764, "cos_a2b/round1": 0.417075, "cos_c2b/round1": 0.275733, "cos_b2a/round1": 0.219526, "cos_b2c/round1": 0.131257, "rms_a/round1": 1.527434, "rms_b/round1": 3.208932, "rms_c/round1": 2.121, "cos_a2b/round2": 0.508943, "cos_c2b/round2": 0.476834, "cos_b2a/round2": 0.028725, "cos_b2c/round2": 0.111583, "rms_a/round2": 3.363702, "rms_b/round2": 7.118044, "rms_c/round2": 3.407282}, "alpha_mean": 0.4276897265138828, "alpha_hist": [0.04755, 0.13991, 0.16667, 0.1564, 0.11902, 0.10332, 0.10304, 0.09723, 0.05862, 0.00824], "alpha_by_type": {"special": {"share": 0.00105, "mean": 0.66307, "std": 0.08858}, "whitespace": {"share": 0.03654, "mean": 0.64576, "std": 0.19142}, "digit": {"share": 0.00989, "mean": 0.35928, "std": 0.18178}, "word_start": {"share": 0.57411, "mean": 0.44873, "std": 0.20867}, "word_cont": {"share": 0.26021, "mean": 0.25755, "std": 0.15395}, "punct": {"share": 0.11819, "mean": 0.63626, "std": 0.20129}}, "alpha_p05": 0.11556, "alpha_p50": 0.38445, "alpha_p95": 0.81989}
{"step": 24910, "total_steps": 72479, "tokens": 13060014080, "loss": 3.4373379051685333, "l_mix": 2.1138960123062134, "l_a": 2.235701769590378, "l_c": 2.170639991760254, "z_loss": 15.39259159564972, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.192485511302948, "tokens_per_second": 1411701.05838712, "eta_seconds": 17666.52771408558, "data_wait_seconds": 1.6689300537109375e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 24920, "total_steps": 72479, "tokens": 13065256960, "loss": 3.4877012073993683, "l_mix": 2.144736021757126, "l_a": 2.271585762500763, "l_c": 2.199645847082138, "z_loss": 15.95661449432373, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2549362778663635, "tokens_per_second": 1398351.33358447, "eta_seconds": 17831.43648748398, "data_wait_seconds": 2.1725893020629883e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 24930, "total_steps": 72479, "tokens": 13070499840, "loss": 3.4543986916542053, "l_mix": 2.1241878271102905, "l_a": 2.243692010641098, "l_c": 2.185176730155945, "z_loss": 15.501676440238953, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.25230395793914795, "tokens_per_second": 1448734.9927083086, "eta_seconds": 17207.681347846985, "data_wait_seconds": 2.5451183319091797e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 24940, "total_steps": 72479, "tokens": 13075742720, "loss": 4.584781229496002, "l_mix": 2.838388890028, "l_a": 2.932457387447357, "l_c": 2.883261740207672, "z_loss": 16.764809608459473, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1763344705104828, "tokens_per_second": 1424628.802315765, "eta_seconds": 17495.172912049293, "data_wait_seconds": 1.6063451766967773e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08232}}
{"step": 24950, "total_steps": 72479, "tokens": 13080985600, "loss": 4.64436936378479, "l_mix": 2.8751451075077057, "l_a": 2.9705627858638763, "l_c": 2.9213425517082214, "z_loss": 16.52449607849121, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19406114518642426, "tokens_per_second": 1267326.86968041, "eta_seconds": 19662.555058336256, "data_wait_seconds": 2.014636993408203e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08232}}
{"step": 24960, "total_steps": 72479, "tokens": 13086228480, "loss": 4.736889898777008, "l_mix": 2.9320699870586395, "l_a": 3.0364990830421448, "l_c": 2.9739095270633698, "z_loss": 16.972306728363037, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21351191401481628, "tokens_per_second": 1459814.4067434082, "eta_seconds": 17066.307440805434, "data_wait_seconds": 2.8371810913085938e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 24970, "total_steps": 72479, "tokens": 13091471360, "loss": 4.738186597824097, "l_mix": 2.9336326122283936, "l_a": 3.0301076471805573, "l_c": 2.979609966278076, "z_loss": 16.386029481887817, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20647494494915009, "tokens_per_second": 1315684.986315585, "eta_seconds": 18931.886318588255, "data_wait_seconds": 2.0682811737060547e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 24980, "total_steps": 72479, "tokens": 13096714240, "loss": 3.4880013167858124, "l_mix": 2.144099324941635, "l_a": 2.270101845264435, "l_c": 2.204446792602539, "z_loss": 15.37470531463623, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19197027385234833, "tokens_per_second": 1209845.6159526277, "eta_seconds": 20583.746705889702, "data_wait_seconds": 1.9252300262451172e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 24990, "total_steps": 72479, "tokens": 13101957120, "loss": 3.4117022454738617, "l_mix": 2.0975160002708435, "l_a": 2.2236128449440002, "l_c": 2.1517706513404846, "z_loss": 15.711416363716125, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22345180809497833, "tokens_per_second": 1451627.9353580216, "eta_seconds": 17151.71789240837, "data_wait_seconds": 3.8564205169677734e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 25000, "total_steps": 72479, "tokens": 13107200000, "loss": 4.9307942390441895, "l_mix": 3.056105315685272, "l_a": 3.1408064365386963, "l_c": 3.1026505529880524, "z_loss": 16.51790738105774, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17167574167251587, "tokens_per_second": 1247162.9359753928, "eta_seconds": 19959.436921954155, "data_wait_seconds": 4.8041343688964844e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.20044, "finemath": 0.08232}, "gates": {"gate_a2b/L1_absmean": 0.31585762, "gate_a2b/L1_absmax": 0.66347092, "gate_a2b/L1_rms": 0.33078468, "gate_a2b/L3_absmean": 0.33722922, "gate_a2b/L3_absmax": 0.5383215, "gate_a2b/L3_rms": 0.34173048, "gate_a2b/L5_absmean": 0.51462138, "gate_a2b/L5_absmax": 0.67549235, "gate_a2b/L5_rms": 0.52022618, "gate_c2b/L1_absmean": 0.01516892, "gate_c2b/L1_absmax": 0.43158561, "gate_c2b/L1_rms": 0.05371859, "gate_c2b/L3_absmean": 0.21077934, "gate_c2b/L3_absmax": 0.44672406, "gate_c2b/L3_rms": 0.21473618, "gate_c2b/L5_absmean": 0.52896619, "gate_c2b/L5_absmax": 0.7093206, "gate_c2b/L5_rms": 0.54063094, "gate_b2a/L7_absmean": 0.08894083, "gate_b2a/L7_absmax": 0.28083745, "gate_b2a/L7_rms": 0.09214251, "gate_b2a/L11_absmean": 0.13567993, "gate_b2a/L11_absmax": 0.47472981, "gate_b2a/L11_rms": 0.13866128, "gate_b2a/L14_absmean": 0.32528222, "gate_b2a/L14_absmax": 0.57738763, "gate_b2a/L14_rms": 0.32694116, "gate_b2c/L3_absmean": 0.10506447, "gate_b2c/L3_absmax": 0.63164902, "gate_b2c/L3_rms": 0.12044074, "gate_b2c/L5_absmean": 0.12632322, "gate_b2c/L5_absmax": 0.77225375, "gate_b2c/L5_rms": 0.13438722, "gate_b2c/L6_absmean": 0.18601975, "gate_b2c/L6_absmax": 0.53150159, "gate_b2c/L6_rms": 0.19083244}, "streams": {"cos_a2b/round0": 0.388297, "cos_c2b/round0": 0.299771, "cos_b2a/round0": 0.115418, "cos_b2c/round0": 0.154904, "rms_a/round0": 0.836223, "rms_b/round0": 1.560974, "rms_c/round0": 1.239268, "cos_a2b/round1": 0.413055, "cos_c2b/round1": 0.276597, "cos_b2a/round1": 0.236074, "cos_b2c/round1": 0.133218, "rms_a/round1": 1.458921, "rms_b/round1": 3.063761, "rms_c/round1": 2.048516, "cos_a2b/round2": 0.557866, "cos_c2b/round2": 0.4899, "cos_b2a/round2": 0.075128, "cos_b2c/round2": 0.144813, "rms_a/round2": 3.339697, "rms_b/round2": 6.787195, "rms_c/round2": 3.308215}, "alpha_mean": 0.4694479872821873, "alpha_hist": [0.00789, 0.04785, 0.12167, 0.1835, 0.19768, 0.1992, 0.14708, 0.08044, 0.01411, 0.00056], "alpha_by_type": {"special": {"share": 0.0007, "mean": 0.58045, "std": 0.16481}, "whitespace": {"share": 0.0517, "mean": 0.56862, "std": 0.12105}, "digit": {"share": 0.03146, "mean": 0.50763, "std": 0.12509}, "word_start": {"share": 0.56346, "mean": 0.48195, "std": 0.15935}, "word_cont": {"share": 0.24374, "mean": 0.35757, "std": 0.14112}, "punct": {"share": 0.10893, "mean": 0.59634, "std": 0.15603}}, "alpha_p05": 0.19378, "alpha_p50": 0.46697, "alpha_p95": 0.74242}
{"step": 25010, "total_steps": 72479, "tokens": 13112442880, "loss": 5.014615058898926, "l_mix": 3.108138680458069, "l_a": 3.1982430517673492, "l_c": 3.1510824859142303, "z_loss": 16.785917043685913, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18261781334877014, "tokens_per_second": 1376620.9512431556, "eta_seconds": 18078.6345359087, "data_wait_seconds": 3.567337989807129e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 25020, "total_steps": 72479, "tokens": 13117685760, "loss": 4.861521124839783, "l_mix": 3.012727200984955, "l_a": 3.097596287727356, "l_c": 3.0593009293079376, "z_loss": 17.24643874168396, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21581250429153442, "tokens_per_second": 1477034.4763014372, "eta_seconds": 16846.041572642327, "data_wait_seconds": 2.232193946838379e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 25030, "total_steps": 72479, "tokens": 13122928640, "loss": 4.871572494506836, "l_mix": 3.0199153423309326, "l_a": 3.1024852097034454, "l_c": 3.064080625772476, "z_loss": 16.87468695640564, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.23085269331932068, "tokens_per_second": 1412241.5810208041, "eta_seconds": 17615.216579318047, "data_wait_seconds": 1.7315149307250977e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 25040, "total_steps": 72479, "tokens": 13128171520, "loss": 3.460726737976074, "l_mix": 2.128695070743561, "l_a": 2.251026839017868, "l_c": 2.183874487876892, "z_loss": 15.612167000770569, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.28183385729789734, "tokens_per_second": 1453423.9109908552, "eta_seconds": 17112.48744699955, "data_wait_seconds": 1.6123056411743164e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 25050, "total_steps": 72479, "tokens": 13133414400, "loss": 3.480669856071472, "l_mix": 2.1403933465480804, "l_a": 2.2612538933753967, "l_c": 2.2010903358459473, "z_loss": 15.732620120048523, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22081005573272705, "tokens_per_second": 1344808.6200110053, "eta_seconds": 18490.702083539963, "data_wait_seconds": 2.714991569519043e-05, "mix": {"fineweb_hq": 0.50954, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08236}}
{"step": 25060, "total_steps": 72479, "tokens": 13138657280, "loss": 4.710137784481049, "l_mix": 2.916692316532135, "l_a": 3.013621687889099, "l_c": 2.9588799476623535, "z_loss": 16.949246168136597, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2034071534872055, "tokens_per_second": 1262653.1004104642, "eta_seconds": 19689.661922121046, "data_wait_seconds": 1.811981201171875e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08232}}
{"step": 25070, "total_steps": 72479, "tokens": 13143900160, "loss": 4.737627685070038, "l_mix": 2.933152884244919, "l_a": 3.0320886075496674, "l_c": 2.977378875017166, "z_loss": 16.34551739692688, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2233368456363678, "tokens_per_second": 1414574.956917996, "eta_seconds": 17571.334534406662, "data_wait_seconds": 3.4809112548828125e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 25080, "total_steps": 72479, "tokens": 13149143040, "loss": 4.717122733592987, "l_mix": 2.920448452234268, "l_a": 3.020553082227707, "l_c": 2.962798148393631, "z_loss": 16.68800449371338, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19928020238876343, "tokens_per_second": 1429388.286492924, "eta_seconds": 17385.567761278155, "data_wait_seconds": 1.6450881958007812e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 25090, "total_steps": 72479, "tokens": 13154385920, "loss": 3.4736259281635284, "l_mix": 2.1357372403144836, "l_a": 2.2618141174316406, "l_c": 2.1927440464496613, "z_loss": 15.212596654891968, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2078651636838913, "tokens_per_second": 1353376.5036087811, "eta_seconds": 18358.146432828904, "data_wait_seconds": 2.2649765014648438e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 25100, "total_steps": 72479, "tokens": 13159628800, "loss": 3.4498862624168396, "l_mix": 2.120458483695984, "l_a": 2.2465504109859467, "l_c": 2.1796700060367584, "z_loss": 15.616071820259094, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2267056405544281, "tokens_per_second": 1473425.1623862856, "eta_seconds": 16858.841416668893, "data_wait_seconds": 2.6017427444458008e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.31681585, "gate_a2b/L1_absmax": 0.65623784, "gate_a2b/L1_rms": 0.33153087, "gate_a2b/L3_absmean": 0.33732221, "gate_a2b/L3_absmax": 0.53518963, "gate_a2b/L3_rms": 0.3418296, "gate_a2b/L5_absmean": 0.51502723, "gate_a2b/L5_absmax": 0.67632496, "gate_a2b/L5_rms": 0.52062595, "gate_c2b/L1_absmean": 0.01404119, "gate_c2b/L1_absmax": 0.43030551, "gate_c2b/L1_rms": 0.05346278, "gate_c2b/L3_absmean": 0.21061864, "gate_c2b/L3_absmax": 0.437599, "gate_c2b/L3_rms": 0.21461621, "gate_c2b/L5_absmean": 0.52921236, "gate_c2b/L5_absmax": 0.70999521, "gate_c2b/L5_rms": 0.54094416, "gate_b2a/L7_absmean": 0.08896682, "gate_b2a/L7_absmax": 0.27863902, "gate_b2a/L7_rms": 0.09206889, "gate_b2a/L11_absmean": 0.13519853, "gate_b2a/L11_absmax": 0.47432923, "gate_b2a/L11_rms": 0.13822794, "gate_b2a/L14_absmean": 0.3254053, "gate_b2a/L14_absmax": 0.57698327, "gate_b2a/L14_rms": 0.32700801, "gate_b2c/L3_absmean": 0.10550638, "gate_b2c/L3_absmax": 0.63196301, "gate_b2c/L3_rms": 0.12080553, "gate_b2c/L5_absmean": 0.12649545, "gate_b2c/L5_absmax": 0.7762773, "gate_b2c/L5_rms": 0.13470742, "gate_b2c/L6_absmean": 0.1858308, "gate_b2c/L6_absmax": 0.52954268, "gate_b2c/L6_rms": 0.19061956}, "streams": {"cos_a2b/round0": 0.37932, "cos_c2b/round0": 0.266375, "cos_b2a/round0": 0.097289, "cos_b2c/round0": 0.112791, "rms_a/round0": 0.859788, "rms_b/round0": 1.555591, "rms_c/round0": 1.252037, "cos_a2b/round1": 0.410517, "cos_c2b/round1": 0.274004, "cos_b2a/round1": 0.215506, "cos_b2c/round1": 0.125299, "rms_a/round1": 1.521215, "rms_b/round1": 3.198314, "rms_c/round1": 2.135175, "cos_a2b/round2": 0.500909, "cos_c2b/round2": 0.483987, "cos_b2a/round2": 0.029021, "cos_b2c/round2": 0.108955, "rms_a/round2": 3.325532, "rms_b/round2": 7.074975, "rms_c/round2": 3.431245}, "alpha_mean": 0.418755794722955, "alpha_hist": [0.06407, 0.14572, 0.16322, 0.14117, 0.11836, 0.10986, 0.10263, 0.09576, 0.05278, 0.00641], "alpha_by_type": {"special": {"share": 0.00102, "mean": 0.6021, "std": 0.11167}, "whitespace": {"share": 0.03915, "mean": 0.61417, "std": 0.19927}, "digit": {"share": 0.00827, "mean": 0.38651, "std": 0.19617}, "word_start": {"share": 0.55849, "mean": 0.43672, "std": 0.21327}, "word_cont": {"share": 0.25943, "mean": 0.2486, "std": 0.15656}, "punct": {"share": 0.13364, "mean": 0.61735, "std": 0.19527}}, "alpha_p05": 0.07056, "alpha_p50": 0.39304, "alpha_p95": 0.80225}
{"step": 25110, "total_steps": 72479, "tokens": 13164871680, "loss": 3.2012746334075928, "l_mix": 1.9768928438425064, "l_a": 2.068512976169586, "l_c": 2.0065061300992966, "z_loss": 18.76069402694702, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.25517940521240234, "tokens_per_second": 1427533.6689259198, "eta_seconds": 17397.136622834205, "data_wait_seconds": 2.7865171432495117e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08236}}
{"step": 25120, "total_steps": 72479, "tokens": 13170114560, "loss": 4.878191590309143, "l_mix": 3.021701365709305, "l_a": 3.1121144890785217, "l_c": 3.070702940225601, "z_loss": 16.4496853351593, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21169088780879974, "tokens_per_second": 1353272.3123425315, "eta_seconds": 18347.937193083762, "data_wait_seconds": 2.4497509002685547e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20768, "cosmopedia_v2": 0.20044, "finemath": 0.08232}}
{"step": 25130, "total_steps": 72479, "tokens": 13175357440, "loss": 4.840791285037994, "l_mix": 2.999646633863449, "l_a": 3.084826648235321, "l_c": 3.046653836965561, "z_loss": 17.0044846534729, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19750134646892548, "tokens_per_second": 1339453.5717969977, "eta_seconds": 18533.313161945345, "data_wait_seconds": 2.086162567138672e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20768, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 25140, "total_steps": 72479, "tokens": 13180600320, "loss": 4.622387170791626, "l_mix": 2.8620440661907196, "l_a": 2.9557814598083496, "l_c": 2.90630641579628, "z_loss": 17.167542457580566, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2473386526107788, "tokens_per_second": 1460944.0798189219, "eta_seconds": 16988.514464616776, "data_wait_seconds": 2.104043960571289e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 25150, "total_steps": 72479, "tokens": 13185843200, "loss": 4.785741627216339, "l_mix": 2.9635086953639984, "l_a": 3.059885084629059, "l_c": 3.008575439453125, "z_loss": 16.948179721832275, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20295342803001404, "tokens_per_second": 1436246.330264192, "eta_seconds": 17276.999236917494, "data_wait_seconds": 1.722574234008789e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20763, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 25160, "total_steps": 72479, "tokens": 13191086080, "loss": 4.685045659542084, "l_mix": 2.900457501411438, "l_a": 2.9950420260429382, "l_c": 2.9478827714920044, "z_loss": 17.107712745666504, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18535485863685608, "tokens_per_second": 1483769.998752405, "eta_seconds": 16720.100752043723, "data_wait_seconds": 2.0295381546020508e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20763, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 25170, "total_steps": 72479, "tokens": 13196328960, "loss": 4.694210767745972, "l_mix": 2.9065478146076202, "l_a": 3.0048274397850037, "l_c": 2.9483840465545654, "z_loss": 16.993852138519287, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16068580746650696, "tokens_per_second": 1423465.0952779392, "eta_seconds": 17424.762345266343, "data_wait_seconds": 2.4437904357910156e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08232}}
{"step": 25180, "total_steps": 72479, "tokens": 13201571840, "loss": 4.749283313751221, "l_mix": 2.941845566034317, "l_a": 3.03787162899971, "l_c": 2.9809702336788177, "z_loss": 17.85218095779419, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.40854501724243164, "tokens_per_second": 1400243.0981938902, "eta_seconds": 17709.99488873482, "data_wait_seconds": 2.0384788513183594e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 25190, "total_steps": 72479, "tokens": 13206814720, "loss": 4.796226799488068, "l_mix": 2.96920382976532, "l_a": 3.0661994218826294, "l_c": 3.01812082529068, "z_loss": 17.267560958862305, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21279101073741913, "tokens_per_second": 1316131.468802747, "eta_seconds": 18837.825718545915, "data_wait_seconds": 2.7239322662353516e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 25200, "total_steps": 72479, "tokens": 13212057600, "loss": 3.434653639793396, "l_mix": 2.1116347908973694, "l_a": 2.237277626991272, "l_c": 2.1675035059452057, "z_loss": 15.845004916191101, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19827385246753693, "tokens_per_second": 1410609.8520543145, "eta_seconds": 17572.408356499673, "data_wait_seconds": 1.6301870346069336e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.31745785, "gate_a2b/L1_absmax": 0.65600151, "gate_a2b/L1_rms": 0.33211464, "gate_a2b/L3_absmean": 0.33851126, "gate_a2b/L3_absmax": 0.53494704, "gate_a2b/L3_rms": 0.34296653, "gate_a2b/L5_absmean": 0.51482511, "gate_a2b/L5_absmax": 0.6842559, "gate_a2b/L5_rms": 0.52047187, "gate_c2b/L1_absmean": 0.01486556, "gate_c2b/L1_absmax": 0.42748278, "gate_c2b/L1_rms": 0.05354483, "gate_c2b/L3_absmean": 0.21030043, "gate_c2b/L3_absmax": 0.43761557, "gate_c2b/L3_rms": 0.21427195, "gate_c2b/L5_absmean": 0.52988684, "gate_c2b/L5_absmax": 0.71679819, "gate_c2b/L5_rms": 0.54170763, "gate_b2a/L7_absmean": 0.08942077, "gate_b2a/L7_absmax": 0.27968886, "gate_b2a/L7_rms": 0.0925419, "gate_b2a/L11_absmean": 0.13526183, "gate_b2a/L11_absmax": 0.47484013, "gate_b2a/L11_rms": 0.13824959, "gate_b2a/L14_absmean": 0.32561642, "gate_b2a/L14_absmax": 0.57313329, "gate_b2a/L14_rms": 0.32723722, "gate_b2c/L3_absmean": 0.10514786, "gate_b2c/L3_absmax": 0.63175839, "gate_b2c/L3_rms": 0.1206737, "gate_b2c/L5_absmean": 0.12649906, "gate_b2c/L5_absmax": 0.78265601, "gate_b2c/L5_rms": 0.13481741, "gate_b2c/L6_absmean": 0.18619955, "gate_b2c/L6_absmax": 0.53211766, "gate_b2c/L6_rms": 0.1910788}, "streams": {"cos_a2b/round0": 0.379161, "cos_c2b/round0": 0.262096, "cos_b2a/round0": 0.100881, "cos_b2c/round0": 0.113836, "rms_a/round0": 0.857901, "rms_b/round0": 1.542735, "rms_c/round0": 1.268419, "cos_a2b/round1": 0.418174, "cos_c2b/round1": 0.278152, "cos_b2a/round1": 0.220785, "cos_b2c/round1": 0.131579, "rms_a/round1": 1.527017, "rms_b/round1": 3.195204, "rms_c/round1": 2.136682, "cos_a2b/round2": 0.506982, "cos_c2b/round2": 0.479713, "cos_b2a/round2": 0.028615, "cos_b2c/round2": 0.103221, "rms_a/round2": 3.366076, "rms_b/round2": 7.047143, "rms_c/round2": 3.447992}, "alpha_mean": 0.4330804072801797, "alpha_hist": [0.04697, 0.14415, 0.16689, 0.14651, 0.11754, 0.09822, 0.09758, 0.10173, 0.07098, 0.00943], "alpha_by_type": {"special": {"share": 0.00108, "mean": 0.64767, "std": 0.09456}, "whitespace": {"share": 0.03307, "mean": 0.63528, "std": 0.1898}, "digit": {"share": 0.01155, "mean": 0.42087, "std": 0.17778}, "word_start": {"share": 0.57123, "mean": 0.4476, "std": 0.22239}, "word_cont": {"share": 0.26714, "mean": 0.27635, "std": 0.16052}, "punct": {"share": 0.11594, "mean": 0.66418, "std": 0.19094}}, "alpha_p05": 0.10125, "alpha_p50": 0.42179, "alpha_p95": 0.83575}
{"step": 25210, "total_steps": 72479, "tokens": 13217300480, "loss": 3.455247312784195, "l_mix": 2.1259322464466095, "l_a": 2.2468904554843903, "l_c": 2.1788313686847687, "z_loss": 15.984746098518372, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.23275473713874817, "tokens_per_second": 1459282.2779852243, "eta_seconds": 16982.711190199854, "data_wait_seconds": 2.4050474166870117e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 25220, "total_steps": 72479, "tokens": 13222543360, "loss": 3.4644675552845, "l_mix": 2.1300255954265594, "l_a": 2.2541070878505707, "l_c": 2.1889249980449677, "z_loss": 15.323173761367798, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2706585228443146, "tokens_per_second": 1417839.308957974, "eta_seconds": 17475.412365460394, "data_wait_seconds": 2.6553869247436523e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08236}}
{"step": 25230, "total_steps": 72479, "tokens": 13227786240, "loss": 4.7181384563446045, "l_mix": 2.920857459306717, "l_a": 3.0161283016204834, "l_c": 2.969071000814438, "z_loss": 17.211875438690186, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1682431995868683, "tokens_per_second": 1478890.2290722264, "eta_seconds": 16750.45464837551, "data_wait_seconds": 1.621246337890625e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08232}}
{"step": 25240, "total_steps": 72479, "tokens": 13233029120, "loss": 4.78089302778244, "l_mix": 2.9600393772125244, "l_a": 3.0563377737998962, "l_c": 3.007661908864975, "z_loss": 16.536283135414124, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21490758657455444, "tokens_per_second": 1408351.851599327, "eta_seconds": 17585.691248869894, "data_wait_seconds": 1.6421079635620117e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 25250, "total_steps": 72479, "tokens": 13238272000, "loss": 4.680819809436798, "l_mix": 2.8978424668312073, "l_a": 2.9960639774799347, "l_c": 2.9413770139217377, "z_loss": 17.448758602142334, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22375597059726715, "tokens_per_second": 1437988.0618512733, "eta_seconds": 17219.61301968098, "data_wait_seconds": 1.5944242477416992e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 25260, "total_steps": 72479, "tokens": 13243514880, "loss": 4.760795056819916, "l_mix": 2.947892487049103, "l_a": 3.0435104072093964, "l_c": 2.9939508140087128, "z_loss": 16.640848398208618, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2614700496196747, "tokens_per_second": 1408886.833276184, "eta_seconds": 17571.571035575867, "data_wait_seconds": 1.7642974853515625e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 25270, "total_steps": 72479, "tokens": 13248757760, "loss": 4.755249738693237, "l_mix": 2.944003999233246, "l_a": 3.039996922016144, "l_c": 2.9916894137859344, "z_loss": 17.39805269241333, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2262571007013321, "tokens_per_second": 1380236.8217505447, "eta_seconds": 17932.511147332192, "data_wait_seconds": 1.7464160919189453e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 25280, "total_steps": 72479, "tokens": 13254000640, "loss": 4.797424733638763, "l_mix": 2.972034752368927, "l_a": 3.0624682009220123, "l_c": 3.0165125727653503, "z_loss": 16.955349445343018, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.24477523565292358, "tokens_per_second": 1394067.4944377358, "eta_seconds": 17750.840192985535, "data_wait_seconds": 2.822279930114746e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20763, "cosmopedia_v2": 0.20044, "finemath": 0.08236}}
{"step": 25290, "total_steps": 72479, "tokens": 13259243520, "loss": 5.001798003911972, "l_mix": 3.0997096598148346, "l_a": 3.188301980495453, "l_c": 3.146471083164215, "z_loss": 16.562586426734924, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20817819237709045, "tokens_per_second": 1422105.898334925, "eta_seconds": 17397.17587907314, "data_wait_seconds": 2.345442771911621e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 25300, "total_steps": 72479, "tokens": 13264486400, "loss": 4.825707852840424, "l_mix": 2.990102082490921, "l_a": 3.071514368057251, "l_c": 3.0415503680706024, "z_loss": 16.861518144607544, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2029026746749878, "tokens_per_second": 1452553.5281134832, "eta_seconds": 17028.896404337884, "data_wait_seconds": 1.7195940017700195e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.31784603, "gate_a2b/L1_absmax": 0.65485513, "gate_a2b/L1_rms": 0.33248082, "gate_a2b/L3_absmean": 0.33799249, "gate_a2b/L3_absmax": 0.5418427, "gate_a2b/L3_rms": 0.34237412, "gate_a2b/L5_absmean": 0.51507187, "gate_a2b/L5_absmax": 0.68322557, "gate_a2b/L5_rms": 0.52073312, "gate_c2b/L1_absmean": 0.01494996, "gate_c2b/L1_absmax": 0.42805865, "gate_c2b/L1_rms": 0.05359349, "gate_c2b/L3_absmean": 0.21048692, "gate_c2b/L3_absmax": 0.44493625, "gate_c2b/L3_rms": 0.21445647, "gate_c2b/L5_absmean": 0.52989072, "gate_c2b/L5_absmax": 0.71260637, "gate_c2b/L5_rms": 0.54166722, "gate_b2a/L7_absmean": 0.08910103, "gate_b2a/L7_absmax": 0.27998033, "gate_b2a/L7_rms": 0.09223872, "gate_b2a/L11_absmean": 0.1354388, "gate_b2a/L11_absmax": 0.48026425, "gate_b2a/L11_rms": 0.13849339, "gate_b2a/L14_absmean": 0.32621899, "gate_b2a/L14_absmax": 0.57483298, "gate_b2a/L14_rms": 0.32782325, "gate_b2c/L3_absmean": 0.10520861, "gate_b2c/L3_absmax": 0.63395029, "gate_b2c/L3_rms": 0.12068293, "gate_b2c/L5_absmean": 0.12631963, "gate_b2c/L5_absmax": 0.78729463, "gate_b2c/L5_rms": 0.13476428, "gate_b2c/L6_absmean": 0.18618427, "gate_b2c/L6_absmax": 0.5335601, "gate_b2c/L6_rms": 0.1911073}, "streams": {"cos_a2b/round0": 0.390994, "cos_c2b/round0": 0.301336, "cos_b2a/round0": 0.111711, "cos_b2c/round0": 0.153363, "rms_a/round0": 0.817864, "rms_b/round0": 1.538292, "rms_c/round0": 1.244947, "cos_a2b/round1": 0.411557, "cos_c2b/round1": 0.271602, "cos_b2a/round1": 0.232953, "cos_b2c/round1": 0.123532, "rms_a/round1": 1.435269, "rms_b/round1": 3.078527, "rms_c/round1": 2.020996, "cos_a2b/round2": 0.539742, "cos_c2b/round2": 0.494565, "cos_b2a/round2": 0.07777, "cos_b2c/round2": 0.141622, "rms_a/round2": 3.285013, "rms_b/round2": 6.856978, "rms_c/round2": 3.252529}, "alpha_mean": 0.4263467573321549, "alpha_hist": [0.0107, 0.06401, 0.16876, 0.21797, 0.21005, 0.16296, 0.11076, 0.04919, 0.00551, 8e-05], "alpha_by_type": {"special": {"share": 0.00044, "mean": 0.50111, "std": 0.08994}, "whitespace": {"share": 0.04333, "mean": 0.48514, "std": 0.11593}, "digit": {"share": 0.05186, "mean": 0.43412, "std": 0.10407}, "word_start": {"share": 0.54791, "mean": 0.44642, "std": 0.15653}, "word_cont": {"share": 0.24313, "mean": 0.31363, "std": 0.12556}, "punct": {"share": 0.11331, "mean": 0.5448, "std": 0.16084}}, "alpha_p05": 0.17413, "alpha_p50": 0.41703, "alpha_p95": 0.71083}
{"step": 25310, "total_steps": 72479, "tokens": 13269729280, "loss": 4.950822651386261, "l_mix": 3.0668329000473022, "l_a": 3.159538358449936, "l_c": 3.1146259903907776, "z_loss": 17.40297222137451, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22189748287200928, "tokens_per_second": 1449646.3866327535, "eta_seconds": 17059.42973406315, "data_wait_seconds": 3.17990779876709e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 25320, "total_steps": 72479, "tokens": 13274972160, "loss": 3.477755218744278, "l_mix": 2.140107035636902, "l_a": 2.2536463141441345, "l_c": 2.2000249326229095, "z_loss": 15.467838525772095, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2723771929740906, "tokens_per_second": 1349161.314335494, "eta_seconds": 18326.124184918404, "data_wait_seconds": 2.396106719970703e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 25330, "total_steps": 72479, "tokens": 13280215040, "loss": 3.500092774629593, "l_mix": 2.1518297493457794, "l_a": 2.2811757624149323, "l_c": 2.207857519388199, "z_loss": 15.529746055603027, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2543487250804901, "tokens_per_second": 1440214.2573101888, "eta_seconds": 17163.873212981227, "data_wait_seconds": 3.796815872192383e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 25340, "total_steps": 72479, "tokens": 13285457920, "loss": 4.728331208229065, "l_mix": 2.92611300945282, "l_a": 3.029628723859787, "l_c": 2.97200670838356, "z_loss": 17.274410009384155, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18277522921562195, "tokens_per_second": 1267181.541199157, "eta_seconds": 19503.450159645083, "data_wait_seconds": 2.7865171432495117e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08232}}
{"step": 25350, "total_steps": 72479, "tokens": 13290700800, "loss": 4.664598166942596, "l_mix": 2.888048529624939, "l_a": 2.9805735051631927, "l_c": 2.9352866113185883, "z_loss": 17.915724754333496, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.27807554602622986, "tokens_per_second": 1425976.368826865, "eta_seconds": 17327.89525279999, "data_wait_seconds": 1.6808509826660156e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 25360, "total_steps": 72479, "tokens": 13295943680, "loss": 4.750021815299988, "l_mix": 2.9413458108901978, "l_a": 3.0343031883239746, "l_c": 2.9890496134757996, "z_loss": 16.701236248016357, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18046438694000244, "tokens_per_second": 1437252.629185031, "eta_seconds": 17188.297847127913, "data_wait_seconds": 3.4362077713012695e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 25370, "total_steps": 72479, "tokens": 13301186560, "loss": 4.707010626792908, "l_mix": 2.9141936898231506, "l_a": 3.0107059478759766, "l_c": 2.9596658051013947, "z_loss": 17.052420616149902, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1676429659128189, "tokens_per_second": 1480043.8526850329, "eta_seconds": 16687.805126309395, "data_wait_seconds": 1.6510486602783203e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 25380, "total_steps": 72479, "tokens": 13306429440, "loss": 4.654850959777832, "l_mix": 2.8818266093730927, "l_a": 2.9767497181892395, "l_c": 2.927702635526657, "z_loss": 16.886265993118286, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20289205014705658, "tokens_per_second": 1402835.442732799, "eta_seconds": 17602.521122431757, "data_wait_seconds": 1.8864870071411133e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20764, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 25390, "total_steps": 72479, "tokens": 13311672320, "loss": 4.7514758706092834, "l_mix": 2.941679894924164, "l_a": 3.0438144207000732, "l_c": 2.9830698668956757, "z_loss": 17.306764602661133, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.24837848544120789, "tokens_per_second": 1325906.9195160703, "eta_seconds": 18619.857298135757, "data_wait_seconds": 2.6941299438476562e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20764, "cosmopedia_v2": 0.20043, "finemath": 0.08236}}
{"step": 25400, "total_steps": 72479, "tokens": 13316915200, "loss": 4.85816890001297, "l_mix": 3.01125705242157, "l_a": 3.092333137989044, "l_c": 3.0583881735801697, "z_loss": 16.954375982284546, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20532654225826263, "tokens_per_second": 1391364.0016192547, "eta_seconds": 17740.113100004197, "data_wait_seconds": 1.7762184143066406e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20768, "cosmopedia_v2": 0.20043, "finemath": 0.08232}, "gates": {"gate_a2b/L1_absmean": 0.31853318, "gate_a2b/L1_absmax": 0.66080791, "gate_a2b/L1_rms": 0.33321384, "gate_a2b/L3_absmean": 0.33873564, "gate_a2b/L3_absmax": 0.53493363, "gate_a2b/L3_rms": 0.34314916, "gate_a2b/L5_absmean": 0.51570052, "gate_a2b/L5_absmax": 0.67368722, "gate_a2b/L5_rms": 0.52122647, "gate_c2b/L1_absmean": 0.01516963, "gate_c2b/L1_absmax": 0.42590171, "gate_c2b/L1_rms": 0.05361806, "gate_c2b/L3_absmean": 0.21076246, "gate_c2b/L3_absmax": 0.44250727, "gate_c2b/L3_rms": 0.21472861, "gate_c2b/L5_absmean": 0.52942365, "gate_c2b/L5_absmax": 0.70880401, "gate_c2b/L5_rms": 0.54102582, "gate_b2a/L7_absmean": 0.08975611, "gate_b2a/L7_absmax": 0.28288093, "gate_b2a/L7_rms": 0.09288883, "gate_b2a/L11_absmean": 0.13579732, "gate_b2a/L11_absmax": 0.47031495, "gate_b2a/L11_rms": 0.13871999, "gate_b2a/L14_absmean": 0.32656965, "gate_b2a/L14_absmax": 0.58276403, "gate_b2a/L14_rms": 0.3281717, "gate_b2c/L3_absmean": 0.10561561, "gate_b2c/L3_absmax": 0.64101577, "gate_b2c/L3_rms": 0.12110092, "gate_b2c/L5_absmean": 0.126334, "gate_b2c/L5_absmax": 0.79629695, "gate_b2c/L5_rms": 0.13490401, "gate_b2c/L6_absmean": 0.18581431, "gate_b2c/L6_absmax": 0.51542222, "gate_b2c/L6_rms": 0.19067147}, "streams": {"cos_a2b/round0": 0.386693, "cos_c2b/round0": 0.307808, "cos_b2a/round0": 0.12115, "cos_b2c/round0": 0.16078, "rms_a/round0": 0.822882, "rms_b/round0": 1.523121, "rms_c/round0": 1.204662, "cos_a2b/round1": 0.410711, "cos_c2b/round1": 0.271754, "cos_b2a/round1": 0.23625, "cos_b2c/round1": 0.136965, "rms_a/round1": 1.428208, "rms_b/round1": 2.987016, "rms_c/round1": 1.987504, "cos_a2b/round2": 0.548683, "cos_c2b/round2": 0.49203, "cos_b2a/round2": 0.085652, "cos_b2c/round2": 0.156998, "rms_a/round2": 3.24254, "rms_b/round2": 6.68736, "rms_c/round2": 3.169883}, "alpha_mean": 0.45905819154691585, "alpha_hist": [0.00826, 0.04785, 0.12451, 0.17978, 0.22118, 0.21863, 0.13071, 0.06042, 0.00848, 0.00018], "alpha_by_type": {"special": {"share": 0.00041, "mean": 0.55108, "std": 0.12222}, "whitespace": {"share": 0.11147, "mean": 0.48842, "std": 0.10301}, "digit": {"share": 0.0381, "mean": 0.32867, "std": 0.16544}, "word_start": {"share": 0.52802, "mean": 0.48301, "std": 0.1485}, "word_cont": {"share": 0.215, "mean": 0.3495, "std": 0.14036}, "punct": {"share": 0.10701, "mean": 0.57646, "std": 0.15149}}, "alpha_p05": 0.19591, "alpha_p50": 0.47329, "alpha_p95": 0.71741}
{"step": 25410, "total_steps": 72479, "tokens": 13322158080, "loss": 4.894569277763367, "l_mix": 3.0327646136283875, "l_a": 3.122550219297409, "l_c": 3.0778898000717163, "z_loss": 16.72691035270691, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19865429401397705, "tokens_per_second": 1415230.7957348651, "eta_seconds": 17437.23493466377, "data_wait_seconds": 2.530217170715332e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 25420, "total_steps": 72479, "tokens": 13327400960, "loss": 4.90728896856308, "l_mix": 3.0411759316921234, "l_a": 3.1271701753139496, "l_c": 3.0875718891620636, "z_loss": 16.903462171554565, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17256887257099152, "tokens_per_second": 1366653.3372569561, "eta_seconds": 18053.20216867924, "data_wait_seconds": 2.2649765014648438e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 25430, "total_steps": 72479, "tokens": 13332643840, "loss": 4.798699975013733, "l_mix": 2.9723711609840393, "l_a": 3.061381906270981, "l_c": 3.020643472671509, "z_loss": 17.21094036102295, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1717335283756256, "tokens_per_second": 1458240.36053103, "eta_seconds": 16915.747759866714, "data_wait_seconds": 5.5789947509765625e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 25440, "total_steps": 72479, "tokens": 13337886720, "loss": 4.859864354133606, "l_mix": 3.0113591849803925, "l_a": 3.0964150428771973, "l_c": 3.0595391392707825, "z_loss": 17.187753200531006, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16834068298339844, "tokens_per_second": 1338026.571960456, "eta_seconds": 18431.609467864037, "data_wait_seconds": 2.9742717742919922e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 25450, "total_steps": 72479, "tokens": 13343129600, "loss": 4.661313951015472, "l_mix": 2.8850244879722595, "l_a": 2.9858050644397736, "l_c": 2.9294704496860504, "z_loss": 17.06656265258789, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18630510568618774, "tokens_per_second": 1430367.875507925, "eta_seconds": 17238.041188001633, "data_wait_seconds": 1.8030405044555664e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08232}}
{"step": 25460, "total_steps": 72479, "tokens": 13348372480, "loss": 4.628196358680725, "l_mix": 2.8659134209156036, "l_a": 2.9599527716636658, "l_c": 2.9088503420352936, "z_loss": 16.4188871383667, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21956166625022888, "tokens_per_second": 1316948.3626067278, "eta_seconds": 18718.651521921158, "data_wait_seconds": 1.9371509552001953e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 25470, "total_steps": 72479, "tokens": 13353615360, "loss": 4.737776458263397, "l_mix": 2.9331018030643463, "l_a": 3.031263291835785, "l_c": 2.9786853194236755, "z_loss": 16.901340007781982, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20098643004894257, "tokens_per_second": 1431059.8630622302, "eta_seconds": 17222.378481960295, "data_wait_seconds": 1.5169382095336914e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 25480, "total_steps": 72479, "tokens": 13358858240, "loss": 4.710260331630707, "l_mix": 2.9164433777332306, "l_a": 3.013848751783371, "l_c": 2.959893047809601, "z_loss": 16.944474458694458, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20091865956783295, "tokens_per_second": 1470605.6561155783, "eta_seconds": 16755.689473605158, "data_wait_seconds": 1.6063451766967773e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 25490, "total_steps": 72479, "tokens": 13364101120, "loss": 4.684817910194397, "l_mix": 2.900827556848526, "l_a": 2.994723528623581, "l_c": 2.9462481439113617, "z_loss": 16.98769474029541, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20285296440124512, "tokens_per_second": 1122700.501417554, "eta_seconds": 21943.3132887125, "data_wait_seconds": 0.09828019142150879, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 25500, "total_steps": 72479, "tokens": 13369344000, "loss": 4.699309527873993, "l_mix": 2.909296214580536, "l_a": 3.007096976041794, "l_c": 2.9541594684123993, "z_loss": 16.36353302001953, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2488563060760498, "tokens_per_second": 1387942.0077078883, "eta_seconds": 17746.077152514456, "data_wait_seconds": 3.1501054763793945e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.31947458, "gate_a2b/L1_absmax": 0.65669006, "gate_a2b/L1_rms": 0.33397475, "gate_a2b/L3_absmean": 0.33862692, "gate_a2b/L3_absmax": 0.53796953, "gate_a2b/L3_rms": 0.34306252, "gate_a2b/L5_absmean": 0.51582289, "gate_a2b/L5_absmax": 0.68177515, "gate_a2b/L5_rms": 0.52142572, "gate_c2b/L1_absmean": 0.01488195, "gate_c2b/L1_absmax": 0.42625549, "gate_c2b/L1_rms": 0.05338833, "gate_c2b/L3_absmean": 0.21062389, "gate_c2b/L3_absmax": 0.43553048, "gate_c2b/L3_rms": 0.21452254, "gate_c2b/L5_absmean": 0.53017038, "gate_c2b/L5_absmax": 0.70982313, "gate_c2b/L5_rms": 0.54191309, "gate_b2a/L7_absmean": 0.08952475, "gate_b2a/L7_absmax": 0.28819788, "gate_b2a/L7_rms": 0.09279735, "gate_b2a/L11_absmean": 0.13604552, "gate_b2a/L11_absmax": 0.47765359, "gate_b2a/L11_rms": 0.1390347, "gate_b2a/L14_absmean": 0.32633871, "gate_b2a/L14_absmax": 0.5762974, "gate_b2a/L14_rms": 0.32791731, "gate_b2c/L3_absmean": 0.10628249, "gate_b2c/L3_absmax": 0.63790369, "gate_b2c/L3_rms": 0.12162462, "gate_b2c/L5_absmean": 0.12679201, "gate_b2c/L5_absmax": 0.78398883, "gate_b2c/L5_rms": 0.13521188, "gate_b2c/L6_absmean": 0.18633634, "gate_b2c/L6_absmax": 0.53286719, "gate_b2c/L6_rms": 0.19126403}, "streams": {"cos_a2b/round0": 0.387844, "cos_c2b/round0": 0.297094, "cos_b2a/round0": 0.108343, "cos_b2c/round0": 0.149302, "rms_a/round0": 0.81902, "rms_b/round0": 1.532238, "rms_c/round0": 1.251263, "cos_a2b/round1": 0.416097, "cos_c2b/round1": 0.277443, "cos_b2a/round1": 0.223352, "cos_b2c/round1": 0.112225, "rms_a/round1": 1.436594, "rms_b/round1": 3.004295, "rms_c/round1": 2.020499, "cos_a2b/round2": 0.546427, "cos_c2b/round2": 0.501353, "cos_b2a/round2": 0.068905, "cos_b2c/round2": 0.139118, "rms_a/round2": 3.276997, "rms_b/round2": 6.790958, "rms_c/round2": 3.199209}, "alpha_mean": 0.45101038096333923, "alpha_hist": [0.00664, 0.03967, 0.1273, 0.21053, 0.24358, 0.19153, 0.1218, 0.05107, 0.00786, 3e-05], "alpha_by_type": {"special": {"share": 0.00046, "mean": 0.52461, "std": 0.11095}, "whitespace": {"share": 0.02127, "mean": 0.51333, "std": 0.1125}, "digit": {"share": 0.03278, "mean": 0.43469, "std": 0.11084}, "word_start": {"share": 0.59358, "mean": 0.46975, "std": 0.14544}, "word_cont": {"share": 0.23917, "mean": 0.34626, "std": 0.12491}, "punct": {"share": 0.11275, "mean": 0.56721, "std": 0.14111}}, "alpha_p05": 0.21078, "alpha_p50": 0.44059, "alpha_p95": 0.69699}
{"step": 25510, "total_steps": 72479, "tokens": 13374586880, "loss": 3.2147594690322876, "l_mix": 1.986535057425499, "l_a": 2.0732901096343994, "l_c": 2.014563024044037, "z_loss": 18.6851966381073, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.3045601546764374, "tokens_per_second": 1479866.7606072652, "eta_seconds": 16640.202839541435, "data_wait_seconds": 1.7583370208740234e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20764, "cosmopedia_v2": 0.20043, "finemath": 0.08236}}
{"step": 25520, "total_steps": 72479, "tokens": 13379829760, "loss": 4.813803970813751, "l_mix": 2.9828598499298096, "l_a": 3.067499041557312, "l_c": 3.02993243932724, "z_loss": 17.14654016494751, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16686619818210602, "tokens_per_second": 1423710.3308754573, "eta_seconds": 17292.8717717886, "data_wait_seconds": 2.816319465637207e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 25530, "total_steps": 72479, "tokens": 13385072640, "loss": 4.835972428321838, "l_mix": 2.9969567954540253, "l_a": 3.08308407664299, "l_c": 3.041281282901764, "z_loss": 17.059921979904175, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17795558273792267, "tokens_per_second": 1257228.9934361791, "eta_seconds": 19578.61092967987, "data_wait_seconds": 2.1398067474365234e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 25540, "total_steps": 72479, "tokens": 13390315520, "loss": 4.957619786262512, "l_mix": 3.072965145111084, "l_a": 3.1548176407814026, "l_c": 3.121809810400009, "z_loss": 16.663356065750122, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.24985060095787048, "tokens_per_second": 1452523.3052237707, "eta_seconds": 16942.62277479172, "data_wait_seconds": 1.9848346710205078e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 25550, "total_steps": 72479, "tokens": 13395558400, "loss": 4.8572505712509155, "l_mix": 3.0092690885066986, "l_a": 3.101625978946686, "l_c": 3.052609622478485, "z_loss": 17.10734748840332, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17560236155986786, "tokens_per_second": 1395549.6434100578, "eta_seconds": 17630.552713179586, "data_wait_seconds": 2.9921531677246094e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 25560, "total_steps": 72479, "tokens": 13400801280, "loss": 3.1241294741630554, "l_mix": 1.931108146905899, "l_a": 2.009020060300827, "l_c": 1.961392804980278, "z_loss": 18.97387456893921, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.252137690782547, "tokens_per_second": 1300928.3854200067, "eta_seconds": 18908.85689611435, "data_wait_seconds": 2.1129846572875977e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08236}}
{"step": 25570, "total_steps": 72479, "tokens": 13406044160, "loss": 4.761251330375671, "l_mix": 2.948186695575714, "l_a": 3.0429787039756775, "l_c": 2.994940400123596, "z_loss": 16.888925313949585, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16716443002223969, "tokens_per_second": 1442440.5439392983, "eta_seconds": 17050.14871866703, "data_wait_seconds": 1.990795135498047e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08232}}
{"step": 25580, "total_steps": 72479, "tokens": 13411287040, "loss": 4.628348886966705, "l_mix": 2.8649403750896454, "l_a": 2.96292981505394, "l_c": 2.9094733893871307, "z_loss": 16.874155521392822, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16465899348258972, "tokens_per_second": 1432075.318649344, "eta_seconds": 17169.895040988922, "data_wait_seconds": 1.7404556274414062e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 25590, "total_steps": 72479, "tokens": 13416529920, "loss": 4.678281486034393, "l_mix": 2.8958408534526825, "l_a": 2.9928167164325714, "l_c": 2.9429752826690674, "z_loss": 17.029130220413208, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15192393958568573, "tokens_per_second": 1445298.5936598135, "eta_seconds": 17009.177300691605, "data_wait_seconds": 2.2202730178833008e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 25600, "total_steps": 72479, "tokens": 13421772800, "loss": 4.644359707832336, "l_mix": 2.8745442926883698, "l_a": 2.974709212779999, "l_c": 2.9189288914203644, "z_loss": 17.23866367340088, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2143535315990448, "tokens_per_second": 1311232.2313525847, "eta_seconds": 18744.274709177018, "data_wait_seconds": 0.02890712022781372, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.31971613, "gate_a2b/L1_absmax": 0.6590693, "gate_a2b/L1_rms": 0.33423042, "gate_a2b/L3_absmean": 0.33899486, "gate_a2b/L3_absmax": 0.54174006, "gate_a2b/L3_rms": 0.34342593, "gate_a2b/L5_absmean": 0.51618475, "gate_a2b/L5_absmax": 0.67647386, "gate_a2b/L5_rms": 0.52175188, "gate_c2b/L1_absmean": 0.01444505, "gate_c2b/L1_absmax": 0.42586765, "gate_c2b/L1_rms": 0.05316126, "gate_c2b/L3_absmean": 0.21044956, "gate_c2b/L3_absmax": 0.43819031, "gate_c2b/L3_rms": 0.21441799, "gate_c2b/L5_absmean": 0.53018743, "gate_c2b/L5_absmax": 0.71309769, "gate_c2b/L5_rms": 0.54195869, "gate_b2a/L7_absmean": 0.08931503, "gate_b2a/L7_absmax": 0.28530401, "gate_b2a/L7_rms": 0.0925329, "gate_b2a/L11_absmean": 0.13596524, "gate_b2a/L11_absmax": 0.47842196, "gate_b2a/L11_rms": 0.13896027, "gate_b2a/L14_absmean": 0.32673812, "gate_b2a/L14_absmax": 0.57583195, "gate_b2a/L14_rms": 0.32830507, "gate_b2c/L3_absmean": 0.10624249, "gate_b2c/L3_absmax": 0.63915879, "gate_b2c/L3_rms": 0.12155889, "gate_b2c/L5_absmean": 0.12708005, "gate_b2c/L5_absmax": 0.78915006, "gate_b2c/L5_rms": 0.13551728, "gate_b2c/L6_absmean": 0.1865544, "gate_b2c/L6_absmax": 0.53597206, "gate_b2c/L6_rms": 0.19146316}, "streams": {"cos_a2b/round0": 0.386783, "cos_c2b/round0": 0.297385, "cos_b2a/round0": 0.111318, "cos_b2c/round0": 0.151999, "rms_a/round0": 0.827085, "rms_b/round0": 1.534877, "rms_c/round0": 1.242192, "cos_a2b/round1": 0.418117, "cos_c2b/round1": 0.268949, "cos_b2a/round1": 0.222152, "cos_b2c/round1": 0.128087, "rms_a/round1": 1.472655, "rms_b/round1": 3.08098, "rms_c/round1": 2.040128, "cos_a2b/round2": 0.54686, "cos_c2b/round2": 0.481172, "cos_b2a/round2": 0.063962, "cos_b2c/round2": 0.129392, "rms_a/round2": 3.332594, "rms_b/round2": 6.826255, "rms_c/round2": 3.284062}, "alpha_mean": 0.46136275303690155, "alpha_hist": [0.01942, 0.07399, 0.13748, 0.1638, 0.17346, 0.16911, 0.13803, 0.09454, 0.02966, 0.00049], "alpha_by_type": {"special": {"share": 0.00122, "mean": 0.58791, "std": 0.12506}, "whitespace": {"share": 0.02553, "mean": 0.5451, "std": 0.1318}, "digit": {"share": 0.03142, "mean": 0.50355, "std": 0.12835}, "word_start": {"share": 0.59625, "mean": 0.48537, "std": 0.17642}, "word_cont": {"share": 0.24625, "mean": 0.32454, "std": 0.15972}, "punct": {"share": 0.09933, "mean": 0.62003, "std": 0.17228}}, "alpha_p05": 0.16614, "alpha_p50": 0.4842, "alpha_p95": 0.77899}
{"step": 25610, "total_steps": 72479, "tokens": 13427015680, "loss": 4.709753215312958, "l_mix": 2.916744291782379, "l_a": 3.0071018040180206, "l_c": 2.9639111161231995, "z_loss": 17.051029920578003, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.24993954598903656, "tokens_per_second": 1289200.68441938, "eta_seconds": 19060.53461573124, "data_wait_seconds": 3.212690353393555e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 25620, "total_steps": 72479, "tokens": 13432258560, "loss": 2.930662900209427, "l_mix": 1.8099140673875809, "l_a": 1.8860612660646439, "l_c": 1.8432047218084335, "z_loss": 19.689222812652588, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2879564166069031, "tokens_per_second": 1425209.5684356815, "eta_seconds": 17237.8939463377, "data_wait_seconds": 2.6881694793701172e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08236}}
{"step": 25630, "total_steps": 72479, "tokens": 13437501440, "loss": 3.3309943974018097, "l_mix": 2.0584044754505157, "l_a": 2.1413752734661102, "l_c": 2.0943793654441833, "z_loss": 18.635603427886963, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.27461880445480347, "tokens_per_second": 1366779.4774370652, "eta_seconds": 17970.981359815596, "data_wait_seconds": 4.8547983169555664e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08236}}
{"step": 25640, "total_steps": 72479, "tokens": 13442744320, "loss": 4.903397619724274, "l_mix": 3.0387014150619507, "l_a": 3.1248497664928436, "l_c": 3.085172861814499, "z_loss": 16.89312434196472, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20151826739311218, "tokens_per_second": 1381273.3695458886, "eta_seconds": 17778.613686060908, "data_wait_seconds": 1.990795135498047e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 25650, "total_steps": 72479, "tokens": 13447987200, "loss": 4.85135555267334, "l_mix": 3.0061050355434418, "l_a": 3.0929000675678253, "l_c": 3.0522860288619995, "z_loss": 16.946703672409058, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.176134392619133, "tokens_per_second": 1456102.3588423347, "eta_seconds": 16861.371457099915, "data_wait_seconds": 2.0503997802734375e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20768, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 25660, "total_steps": 72479, "tokens": 13453230080, "loss": 4.60555762052536, "l_mix": 2.8509721755981445, "l_a": 2.947492331266403, "l_c": 2.8954184651374817, "z_loss": 17.120843172073364, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15483985841274261, "tokens_per_second": 1422769.0166706133, "eta_seconds": 17252.72309446335, "data_wait_seconds": 1.9758939743041992e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20764, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 25670, "total_steps": 72479, "tokens": 13458472960, "loss": 4.680916488170624, "l_mix": 2.898710608482361, "l_a": 2.994084358215332, "l_c": 2.9408918619155884, "z_loss": 17.12947940826416, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18198688328266144, "tokens_per_second": 1471587.9166122384, "eta_seconds": 16676.8133354187, "data_wait_seconds": 2.1666288375854492e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 25680, "total_steps": 72479, "tokens": 13463715840, "loss": 4.604119956493378, "l_mix": 2.8502767384052277, "l_a": 2.947309732437134, "l_c": 2.893210381269455, "z_loss": 16.871666431427002, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16229015588760376, "tokens_per_second": 1458841.6982781074, "eta_seconds": 16818.928428602216, "data_wait_seconds": 1.901388168334961e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08232}}
{"step": 25690, "total_steps": 72479, "tokens": 13468958720, "loss": 4.562160015106201, "l_mix": 2.8241582810878754, "l_a": 2.921836882829666, "l_c": 2.8657892644405365, "z_loss": 17.138514280319214, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.14748446643352509, "tokens_per_second": 1438789.576306795, "eta_seconds": 17049.686511468888, "data_wait_seconds": 1.8864870071411133e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 25700, "total_steps": 72479, "tokens": 13474201600, "loss": 4.71239310503006, "l_mix": 2.917366623878479, "l_a": 3.014798104763031, "l_c": 2.9629724621772766, "z_loss": 16.95297646522522, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.169112890958786, "tokens_per_second": 1339406.0893495623, "eta_seconds": 18310.85325579643, "data_wait_seconds": 1.5556812286376953e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.32090193, "gate_a2b/L1_absmax": 0.65688854, "gate_a2b/L1_rms": 0.3353664, "gate_a2b/L3_absmean": 0.33978701, "gate_a2b/L3_absmax": 0.53961688, "gate_a2b/L3_rms": 0.34416157, "gate_a2b/L5_absmean": 0.51617283, "gate_a2b/L5_absmax": 0.68033916, "gate_a2b/L5_rms": 0.52173549, "gate_c2b/L1_absmean": 0.01425115, "gate_c2b/L1_absmax": 0.4241966, "gate_c2b/L1_rms": 0.05303726, "gate_c2b/L3_absmean": 0.21051991, "gate_c2b/L3_absmax": 0.43733764, "gate_c2b/L3_rms": 0.21444914, "gate_c2b/L5_absmean": 0.52999938, "gate_c2b/L5_absmax": 0.71717978, "gate_c2b/L5_rms": 0.54174691, "gate_b2a/L7_absmean": 0.08927651, "gate_b2a/L7_absmax": 0.28520226, "gate_b2a/L7_rms": 0.09251689, "gate_b2a/L11_absmean": 0.13586944, "gate_b2a/L11_absmax": 0.47973469, "gate_b2a/L11_rms": 0.13885573, "gate_b2a/L14_absmean": 0.32626933, "gate_b2a/L14_absmax": 0.57563257, "gate_b2a/L14_rms": 0.32784891, "gate_b2c/L3_absmean": 0.10644853, "gate_b2c/L3_absmax": 0.63742256, "gate_b2c/L3_rms": 0.12186729, "gate_b2c/L5_absmean": 0.12717341, "gate_b2c/L5_absmax": 0.79445273, "gate_b2c/L5_rms": 0.13572061, "gate_b2c/L6_absmean": 0.1864773, "gate_b2c/L6_absmax": 0.53699696, "gate_b2c/L6_rms": 0.19147809}, "streams": {"cos_a2b/round0": 0.388027, "cos_c2b/round0": 0.301614, "cos_b2a/round0": 0.113858, "cos_b2c/round0": 0.156395, "rms_a/round0": 0.824799, "rms_b/round0": 1.541523, "rms_c/round0": 1.239115, "cos_a2b/round1": 0.42028, "cos_c2b/round1": 0.275962, "cos_b2a/round1": 0.226948, "cos_b2c/round1": 0.122236, "rms_a/round1": 1.465718, "rms_b/round1": 3.065664, "rms_c/round1": 2.035833, "cos_a2b/round2": 0.546147, "cos_c2b/round2": 0.49607, "cos_b2a/round2": 0.068679, "cos_b2c/round2": 0.137152, "rms_a/round2": 3.31164, "rms_b/round2": 6.833107, "rms_c/round2": 3.296649}, "alpha_mean": 0.4417640008056054, "alpha_hist": [0.01155, 0.06145, 0.15541, 0.19601, 0.2074, 0.17297, 0.11595, 0.06812, 0.01109, 5e-05], "alpha_by_type": {"special": {"share": 0.00098, "mean": 0.55218, "std": 0.08923}, "whitespace": {"share": 0.02325, "mean": 0.52252, "std": 0.11169}, "digit": {"share": 0.03374, "mean": 0.46615, "std": 0.11789}, "word_start": {"share": 0.58995, "mean": 0.45725, "std": 0.16007}, "word_cont": {"share": 0.24667, "mean": 0.32833, "std": 0.13635}, "punct": {"share": 0.10541, "mean": 0.59387, "std": 0.15616}}, "alpha_p05": 0.17218, "alpha_p50": 0.43638, "alpha_p95": 0.71701}
{"step": 25710, "total_steps": 72479, "tokens": 13479444480, "loss": 4.595773339271545, "l_mix": 2.8451389372348785, "l_a": 2.9418374598026276, "l_c": 2.8880069255828857, "z_loss": 16.81034779548645, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1975840926170349, "tokens_per_second": 1468232.0446781158, "eta_seconds": 16700.647258639336, "data_wait_seconds": 2.574920654296875e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 25720, "total_steps": 72479, "tokens": 13484687360, "loss": 4.530604779720306, "l_mix": 2.803677022457123, "l_a": 2.902449816465378, "l_c": 2.848305106163025, "z_loss": 17.011680841445923, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1866590827703476, "tokens_per_second": 1404622.632256693, "eta_seconds": 17453.216279602053, "data_wait_seconds": 2.41696834564209e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 25730, "total_steps": 72479, "tokens": 13489930240, "loss": 4.6410229206085205, "l_mix": 2.8718343675136566, "l_a": 2.970930725336075, "l_c": 2.9206454157829285, "z_loss": 17.155982494354248, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22163042426109314, "tokens_per_second": 1384354.638541209, "eta_seconds": 17704.95726284981, "data_wait_seconds": 4.509091377258301e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08236}}
{"step": 25740, "total_steps": 72479, "tokens": 13495173120, "loss": 3.0122534036636353, "l_mix": 1.8625274449586868, "l_a": 1.9297921061515808, "l_c": 1.8959929943084717, "z_loss": 19.903453826904297, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.3197963237762451, "tokens_per_second": 1301039.0663477941, "eta_seconds": 18834.71255078316, "data_wait_seconds": 1.6987323760986328e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08236}}
{"step": 25750, "total_steps": 72479, "tokens": 13500416000, "loss": 4.6482322216033936, "l_mix": 2.8770565688610077, "l_a": 2.9763476848602295, "l_c": 2.9219058454036713, "z_loss": 16.995028495788574, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1666368991136551, "tokens_per_second": 1341521.5851236773, "eta_seconds": 18262.437387275695, "data_wait_seconds": 1.8417835235595703e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 25760, "total_steps": 72479, "tokens": 13505658880, "loss": 4.552689075469971, "l_mix": 2.817225903272629, "l_a": 2.918204754590988, "l_c": 2.8609166741371155, "z_loss": 17.265953540802002, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18379758298397064, "tokens_per_second": 1453607.3178220948, "eta_seconds": 16850.638251256943, "data_wait_seconds": 1.7911195755004883e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 25770, "total_steps": 72479, "tokens": 13510901760, "loss": 4.573385715484619, "l_mix": 2.8298761546611786, "l_a": 2.9296930730342865, "l_c": 2.876303255558014, "z_loss": 17.10517406463623, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19510355591773987, "tokens_per_second": 1463326.3001819456, "eta_seconds": 16735.13842329979, "data_wait_seconds": 1.8477439880371094e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 25780, "total_steps": 72479, "tokens": 13516144640, "loss": 4.592374265193939, "l_mix": 2.8430571854114532, "l_a": 2.939933955669403, "l_c": 2.8855080902576447, "z_loss": 16.84468722343445, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17867760360240936, "tokens_per_second": 1433841.9074968535, "eta_seconds": 17075.610068297385, "data_wait_seconds": 1.8298625946044922e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 25790, "total_steps": 72479, "tokens": 13521387520, "loss": 4.6180140376091, "l_mix": 2.8579883873462677, "l_a": 2.9564546644687653, "l_c": 2.9046250581741333, "z_loss": 17.01633858680725, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20683720707893372, "tokens_per_second": 1408653.1743282431, "eta_seconds": 17377.22448513508, "data_wait_seconds": 3.3289194107055664e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20043, "finemath": 0.08236}}
{"step": 25800, "total_steps": 72479, "tokens": 13526630400, "loss": 4.642305254936218, "l_mix": 2.874122828245163, "l_a": 2.969528943300247, "l_c": 2.9188249707221985, "z_loss": 16.76130986213684, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15336185693740845, "tokens_per_second": 1370268.6231195459, "eta_seconds": 17860.176566171645, "data_wait_seconds": 2.0891427993774414e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.31948408, "gate_a2b/L1_absmax": 0.65153956, "gate_a2b/L1_rms": 0.33375105, "gate_a2b/L3_absmean": 0.33999005, "gate_a2b/L3_absmax": 0.53299755, "gate_a2b/L3_rms": 0.34433678, "gate_a2b/L5_absmean": 0.51628464, "gate_a2b/L5_absmax": 0.67977619, "gate_a2b/L5_rms": 0.52184153, "gate_c2b/L1_absmean": 0.01598135, "gate_c2b/L1_absmax": 0.42597598, "gate_c2b/L1_rms": 0.05367719, "gate_c2b/L3_absmean": 0.21017753, "gate_c2b/L3_absmax": 0.43374923, "gate_c2b/L3_rms": 0.21410008, "gate_c2b/L5_absmean": 0.52983707, "gate_c2b/L5_absmax": 0.72122562, "gate_c2b/L5_rms": 0.54163069, "gate_b2a/L7_absmean": 0.08952116, "gate_b2a/L7_absmax": 0.28858232, "gate_b2a/L7_rms": 0.09277052, "gate_b2a/L11_absmean": 0.13661668, "gate_b2a/L11_absmax": 0.48740265, "gate_b2a/L11_rms": 0.13961259, "gate_b2a/L14_absmean": 0.32643229, "gate_b2a/L14_absmax": 0.5757547, "gate_b2a/L14_rms": 0.32799685, "gate_b2c/L3_absmean": 0.10636653, "gate_b2c/L3_absmax": 0.63990736, "gate_b2c/L3_rms": 0.12168629, "gate_b2c/L5_absmean": 0.12736097, "gate_b2c/L5_absmax": 0.79453605, "gate_b2c/L5_rms": 0.13589315, "gate_b2c/L6_absmean": 0.18699846, "gate_b2c/L6_absmax": 0.53473336, "gate_b2c/L6_rms": 0.19197735}, "streams": {"cos_a2b/round0": 0.384886, "cos_c2b/round0": 0.303688, "cos_b2a/round0": 0.111315, "cos_b2c/round0": 0.15323, "rms_a/round0": 0.823663, "rms_b/round0": 1.523086, "rms_c/round0": 1.237344, "cos_a2b/round1": 0.417537, "cos_c2b/round1": 0.269161, "cos_b2a/round1": 0.22664, "cos_b2c/round1": 0.117286, "rms_a/round1": 1.455423, "rms_b/round1": 3.034606, "rms_c/round1": 2.006925, "cos_a2b/round2": 0.54677, "cos_c2b/round2": 0.493001, "cos_b2a/round2": 0.072719, "cos_b2c/round2": 0.13854, "rms_a/round2": 3.313314, "rms_b/round2": 6.861983, "rms_c/round2": 3.211089}, "alpha_mean": 0.44707370894293064, "alpha_hist": [0.01137, 0.05746, 0.14922, 0.19604, 0.19606, 0.18442, 0.12892, 0.06737, 0.00911, 3e-05], "alpha_by_type": {"special": {"share": 0.00114, "mean": 0.5808, "std": 0.10736}, "whitespace": {"share": 0.02959, "mean": 0.53928, "std": 0.12376}, "digit": {"share": 0.03369, "mean": 0.49558, "std": 0.11792}, "word_start": {"share": 0.5504, "mean": 0.46758, "std": 0.15601}, "word_cont": {"share": 0.26727, "mean": 0.33146, "std": 0.14127}, "punct": {"share": 0.1179, "mean": 0.57514, "std": 0.15582}}, "alpha_p05": 0.18447, "alpha_p50": 0.44083, "alpha_p95": 0.73144}
{"step": 25810, "total_steps": 72479, "tokens": 13531873280, "loss": 4.6279430985450745, "l_mix": 2.8646328151226044, "l_a": 2.9621153473854065, "l_c": 2.909934103488922, "z_loss": 16.95503544807434, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16221514344215393, "tokens_per_second": 1423182.8241525097, "eta_seconds": 17192.447981214525, "data_wait_seconds": 1.8149614334106445e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 25820, "total_steps": 72479, "tokens": 13537116160, "loss": 4.944689452648163, "l_mix": 3.065196543931961, "l_a": 3.1475742161273956, "l_c": 3.1116701662540436, "z_loss": 17.19444751739502, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20174692571163177, "tokens_per_second": 1437418.259092457, "eta_seconds": 17018.53558437824, "data_wait_seconds": 2.822279930114746e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 25830, "total_steps": 72479, "tokens": 13542359040, "loss": 4.851543843746185, "l_mix": 3.0062271058559418, "l_a": 3.092606693506241, "l_c": 3.0526941418647766, "z_loss": 17.263067483901978, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17086827754974365, "tokens_per_second": 1319286.7213197958, "eta_seconds": 18538.434835100175, "data_wait_seconds": 2.092123031616211e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 25840, "total_steps": 72479, "tokens": 13547601920, "loss": 4.8551629185676575, "l_mix": 3.0085860788822174, "l_a": 3.094752609729767, "l_c": 3.0547719299793243, "z_loss": 17.194467067718506, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19025632739067078, "tokens_per_second": 1416975.3107080187, "eta_seconds": 17256.66484603882, "data_wait_seconds": 2.6404857635498047e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 25850, "total_steps": 72479, "tokens": 13552844800, "loss": 3.0249461233615875, "l_mix": 1.8699617460370064, "l_a": 1.9414660334587097, "l_c": 1.9026083946228027, "z_loss": 17.619511127471924, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.25348415970802307, "tokens_per_second": 1422107.4617829637, "eta_seconds": 17190.70169377327, "data_wait_seconds": 2.703070640563965e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08236}}
{"step": 25860, "total_steps": 72479, "tokens": 13558087680, "loss": 4.651603519916534, "l_mix": 2.8795828223228455, "l_a": 2.976279228925705, "l_c": 2.924806773662567, "z_loss": 16.950162172317505, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1647750288248062, "tokens_per_second": 1431723.1569897463, "eta_seconds": 17071.58409268856, "data_wait_seconds": 2.6524066925048828e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 25870, "total_steps": 72479, "tokens": 13563330560, "loss": 4.650889575481415, "l_mix": 2.8790321946144104, "l_a": 2.9758372008800507, "l_c": 2.9247280955314636, "z_loss": 16.878709077835083, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19311419129371643, "tokens_per_second": 1356754.9427571462, "eta_seconds": 18011.01925034523, "data_wait_seconds": 2.8312206268310547e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 25880, "total_steps": 72479, "tokens": 13568573440, "loss": 4.591887474060059, "l_mix": 2.842370241880417, "l_a": 2.940584123134613, "l_c": 2.885496348142624, "z_loss": 16.930683135986328, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1610129475593567, "tokens_per_second": 1430347.593263661, "eta_seconds": 17080.67089920044, "data_wait_seconds": 2.2590160369873047e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 25890, "total_steps": 72479, "tokens": 13573816320, "loss": 4.591771185398102, "l_mix": 2.842884808778763, "l_a": 2.939502716064453, "l_c": 2.884473592042923, "z_loss": 16.935014963150024, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1801440566778183, "tokens_per_second": 1410429.7163507675, "eta_seconds": 17318.16435008049, "data_wait_seconds": 2.5212764739990234e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 25900, "total_steps": 72479, "tokens": 13579059200, "loss": 4.636851012706757, "l_mix": 2.8708119988441467, "l_a": 2.9681995511054993, "l_c": 2.9129543006420135, "z_loss": 16.929490566253662, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19923019409179688, "tokens_per_second": 1382413.233931392, "eta_seconds": 17665.3479238987, "data_wait_seconds": 1.633167266845703e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20042, "finemath": 0.08236}, "gates": {"gate_a2b/L1_absmean": 0.32151896, "gate_a2b/L1_absmax": 0.64435685, "gate_a2b/L1_rms": 0.33557588, "gate_a2b/L3_absmean": 0.34036347, "gate_a2b/L3_absmax": 0.53165275, "gate_a2b/L3_rms": 0.34477082, "gate_a2b/L5_absmean": 0.51651424, "gate_a2b/L5_absmax": 0.67433435, "gate_a2b/L5_rms": 0.52208346, "gate_c2b/L1_absmean": 0.01447147, "gate_c2b/L1_absmax": 0.42428774, "gate_c2b/L1_rms": 0.05262459, "gate_c2b/L3_absmean": 0.21097384, "gate_c2b/L3_absmax": 0.43993947, "gate_c2b/L3_rms": 0.21486621, "gate_c2b/L5_absmean": 0.52945566, "gate_c2b/L5_absmax": 0.71752554, "gate_c2b/L5_rms": 0.54115897, "gate_b2a/L7_absmean": 0.08966234, "gate_b2a/L7_absmax": 0.29098073, "gate_b2a/L7_rms": 0.09298518, "gate_b2a/L11_absmean": 0.1369064, "gate_b2a/L11_absmax": 0.48346978, "gate_b2a/L11_rms": 0.13990264, "gate_b2a/L14_absmean": 0.32683492, "gate_b2a/L14_absmax": 0.57754707, "gate_b2a/L14_rms": 0.32843539, "gate_b2c/L3_absmean": 0.10799831, "gate_b2c/L3_absmax": 0.63895243, "gate_b2c/L3_rms": 0.12302169, "gate_b2c/L5_absmean": 0.12792614, "gate_b2c/L5_absmax": 0.7812829, "gate_b2c/L5_rms": 0.13618137, "gate_b2c/L6_absmean": 0.18739581, "gate_b2c/L6_absmax": 0.53624225, "gate_b2c/L6_rms": 0.19241405}, "streams": {"cos_a2b/round0": 0.386925, "cos_c2b/round0": 0.293143, "cos_b2a/round0": 0.114226, "cos_b2c/round0": 0.144463, "rms_a/round0": 0.844554, "rms_b/round0": 1.530981, "rms_c/round0": 1.260544, "cos_a2b/round1": 0.408842, "cos_c2b/round1": 0.27655, "cos_b2a/round1": 0.234725, "cos_b2c/round1": 0.12007, "rms_a/round1": 1.445634, "rms_b/round1": 3.034816, "rms_c/round1": 2.04716, "cos_a2b/round2": 0.52933, "cos_c2b/round2": 0.512145, "cos_b2a/round2": 0.075078, "cos_b2c/round2": 0.140518, "rms_a/round2": 3.252499, "rms_b/round2": 6.815714, "rms_c/round2": 3.266702}, "alpha_mean": 0.4327432266422946, "alpha_hist": [0.01718, 0.05685, 0.15337, 0.21068, 0.22337, 0.16457, 0.10999, 0.05673, 0.00723, 3e-05], "alpha_by_type": {"special": {"share": 0.00117, "mean": 0.5412, "std": 0.08056}, "whitespace": {"share": 0.03032, "mean": 0.51639, "std": 0.10059}, "digit": {"share": 0.03061, "mean": 0.39279, "std": 0.12565}, "word_start": {"share": 0.51913, "mean": 0.45525, "std": 0.16214}, "word_cont": {"share": 0.23763, "mean": 0.33224, "std": 0.14031}, "punct": {"share": 0.18114, "mean": 0.49213, "std": 0.1598}}, "alpha_p05": 0.16749, "alpha_p50": 0.4206, "alpha_p95": 0.69451}
{"step": 25910, "total_steps": 72479, "tokens": 13584302080, "loss": 4.886794328689575, "l_mix": 3.028213918209076, "l_a": 3.117356240749359, "l_c": 3.0721935629844666, "z_loss": 17.154487133026123, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1609523445367813, "tokens_per_second": 1411516.8385643633, "eta_seconds": 17297.39752650261, "data_wait_seconds": 2.300739288330078e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20768, "cosmopedia_v2": 0.20042, "finemath": 0.08232}}
{"step": 25920, "total_steps": 72479, "tokens": 13589544960, "loss": 4.931480169296265, "l_mix": 3.0567133128643036, "l_a": 3.1421209573745728, "l_c": 3.1015182733535767, "z_loss": 16.750779628753662, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18106840550899506, "tokens_per_second": 1439946.6142013904, "eta_seconds": 16952.24305627346, "data_wait_seconds": 1.6421079635620117e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20768, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 25930, "total_steps": 72479, "tokens": 13594787840, "loss": 4.963916301727295, "l_mix": 3.076696127653122, "l_a": 3.1600133180618286, "l_c": 3.1249634623527527, "z_loss": 17.27056622505188, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1876276433467865, "tokens_per_second": 1341904.6237199486, "eta_seconds": 18186.897697949407, "data_wait_seconds": 3.612041473388672e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 25940, "total_steps": 72479, "tokens": 13600030720, "loss": 4.574813783168793, "l_mix": 2.8347119987010956, "l_a": 2.9152642637491226, "l_c": 2.8794784992933273, "z_loss": 16.78905200958252, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19499583542346954, "tokens_per_second": 1313383.2912726633, "eta_seconds": 18577.851107239723, "data_wait_seconds": 1.9431114196777344e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 25950, "total_steps": 72479, "tokens": 13605273600, "loss": 4.885700583457947, "l_mix": 3.0279192328453064, "l_a": 3.113206297159195, "l_c": 3.073908895254135, "z_loss": 16.466565132141113, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1823790818452835, "tokens_per_second": 1424818.3071678812, "eta_seconds": 17121.19800066948, "data_wait_seconds": 1.6063451766967773e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 25960, "total_steps": 72479, "tokens": 13610516480, "loss": 3.19134122133255, "l_mix": 1.9721352607011795, "l_a": 2.0521098971366882, "l_c": 2.0054043382406235, "z_loss": 19.516395568847656, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.27293768525123596, "tokens_per_second": 1415461.0843411493, "eta_seconds": 17230.677509832385, "data_wait_seconds": 3.141164779663086e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08236}}
{"step": 25970, "total_steps": 72479, "tokens": 13615759360, "loss": 4.60975980758667, "l_mix": 2.8527975380420685, "l_a": 2.9530344009399414, "l_c": 2.897736370563507, "z_loss": 17.309098482131958, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15091748535633087, "tokens_per_second": 1438960.7394622597, "eta_seconds": 16945.63994922638, "data_wait_seconds": 2.580881118774414e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 25980, "total_steps": 72479, "tokens": 13621002240, "loss": 4.642512619495392, "l_mix": 2.873465210199356, "l_a": 2.9743713438510895, "l_c": 2.9169277250766754, "z_loss": 16.57539415359497, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16116030514240265, "tokens_per_second": 1391303.6127400813, "eta_seconds": 17522.320425796508, "data_wait_seconds": 1.940131187438965e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 25990, "total_steps": 72479, "tokens": 13626245120, "loss": 4.556745111942291, "l_mix": 2.8204897344112396, "l_a": 2.9174018502235413, "l_c": 2.8644517362117767, "z_loss": 16.993131637573242, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15480639040470123, "tokens_per_second": 1343414.9206746009, "eta_seconds": 18143.03567490578, "data_wait_seconds": 3.001093864440918e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 26000, "total_steps": 72479, "tokens": 13631488000, "loss": 4.586044013500214, "l_mix": 2.8392002284526825, "l_a": 2.935105621814728, "l_c": 2.882126808166504, "z_loss": 16.739734411239624, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15965956449508667, "tokens_per_second": 1347582.9927093405, "eta_seconds": 18083.02871425152, "data_wait_seconds": 2.0742416381835938e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.32137901, "gate_a2b/L1_absmax": 0.65500188, "gate_a2b/L1_rms": 0.33544782, "gate_a2b/L3_absmean": 0.34025264, "gate_a2b/L3_absmax": 0.53865552, "gate_a2b/L3_rms": 0.344652, "gate_a2b/L5_absmean": 0.51673669, "gate_a2b/L5_absmax": 0.675596, "gate_a2b/L5_rms": 0.52231282, "gate_c2b/L1_absmean": 0.01422397, "gate_c2b/L1_absmax": 0.42605036, "gate_c2b/L1_rms": 0.05292454, "gate_c2b/L3_absmean": 0.21145292, "gate_c2b/L3_absmax": 0.44858333, "gate_c2b/L3_rms": 0.21541642, "gate_c2b/L5_absmean": 0.53025717, "gate_c2b/L5_absmax": 0.71541965, "gate_c2b/L5_rms": 0.54197162, "gate_b2a/L7_absmean": 0.09013298, "gate_b2a/L7_absmax": 0.28943619, "gate_b2a/L7_rms": 0.09344381, "gate_b2a/L11_absmean": 0.13750058, "gate_b2a/L11_absmax": 0.48505718, "gate_b2a/L11_rms": 0.1405105, "gate_b2a/L14_absmean": 0.32742232, "gate_b2a/L14_absmax": 0.57651931, "gate_b2a/L14_rms": 0.32902521, "gate_b2c/L3_absmean": 0.10845695, "gate_b2c/L3_absmax": 0.63946748, "gate_b2c/L3_rms": 0.12330402, "gate_b2c/L5_absmean": 0.12846175, "gate_b2c/L5_absmax": 0.78942436, "gate_b2c/L5_rms": 0.13679397, "gate_b2c/L6_absmean": 0.18768486, "gate_b2c/L6_absmax": 0.54621965, "gate_b2c/L6_rms": 0.19273821}, "streams": {"cos_a2b/round0": 0.388417, "cos_c2b/round0": 0.29933, "cos_b2a/round0": 0.108303, "cos_b2c/round0": 0.146524, "rms_a/round0": 0.822343, "rms_b/round0": 1.521425, "rms_c/round0": 1.242204, "cos_a2b/round1": 0.414829, "cos_c2b/round1": 0.270485, "cos_b2a/round1": 0.232947, "cos_b2c/round1": 0.121945, "rms_a/round1": 1.448909, "rms_b/round1": 3.073069, "rms_c/round1": 2.040454, "cos_a2b/round2": 0.53366, "cos_c2b/round2": 0.500082, "cos_b2a/round2": 0.066642, "cos_b2c/round2": 0.132188, "rms_a/round2": 3.298195, "rms_b/round2": 6.938901, "rms_c/round2": 3.330273}, "alpha_mean": 0.44006371201793115, "alpha_hist": [0.01732, 0.06766, 0.15649, 0.19415, 0.19278, 0.16675, 0.11897, 0.07028, 0.01546, 0.00014], "alpha_by_type": {"special": {"share": 0.00143, "mean": 0.54105, "std": 0.08433}, "whitespace": {"share": 0.0278, "mean": 0.52522, "std": 0.11358}, "digit": {"share": 0.03438, "mean": 0.50078, "std": 0.1116}, "word_start": {"share": 0.54819, "mean": 0.4644, "std": 0.16655}, "word_cont": {"share": 0.27849, "mean": 0.31885, "std": 0.14014}, "punct": {"share": 0.10971, "mean": 0.5842, "std": 0.15921}}, "alpha_p05": 0.16831, "alpha_p50": 0.43422, "alpha_p95": 0.73526}
{"step": 26010, "total_steps": 72479, "tokens": 13636730880, "loss": 4.676439881324768, "l_mix": 2.895259141921997, "l_a": 2.9910135865211487, "l_c": 2.940673530101776, "z_loss": 16.74550724029541, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1956457942724228, "tokens_per_second": 1460424.7058424274, "eta_seconds": 16682.228789019584, "data_wait_seconds": 3.612041473388672e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 26020, "total_steps": 72479, "tokens": 13641973760, "loss": 3.318327456712723, "l_mix": 2.052030310034752, "l_a": 2.1281126588582993, "l_c": 2.0863538682460785, "z_loss": 19.57140064239502, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21257221698760986, "tokens_per_second": 1436307.1187644259, "eta_seconds": 16958.696280050277, "data_wait_seconds": 3.9070844650268555e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08236}}
{"step": 26030, "total_steps": 72479, "tokens": 13647216640, "loss": 4.88072270154953, "l_mix": 3.0210126638412476, "l_a": 3.088295191526413, "l_c": 3.105128675699234, "z_loss": 16.828283309936523, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.8487308025360107, "tokens_per_second": 1408757.675006821, "eta_seconds": 17286.616246390342, "data_wait_seconds": 2.09808349609375e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20768, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 26040, "total_steps": 72479, "tokens": 13652459520, "loss": 4.901068389415741, "l_mix": 3.036876380443573, "l_a": 3.1237961053848267, "l_c": 3.084465265274048, "z_loss": 17.135616064071655, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20156463980674744, "tokens_per_second": 1482195.0460105818, "eta_seconds": 16426.59007499218, "data_wait_seconds": 1.850724220275879e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 26050, "total_steps": 72479, "tokens": 13657702400, "loss": 4.897359013557434, "l_mix": 3.035585731267929, "l_a": 3.1149292290210724, "l_c": 3.0853854417800903, "z_loss": 16.788424968719482, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18074829876422882, "tokens_per_second": 1453275.7013629982, "eta_seconds": 16749.862073087694, "data_wait_seconds": 1.9252300262451172e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 26060, "total_steps": 72479, "tokens": 13662945280, "loss": 4.943274915218353, "l_mix": 3.063672035932541, "l_a": 3.1502071022987366, "l_c": 3.109542280435562, "z_loss": 16.779894590377808, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19311876595020294, "tokens_per_second": 1340658.401749678, "eta_seconds": 18152.96472258568, "data_wait_seconds": 2.613663673400879e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 26070, "total_steps": 72479, "tokens": 13668188160, "loss": 5.023052930831909, "l_mix": 3.1136227548122406, "l_a": 3.2002249658107758, "l_c": 3.1589315235614777, "z_loss": 16.832496404647827, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18687744438648224, "tokens_per_second": 1421354.6455971436, "eta_seconds": 17118.65639400482, "data_wait_seconds": 1.9848346710205078e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08236}}
{"step": 26080, "total_steps": 72479, "tokens": 13673431040, "loss": 4.578633069992065, "l_mix": 2.8334556818008423, "l_a": 2.9325969517230988, "l_c": 2.878993809223175, "z_loss": 17.000622749328613, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15041255950927734, "tokens_per_second": 1253730.1530042028, "eta_seconds": 19403.24945819378, "data_wait_seconds": 1.8596649169921875e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08232}}
{"step": 26090, "total_steps": 72479, "tokens": 13678673920, "loss": 4.613178551197052, "l_mix": 2.8556360602378845, "l_a": 2.9526714086532593, "l_c": 2.900217592716217, "z_loss": 16.755878448486328, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16331054270267487, "tokens_per_second": 1405553.1068160487, "eta_seconds": 17303.6478764534, "data_wait_seconds": 1.576542854309082e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 26100, "total_steps": 72479, "tokens": 13683916800, "loss": 4.5737797021865845, "l_mix": 2.830483466386795, "l_a": 2.92891263961792, "l_c": 2.87639519572258, "z_loss": 17.0370512008667, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18543298542499542, "tokens_per_second": 1432697.2690902774, "eta_seconds": 16972.150137090684, "data_wait_seconds": 1.6570091247558594e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.32090354, "gate_a2b/L1_absmax": 0.65301806, "gate_a2b/L1_rms": 0.33510467, "gate_a2b/L3_absmean": 0.33933747, "gate_a2b/L3_absmax": 0.53800505, "gate_a2b/L3_rms": 0.34371483, "gate_a2b/L5_absmean": 0.51595086, "gate_a2b/L5_absmax": 0.6708467, "gate_a2b/L5_rms": 0.52151096, "gate_c2b/L1_absmean": 0.01422534, "gate_c2b/L1_absmax": 0.42572823, "gate_c2b/L1_rms": 0.05298914, "gate_c2b/L3_absmean": 0.21074609, "gate_c2b/L3_absmax": 0.44618145, "gate_c2b/L3_rms": 0.21470003, "gate_c2b/L5_absmean": 0.529746, "gate_c2b/L5_absmax": 0.70499319, "gate_c2b/L5_rms": 0.54140913, "gate_b2a/L7_absmean": 0.09038714, "gate_b2a/L7_absmax": 0.28755093, "gate_b2a/L7_rms": 0.09364098, "gate_b2a/L11_absmean": 0.1369282, "gate_b2a/L11_absmax": 0.48274904, "gate_b2a/L11_rms": 0.13994424, "gate_b2a/L14_absmean": 0.32720482, "gate_b2a/L14_absmax": 0.57905024, "gate_b2a/L14_rms": 0.32882223, "gate_b2c/L3_absmean": 0.10787721, "gate_b2c/L3_absmax": 0.63705838, "gate_b2c/L3_rms": 0.12286732, "gate_b2c/L5_absmean": 0.12800857, "gate_b2c/L5_absmax": 0.79456502, "gate_b2c/L5_rms": 0.13650481, "gate_b2c/L6_absmean": 0.18720469, "gate_b2c/L6_absmax": 0.54636395, "gate_b2c/L6_rms": 0.19225517}, "streams": {"cos_a2b/round0": 0.388695, "cos_c2b/round0": 0.30371, "cos_b2a/round0": 0.115602, "cos_b2c/round0": 0.15208, "rms_a/round0": 0.818335, "rms_b/round0": 1.524263, "rms_c/round0": 1.245606, "cos_a2b/round1": 0.421609, "cos_c2b/round1": 0.275233, "cos_b2a/round1": 0.227984, "cos_b2c/round1": 0.123566, "rms_a/round1": 1.47397, "rms_b/round1": 3.052532, "rms_c/round1": 2.031475, "cos_a2b/round2": 0.539119, "cos_c2b/round2": 0.497768, "cos_b2a/round2": 0.058785, "cos_b2c/round2": 0.133807, "rms_a/round2": 3.322771, "rms_b/round2": 6.90745, "rms_c/round2": 3.297537}, "alpha_mean": 0.4378375566220214, "alpha_hist": [0.01741, 0.08153, 0.15857, 0.18355, 0.1776, 0.16679, 0.1302, 0.07233, 0.01193, 9e-05], "alpha_by_type": {"special": {"share": 0.00117, "mean": 0.52958, "std": 0.09309}, "whitespace": {"share": 0.02557, "mean": 0.52113, "std": 0.12892}, "digit": {"share": 0.02959, "mean": 0.47073, "std": 0.11879}, "word_start": {"share": 0.59438, "mean": 0.45846, "std": 0.1708}, "word_cont": {"share": 0.24239, "mean": 0.31152, "std": 0.14516}, "punct": {"share": 0.1069, "mean": 0.57953, "std": 0.16263}}, "alpha_p05": 0.15896, "alpha_p50": 0.43866, "alpha_p95": 0.71879}
{"step": 26110, "total_steps": 72479, "tokens": 13689159680, "loss": 4.655928015708923, "l_mix": 2.8824011087417603, "l_a": 2.9772080779075623, "l_c": 2.9289913177490234, "z_loss": 16.669886827468872, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17153829336166382, "tokens_per_second": 1451436.405581826, "eta_seconds": 16749.414702916147, "data_wait_seconds": 2.008676528930664e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 26120, "total_steps": 72479, "tokens": 13694402560, "loss": 4.605243682861328, "l_mix": 2.8501808643341064, "l_a": 2.948834091424942, "l_c": 2.8956887423992157, "z_loss": 17.058854341506958, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16205008327960968, "tokens_per_second": 1436695.611226831, "eta_seconds": 16917.617901849746, "data_wait_seconds": 1.4781951904296875e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 26130, "total_steps": 72479, "tokens": 13699645440, "loss": 2.883669141680002, "l_mix": 1.7805690094828606, "l_a": 1.85228999145329, "l_c": 1.8183102756738663, "z_loss": 19.200317859649658, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.33219587802886963, "tokens_per_second": 1326894.5869440143, "eta_seconds": 18313.60588181019, "data_wait_seconds": 1.755356788635254e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08236}}
{"step": 26140, "total_steps": 72479, "tokens": 13704888320, "loss": 2.910573460161686, "l_mix": 1.799175925552845, "l_a": 1.8822225406765938, "l_c": 1.8162410482764244, "z_loss": 18.584944009780884, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.717646598815918, "tokens_per_second": 1286513.656559012, "eta_seconds": 18884.355799984933, "data_wait_seconds": 2.467632293701172e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08236}}
{"step": 26150, "total_steps": 72479, "tokens": 13710131200, "loss": 4.923549950122833, "l_mix": 3.0508624017238617, "l_a": 3.1364789605140686, "l_c": 3.1000534296035767, "z_loss": 17.277488708496094, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18115027248859406, "tokens_per_second": 1299367.084072061, "eta_seconds": 18693.51551978588, "data_wait_seconds": 2.4765729904174805e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20769, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 26160, "total_steps": 72479, "tokens": 13715374080, "loss": 4.864292711019516, "l_mix": 3.0150874853134155, "l_a": 3.0929113626480103, "l_c": 3.0653668344020844, "z_loss": 17.21718955039978, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.7990339398384094, "tokens_per_second": 1456487.7434785936, "eta_seconds": 16673.326624774934, "data_wait_seconds": 2.4527311325073242e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 26170, "total_steps": 72479, "tokens": 13720616960, "loss": 5.0260045528411865, "l_mix": 3.1142081916332245, "l_a": 3.2105701565742493, "l_c": 3.1564351320266724, "z_loss": 16.94667887687683, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2980581820011139, "tokens_per_second": 1433056.073836733, "eta_seconds": 16942.290978884696, "data_wait_seconds": 2.816319465637207e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20768, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 26180, "total_steps": 72479, "tokens": 13725859840, "loss": 4.518210351467133, "l_mix": 2.7961798310279846, "l_a": 2.895576775074005, "l_c": 2.838908165693283, "z_loss": 16.850740432739258, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20343123376369476, "tokens_per_second": 1385905.1942168975, "eta_seconds": 17514.913872385026, "data_wait_seconds": 1.6748905181884766e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20764, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 26190, "total_steps": 72479, "tokens": 13731102720, "loss": 4.526321351528168, "l_mix": 2.8007474839687347, "l_a": 2.9012124836444855, "l_c": 2.845052480697632, "z_loss": 16.941888093948364, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16760654747486115, "tokens_per_second": 1309819.9934146355, "eta_seconds": 18528.322482490537, "data_wait_seconds": 2.294778823852539e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08232}}
{"step": 26200, "total_steps": 72479, "tokens": 13736345600, "loss": 4.748318076133728, "l_mix": 2.9394295811653137, "l_a": 3.038025349378586, "l_c": 2.9859980940818787, "z_loss": 16.8138906955719, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17845475673675537, "tokens_per_second": 1448440.2265549968, "eta_seconds": 16751.484740042688, "data_wait_seconds": 1.537799835205078e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.32015312, "gate_a2b/L1_absmax": 0.65115803, "gate_a2b/L1_rms": 0.33414432, "gate_a2b/L3_absmean": 0.33878103, "gate_a2b/L3_absmax": 0.52696228, "gate_a2b/L3_rms": 0.34331784, "gate_a2b/L5_absmean": 0.51426166, "gate_a2b/L5_absmax": 0.67074865, "gate_a2b/L5_rms": 0.51978129, "gate_c2b/L1_absmean": 0.01478355, "gate_c2b/L1_absmax": 0.43418807, "gate_c2b/L1_rms": 0.05367647, "gate_c2b/L3_absmean": 0.21097659, "gate_c2b/L3_absmax": 0.43671906, "gate_c2b/L3_rms": 0.21492781, "gate_c2b/L5_absmean": 0.52911824, "gate_c2b/L5_absmax": 0.71072763, "gate_c2b/L5_rms": 0.54095399, "gate_b2a/L7_absmean": 0.09026386, "gate_b2a/L7_absmax": 0.29052281, "gate_b2a/L7_rms": 0.09352524, "gate_b2a/L11_absmean": 0.13726479, "gate_b2a/L11_absmax": 0.48771444, "gate_b2a/L11_rms": 0.14030766, "gate_b2a/L14_absmean": 0.32709527, "gate_b2a/L14_absmax": 0.58109593, "gate_b2a/L14_rms": 0.32866061, "gate_b2c/L3_absmean": 0.10801014, "gate_b2c/L3_absmax": 0.63774061, "gate_b2c/L3_rms": 0.12298732, "gate_b2c/L5_absmean": 0.12731272, "gate_b2c/L5_absmax": 0.795578, "gate_b2c/L5_rms": 0.13587765, "gate_b2c/L6_absmean": 0.18682289, "gate_b2c/L6_absmax": 0.53426385, "gate_b2c/L6_rms": 0.19183201}, "streams": {"cos_a2b/round0": 0.38716, "cos_c2b/round0": 0.298481, "cos_b2a/round0": 0.109869, "cos_b2c/round0": 0.148203, "rms_a/round0": 0.826116, "rms_b/round0": 1.537172, "rms_c/round0": 1.244591, "cos_a2b/round1": 0.417621, "cos_c2b/round1": 0.269921, "cos_b2a/round1": 0.225171, "cos_b2c/round1": 0.132569, "rms_a/round1": 1.470157, "rms_b/round1": 3.090448, "rms_c/round1": 2.037679, "cos_a2b/round2": 0.543656, "cos_c2b/round2": 0.493513, "cos_b2a/round2": 0.064075, "cos_b2c/round2": 0.127799, "rms_a/round2": 3.345124, "rms_b/round2": 6.926264, "rms_c/round2": 3.319467}, "alpha_mean": 0.43291601405413616, "alpha_hist": [0.01872, 0.08244, 0.16315, 0.18559, 0.18188, 0.1647, 0.12428, 0.06821, 0.01083, 0.00018], "alpha_by_type": {"special": {"share": 0.00121, "mean": 0.54608, "std": 0.10008}, "whitespace": {"share": 0.02834, "mean": 0.50782, "std": 0.12049}, "digit": {"share": 0.03813, "mean": 0.44198, "std": 0.12785}, "word_start": {"share": 0.58075, "mean": 0.45553, "std": 0.16913}, "word_cont": {"share": 0.24704, "mean": 0.30426, "std": 0.13847}, "punct": {"share": 0.10454, "mean": 0.58638, "std": 0.16141}}, "alpha_p05": 0.15079, "alpha_p50": 0.43398, "alpha_p95": 0.72739}
{"step": 26210, "total_steps": 72479, "tokens": 13741588480, "loss": 4.642892301082611, "l_mix": 2.8734156489372253, "l_a": 2.9745136499404907, "l_c": 2.91810342669487, "z_loss": 16.91473913192749, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17177322506904602, "tokens_per_second": 1442132.3498553785, "eta_seconds": 16821.12011039257, "data_wait_seconds": 1.8298625946044922e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 26220, "total_steps": 72479, "tokens": 13746831360, "loss": 4.668403089046478, "l_mix": 2.889744281768799, "l_a": 2.9889776706695557, "l_c": 2.934203952550888, "z_loss": 17.042425870895386, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16277825832366943, "tokens_per_second": 1347160.144496866, "eta_seconds": 18003.0850014925, "data_wait_seconds": 2.2351741790771484e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 26230, "total_steps": 72479, "tokens": 13752074240, "loss": 4.687690496444702, "l_mix": 2.9022110998630524, "l_a": 3.0000988841056824, "l_c": 2.9457652866840363, "z_loss": 17.200168132781982, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18462836742401123, "tokens_per_second": 1384585.2745353598, "eta_seconds": 17512.677736759186, "data_wait_seconds": 1.817941665649414e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 26240, "total_steps": 72479, "tokens": 13757317120, "loss": 3.280978351831436, "l_mix": 2.0279026925563812, "l_a": 2.114413097500801, "l_c": 2.0559837073087692, "z_loss": 19.565301179885864, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2782523036003113, "tokens_per_second": 1339151.601655878, "eta_seconds": 18102.918894338607, "data_wait_seconds": 2.6047229766845703e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08236}}
{"step": 26250, "total_steps": 72479, "tokens": 13762560000, "loss": 3.14297091960907, "l_mix": 1.9423547834157944, "l_a": 2.0110277831554413, "l_c": 1.9845609664916992, "z_loss": 19.394763469696045, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.252056360244751, "tokens_per_second": 1461156.7678065717, "eta_seconds": 16587.754637980463, "data_wait_seconds": 2.3752450942993164e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08236}}
{"step": 26260, "total_steps": 72479, "tokens": 13767802880, "loss": 4.676034986972809, "l_mix": 2.8942639231681824, "l_a": 2.9946361780166626, "l_c": 2.938821941614151, "z_loss": 17.3359637260437, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17321564257144928, "tokens_per_second": 1399104.986641202, "eta_seconds": 17319.691733908654, "data_wait_seconds": 2.9742717742919922e-05, "mix": {"fineweb_hq": 0.5096, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 26270, "total_steps": 72479, "tokens": 13773045760, "loss": 4.575785577297211, "l_mix": 2.8321936428546906, "l_a": 2.9314842224121094, "l_c": 2.8748737275600433, "z_loss": 16.84482431411743, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15519487857818604, "tokens_per_second": 1423572.8189622296, "eta_seconds": 17018.324506688117, "data_wait_seconds": 1.8656253814697266e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 26280, "total_steps": 72479, "tokens": 13778288640, "loss": 4.585213720798492, "l_mix": 2.838299334049225, "l_a": 2.934045284986496, "l_c": 2.883392184972763, "z_loss": 16.829925775527954, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17225739359855652, "tokens_per_second": 1393005.9271186246, "eta_seconds": 17387.995873141288, "data_wait_seconds": 1.5944242477416992e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 26290, "total_steps": 72479, "tokens": 13783531520, "loss": 4.6784327030181885, "l_mix": 2.8961072862148285, "l_a": 2.994092881679535, "l_c": 2.941334754228592, "z_loss": 16.969589471817017, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.177863210439682, "tokens_per_second": 1377184.96197062, "eta_seconds": 17583.94050233364, "data_wait_seconds": 2.0056962966918945e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 26300, "total_steps": 72479, "tokens": 13788774400, "loss": 4.61945652961731, "l_mix": 2.8589518666267395, "l_a": 2.9587591886520386, "l_c": 2.903824269771576, "z_loss": 17.295039892196655, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16296933591365814, "tokens_per_second": 1473742.927366688, "eta_seconds": 16428.30313374996, "data_wait_seconds": 1.5348196029663086e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20042, "finemath": 0.08236}, "gates": {"gate_a2b/L1_absmean": 0.32105324, "gate_a2b/L1_absmax": 0.66008711, "gate_a2b/L1_rms": 0.3349815, "gate_a2b/L3_absmean": 0.33937481, "gate_a2b/L3_absmax": 0.52750951, "gate_a2b/L3_rms": 0.34379905, "gate_a2b/L5_absmean": 0.51559275, "gate_a2b/L5_absmax": 0.67175943, "gate_a2b/L5_rms": 0.52108407, "gate_c2b/L1_absmean": 0.01471862, "gate_c2b/L1_absmax": 0.42962947, "gate_c2b/L1_rms": 0.05355271, "gate_c2b/L3_absmean": 0.21102862, "gate_c2b/L3_absmax": 0.43566325, "gate_c2b/L3_rms": 0.21484235, "gate_c2b/L5_absmean": 0.52975839, "gate_c2b/L5_absmax": 0.705679, "gate_c2b/L5_rms": 0.5416171, "gate_b2a/L7_absmean": 0.0904289, "gate_b2a/L7_absmax": 0.29083875, "gate_b2a/L7_rms": 0.09362182, "gate_b2a/L11_absmean": 0.13712285, "gate_b2a/L11_absmax": 0.48761278, "gate_b2a/L11_rms": 0.14016449, "gate_b2a/L14_absmean": 0.32777774, "gate_b2a/L14_absmax": 0.57905012, "gate_b2a/L14_rms": 0.32937834, "gate_b2c/L3_absmean": 0.10779345, "gate_b2c/L3_absmax": 0.63966256, "gate_b2c/L3_rms": 0.12287592, "gate_b2c/L5_absmean": 0.12830323, "gate_b2c/L5_absmax": 0.79704982, "gate_b2c/L5_rms": 0.13685822, "gate_b2c/L6_absmean": 0.18720296, "gate_b2c/L6_absmax": 0.54453665, "gate_b2c/L6_rms": 0.19236916}, "streams": {"cos_a2b/round0": 0.387687, "cos_c2b/round0": 0.307011, "cos_b2a/round0": 0.114381, "cos_b2c/round0": 0.153325, "rms_a/round0": 0.824791, "rms_b/round0": 1.534297, "rms_c/round0": 1.246231, "cos_a2b/round1": 0.420179, "cos_c2b/round1": 0.268185, "cos_b2a/round1": 0.231, "cos_b2c/round1": 0.121712, "rms_a/round1": 1.474226, "rms_b/round1": 3.113394, "rms_c/round1": 2.03976, "cos_a2b/round2": 0.533365, "cos_c2b/round2": 0.502397, "cos_b2a/round2": 0.06433, "cos_b2c/round2": 0.133912, "rms_a/round2": 3.315345, "rms_b/round2": 6.99072, "rms_c/round2": 3.348895}, "alpha_mean": 0.43139962433671997, "alpha_hist": [0.01859, 0.08298, 0.17204, 0.1944, 0.17212, 0.15083, 0.1183, 0.07445, 0.01607, 0.00023], "alpha_by_type": {"special": {"share": 0.00119, "mean": 0.59013, "std": 0.0923}, "whitespace": {"share": 0.02655, "mean": 0.54898, "std": 0.12165}, "digit": {"share": 0.03464, "mean": 0.4846, "std": 0.13102}, "word_start": {"share": 0.58176, "mean": 0.45155, "std": 0.17355}, "word_cont": {"share": 0.25674, "mean": 0.30554, "std": 0.13936}, "punct": {"share": 0.09912, "mean": 0.58713, "std": 0.1733}}, "alpha_p05": 0.15896, "alpha_p50": 0.40145, "alpha_p95": 0.73412}
{"step": 26310, "total_steps": 72479, "tokens": 13794017280, "loss": 4.659700632095337, "l_mix": 2.885351538658142, "l_a": 2.979046791791916, "l_c": 2.929817706346512, "z_loss": 16.89779043197632, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20159180462360382, "tokens_per_second": 1451736.6095396637, "eta_seconds": 16673.72201881409, "data_wait_seconds": 1.6689300537109375e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20768, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 26320, "total_steps": 72479, "tokens": 13799260160, "loss": 4.748216807842255, "l_mix": 2.9404972195625305, "l_a": 3.0372528433799744, "l_c": 2.982869416475296, "z_loss": 16.8266441822052, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15283015370368958, "tokens_per_second": 1416567.7509696365, "eta_seconds": 17083.976234412192, "data_wait_seconds": 3.0219554901123047e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 26330, "total_steps": 72479, "tokens": 13804503040, "loss": 4.596251726150513, "l_mix": 2.8455095887184143, "l_a": 2.9417540431022644, "l_c": 2.888418197631836, "z_loss": 16.90462303161621, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16267631947994232, "tokens_per_second": 1343696.6479259934, "eta_seconds": 18006.56937661171, "data_wait_seconds": 2.9087066650390625e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 26340, "total_steps": 72479, "tokens": 13809745920, "loss": 4.964736640453339, "l_mix": 3.0770888328552246, "l_a": 3.1612752079963684, "l_c": 3.1251629889011383, "z_loss": 17.162301301956177, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16899818181991577, "tokens_per_second": 1406243.4987837945, "eta_seconds": 17201.945504403113, "data_wait_seconds": 2.428889274597168e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 26350, "total_steps": 72479, "tokens": 13814988800, "loss": 5.038800239562988, "l_mix": 3.1237205862998962, "l_a": 3.2064746618270874, "l_c": 3.171503722667694, "z_loss": 16.861072421073914, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21451131999492645, "tokens_per_second": 1437790.9014834, "eta_seconds": 16820.861174631118, "data_wait_seconds": 2.9206275939941406e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 26360, "total_steps": 72479, "tokens": 13820231680, "loss": 3.151402533054352, "l_mix": 1.9483037441968918, "l_a": 2.026701867580414, "l_c": 1.9772346019744873, "z_loss": 19.177439212799072, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2869599163532257, "tokens_per_second": 1365238.6811499223, "eta_seconds": 17710.923815631868, "data_wait_seconds": 2.2590160369873047e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08236}}
{"step": 26370, "total_steps": 72479, "tokens": 13825474560, "loss": 4.621114373207092, "l_mix": 2.8606932759284973, "l_a": 2.959802597761154, "l_c": 2.902608335018158, "z_loss": 16.97882866859436, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1644020825624466, "tokens_per_second": 1435321.8164729078, "eta_seconds": 16842.491429138187, "data_wait_seconds": 1.633167266845703e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 26380, "total_steps": 72479, "tokens": 13830717440, "loss": 4.653570353984833, "l_mix": 2.881512403488159, "l_a": 2.978953331708908, "l_c": 2.9223304092884064, "z_loss": 16.72642946243286, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15349632501602173, "tokens_per_second": 1432736.3806308338, "eta_seconds": 16869.225098729134, "data_wait_seconds": 2.390146255493164e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 26390, "total_steps": 72479, "tokens": 13835960320, "loss": 4.578200280666351, "l_mix": 2.834011524915695, "l_a": 2.930272340774536, "l_c": 2.877983957529068, "z_loss": 17.11775255203247, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1660817563533783, "tokens_per_second": 1358357.687903233, "eta_seconds": 17789.062370824813, "data_wait_seconds": 2.2619962692260742e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 26400, "total_steps": 72479, "tokens": 13841203200, "loss": 4.453332424163818, "l_mix": 2.7565101981163025, "l_a": 2.8512727320194244, "l_c": 2.799217402935028, "z_loss": 16.75214695930481, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17496272921562195, "tokens_per_second": 1473235.8325164968, "eta_seconds": 16398.370321154594, "data_wait_seconds": 2.6345252990722656e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.32099757, "gate_a2b/L1_absmax": 0.66019422, "gate_a2b/L1_rms": 0.33505884, "gate_a2b/L3_absmean": 0.33941168, "gate_a2b/L3_absmax": 0.53286344, "gate_a2b/L3_rms": 0.34385595, "gate_a2b/L5_absmean": 0.51546478, "gate_a2b/L5_absmax": 0.66821277, "gate_a2b/L5_rms": 0.52101451, "gate_c2b/L1_absmean": 0.01464798, "gate_c2b/L1_absmax": 0.43097296, "gate_c2b/L1_rms": 0.05355984, "gate_c2b/L3_absmean": 0.21061413, "gate_c2b/L3_absmax": 0.43528482, "gate_c2b/L3_rms": 0.21456431, "gate_c2b/L5_absmean": 0.52981061, "gate_c2b/L5_absmax": 0.71118355, "gate_c2b/L5_rms": 0.54170245, "gate_b2a/L7_absmean": 0.09027794, "gate_b2a/L7_absmax": 0.28662017, "gate_b2a/L7_rms": 0.093489, "gate_b2a/L11_absmean": 0.13700879, "gate_b2a/L11_absmax": 0.48766449, "gate_b2a/L11_rms": 0.14010948, "gate_b2a/L14_absmean": 0.32763413, "gate_b2a/L14_absmax": 0.57646626, "gate_b2a/L14_rms": 0.32922632, "gate_b2c/L3_absmean": 0.10735242, "gate_b2c/L3_absmax": 0.6397056, "gate_b2c/L3_rms": 0.12242905, "gate_b2c/L5_absmean": 0.12782755, "gate_b2c/L5_absmax": 0.79888707, "gate_b2c/L5_rms": 0.13646762, "gate_b2c/L6_absmean": 0.18724178, "gate_b2c/L6_absmax": 0.54744923, "gate_b2c/L6_rms": 0.19239464}, "streams": {"cos_a2b/round0": 0.391828, "cos_c2b/round0": 0.295664, "cos_b2a/round0": 0.105672, "cos_b2c/round0": 0.145763, "rms_a/round0": 0.836514, "rms_b/round0": 1.53955, "rms_c/round0": 1.24923, "cos_a2b/round1": 0.411538, "cos_c2b/round1": 0.265365, "cos_b2a/round1": 0.226015, "cos_b2c/round1": 0.115513, "rms_a/round1": 1.459942, "rms_b/round1": 3.096446, "rms_c/round1": 2.056051, "cos_a2b/round2": 0.527571, "cos_c2b/round2": 0.503805, "cos_b2a/round2": 0.06154, "cos_b2c/round2": 0.126872, "rms_a/round2": 3.310937, "rms_b/round2": 7.000705, "rms_c/round2": 3.347429}, "alpha_mean": 0.42929552672519833, "alpha_hist": [0.01675, 0.08429, 0.18532, 0.19611, 0.1673, 0.13835, 0.11479, 0.07797, 0.01889, 0.00023], "alpha_by_type": {"special": {"share": 0.00099, "mean": 0.53566, "std": 0.11016}, "whitespace": {"share": 0.02374, "mean": 0.48621, "std": 0.12915}, "digit": {"share": 0.04787, "mean": 0.36711, "std": 0.13828}, "word_start": {"share": 0.53416, "mean": 0.47258, "std": 0.17434}, "word_cont": {"share": 0.28322, "mean": 0.29933, "std": 0.13281}, "punct": {"share": 0.11002, "mean": 0.5675, "std": 0.18164}}, "alpha_p05": 0.14731, "alpha_p50": 0.40263, "alpha_p95": 0.74205}
{"step": 26410, "total_steps": 72479, "tokens": 13846446080, "loss": 4.576878666877747, "l_mix": 2.833172917366028, "l_a": 2.9312380254268646, "l_c": 2.8754660189151764, "z_loss": 16.94560146331787, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18590539693832397, "tokens_per_second": 1366243.2241060662, "eta_seconds": 17678.714482045172, "data_wait_seconds": 1.5854835510253906e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08236}}
{"step": 26420, "total_steps": 72479, "tokens": 13851688960, "loss": 4.559820592403412, "l_mix": 2.822573274374008, "l_a": 2.9193490743637085, "l_c": 2.865694373846054, "z_loss": 17.341203927993774, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22757107019424438, "tokens_per_second": 1445403.282018006, "eta_seconds": 16706.88125066757, "data_wait_seconds": 2.0831823348999023e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08236}}
{"step": 26430, "total_steps": 72479, "tokens": 13856931840, "loss": 4.848830938339233, "l_mix": 3.0046051740646362, "l_a": 3.093476176261902, "l_c": 3.0482783913612366, "z_loss": 16.99316120147705, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.153199702501297, "tokens_per_second": 1481823.9972571384, "eta_seconds": 16292.716379737854, "data_wait_seconds": 1.531839370727539e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20768, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 26440, "total_steps": 72479, "tokens": 13862174720, "loss": 4.91583377122879, "l_mix": 3.046948105096817, "l_a": 3.1289196610450745, "l_c": 3.0950435996055603, "z_loss": 16.967347383499146, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16008040308952332, "tokens_per_second": 1407699.8482478482, "eta_seconds": 17146.904762434962, "data_wait_seconds": 2.4020671844482422e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20768, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 26450, "total_steps": 72479, "tokens": 13867417600, "loss": 4.917939841747284, "l_mix": 3.0476096272468567, "l_a": 3.136125683784485, "l_c": 3.092761069536209, "z_loss": 16.64137864112854, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1581614464521408, "tokens_per_second": 1452170.3182642926, "eta_seconds": 16618.196948719025, "data_wait_seconds": 2.6732683181762695e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 26460, "total_steps": 72479, "tokens": 13872660480, "loss": 4.9798309206962585, "l_mix": 3.086754471063614, "l_a": 3.1712806820869446, "l_c": 3.133234143257141, "z_loss": 17.218770265579224, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22057493031024933, "tokens_per_second": 1395565.8509421968, "eta_seconds": 17288.477971649172, "data_wait_seconds": 7.432699203491211e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 26470, "total_steps": 72479, "tokens": 13877903360, "loss": 5.027034103870392, "l_mix": 3.1158501505851746, "l_a": 3.201535642147064, "l_c": 3.1634068489074707, "z_loss": 17.010963916778564, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1847064197063446, "tokens_per_second": 1392559.6530733004, "eta_seconds": 17322.03467102051, "data_wait_seconds": 2.086162567138672e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08236}}
{"step": 26480, "total_steps": 72479, "tokens": 13883146240, "loss": 4.56024706363678, "l_mix": 2.82319039106369, "l_a": 2.9184339940547943, "l_c": 2.866168826818466, "z_loss": 16.75913906097412, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1533341258764267, "tokens_per_second": 1440174.3591632433, "eta_seconds": 16745.697184896468, "data_wait_seconds": 2.1070241928100586e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 26490, "total_steps": 72479, "tokens": 13888389120, "loss": 4.667352735996246, "l_mix": 2.889610171318054, "l_a": 2.985822767019272, "l_c": 2.934258908033371, "z_loss": 17.179258584976196, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1789030283689499, "tokens_per_second": 1419271.3293778314, "eta_seconds": 16988.633767843246, "data_wait_seconds": 1.77919864654541e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 26500, "total_steps": 72479, "tokens": 13893632000, "loss": 4.580457925796509, "l_mix": 2.834653288125992, "l_a": 2.9314535558223724, "l_c": 2.8822371065616608, "z_loss": 16.97384810447693, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19550594687461853, "tokens_per_second": 1343288.5429951022, "eta_seconds": 17945.68864426613, "data_wait_seconds": 2.5331974029541016e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.3210783, "gate_a2b/L1_absmax": 0.6670084, "gate_a2b/L1_rms": 0.33516669, "gate_a2b/L3_absmean": 0.33899736, "gate_a2b/L3_absmax": 0.53823632, "gate_a2b/L3_rms": 0.34333292, "gate_a2b/L5_absmean": 0.51526558, "gate_a2b/L5_absmax": 0.67235768, "gate_a2b/L5_rms": 0.5208233, "gate_c2b/L1_absmean": 0.01468792, "gate_c2b/L1_absmax": 0.42955992, "gate_c2b/L1_rms": 0.0538071, "gate_c2b/L3_absmean": 0.21098712, "gate_c2b/L3_absmax": 0.43960306, "gate_c2b/L3_rms": 0.2148862, "gate_c2b/L5_absmean": 0.52981305, "gate_c2b/L5_absmax": 0.70713562, "gate_c2b/L5_rms": 0.541646, "gate_b2a/L7_absmean": 0.09037289, "gate_b2a/L7_absmax": 0.28799614, "gate_b2a/L7_rms": 0.09361436, "gate_b2a/L11_absmean": 0.13677885, "gate_b2a/L11_absmax": 0.48939943, "gate_b2a/L11_rms": 0.1398835, "gate_b2a/L14_absmean": 0.32736933, "gate_b2a/L14_absmax": 0.57779133, "gate_b2a/L14_rms": 0.32897788, "gate_b2c/L3_absmean": 0.10782507, "gate_b2c/L3_absmax": 0.64071923, "gate_b2c/L3_rms": 0.12299433, "gate_b2c/L5_absmean": 0.1277912, "gate_b2c/L5_absmax": 0.79888421, "gate_b2c/L5_rms": 0.13638093, "gate_b2c/L6_absmean": 0.18667579, "gate_b2c/L6_absmax": 0.54385918, "gate_b2c/L6_rms": 0.19185226}, "streams": {"cos_a2b/round0": 0.387439, "cos_c2b/round0": 0.298738, "cos_b2a/round0": 0.107141, "cos_b2c/round0": 0.150349, "rms_a/round0": 0.825912, "rms_b/round0": 1.522533, "rms_c/round0": 1.245347, "cos_a2b/round1": 0.415499, "cos_c2b/round1": 0.279809, "cos_b2a/round1": 0.224252, "cos_b2c/round1": 0.119385, "rms_a/round1": 1.467553, "rms_b/round1": 3.066854, "rms_c/round1": 2.043403, "cos_a2b/round2": 0.537011, "cos_c2b/round2": 0.501147, "cos_b2a/round2": 0.064435, "cos_b2c/round2": 0.133865, "rms_a/round2": 3.330727, "rms_b/round2": 6.882929, "rms_c/round2": 3.312083}, "alpha_mean": 0.43011280221601567, "alpha_hist": [0.01434, 0.0692, 0.15573, 0.20287, 0.20929, 0.18007, 0.11201, 0.04991, 0.00655, 3e-05], "alpha_by_type": {"special": {"share": 0.00096, "mean": 0.5068, "std": 0.09611}, "whitespace": {"share": 0.02768, "mean": 0.50165, "std": 0.11532}, "digit": {"share": 0.03532, "mean": 0.43207, "std": 0.11031}, "word_start": {"share": 0.57397, "mean": 0.45065, "std": 0.1561}, "word_cont": {"share": 0.25732, "mean": 0.32122, "std": 0.13533}, "punct": {"share": 0.10474, "mean": 0.56484, "std": 0.15316}}, "alpha_p05": 0.14955, "alpha_p50": 0.43975, "alpha_p95": 0.70679}
{"step": 26510, "total_steps": 72479, "tokens": 13898874880, "loss": 4.542571723461151, "l_mix": 2.812019258737564, "l_a": 2.9068769812583923, "l_c": 2.8559711575508118, "z_loss": 16.978907823562622, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16269338130950928, "tokens_per_second": 1384550.8398682203, "eta_seconds": 17407.085661292076, "data_wait_seconds": 2.5063753128051758e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 26520, "total_steps": 72479, "tokens": 13904117760, "loss": 4.593015611171722, "l_mix": 2.8432587385177612, "l_a": 2.9380803406238556, "l_c": 2.8887555301189423, "z_loss": 17.060108423233032, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17564284801483154, "tokens_per_second": 1432207.7620749664, "eta_seconds": 16824.20164871216, "data_wait_seconds": 1.7344951629638672e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 26530, "total_steps": 72479, "tokens": 13909360640, "loss": 4.553662061691284, "l_mix": 2.8192315101623535, "l_a": 2.9160212576389313, "l_c": 2.8597943782806396, "z_loss": 16.858216524124146, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20083537697792053, "tokens_per_second": 1407128.0395240746, "eta_seconds": 17120.33918402195, "data_wait_seconds": 2.148747444152832e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20041, "finemath": 0.08236}}
{"step": 26540, "total_steps": 72479, "tokens": 13914603520, "loss": 3.3925074338912964, "l_mix": 2.086288034915924, "l_a": 2.206418067216873, "l_c": 2.142389476299286, "z_loss": 15.770838737487793, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19153867661952972, "tokens_per_second": 1456846.5968363774, "eta_seconds": 16532.465727210045, "data_wait_seconds": 2.1517276763916016e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20045, "finemath": 0.08233}}
{"step": 26550, "total_steps": 72479, "tokens": 13919846400, "loss": 3.44884917140007, "l_mix": 2.1196602284908295, "l_a": 2.2427320182323456, "l_c": 2.1828160881996155, "z_loss": 15.24472713470459, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22472302615642548, "tokens_per_second": 1444732.2855633285, "eta_seconds": 16667.464133405687, "data_wait_seconds": 1.8209218978881836e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20045, "finemath": 0.08234}}
{"step": 26560, "total_steps": 72479, "tokens": 13925089280, "loss": 4.759029567241669, "l_mix": 2.9485643804073334, "l_a": 3.035895586013794, "l_c": 2.9933237731456757, "z_loss": 16.992793321609497, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18460588157176971, "tokens_per_second": 1407411.00334927, "eta_seconds": 17105.72150900364, "data_wait_seconds": 1.5437602996826172e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.20041, "finemath": 0.08234}}
{"step": 26570, "total_steps": 72479, "tokens": 13930332160, "loss": 4.919006943702698, "l_mix": 3.0487164556980133, "l_a": 3.1308344900608063, "l_c": 3.0978539884090424, "z_loss": 16.839694499969482, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.232606440782547, "tokens_per_second": 1458405.3501303068, "eta_seconds": 16504.01089782715, "data_wait_seconds": 1.913309097290039e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 26580, "total_steps": 72479, "tokens": 13935575040, "loss": 5.160094082355499, "l_mix": 3.1984239518642426, "l_a": 3.2870190143585205, "l_c": 3.246447652578354, "z_loss": 16.299283385276794, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1972513347864151, "tokens_per_second": 1416128.962294983, "eta_seconds": 16993.010914063456, "data_wait_seconds": 2.0742416381835938e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08236}}
{"step": 26590, "total_steps": 72479, "tokens": 13940817920, "loss": 4.573465347290039, "l_mix": 2.8305357098579407, "l_a": 2.9288016259670258, "l_c": 2.87517186999321, "z_loss": 17.3751163482666, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18065395951271057, "tokens_per_second": 1434824.1485018807, "eta_seconds": 16767.944738817216, "data_wait_seconds": 2.086162567138672e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08232}}
{"step": 26600, "total_steps": 72479, "tokens": 13946060800, "loss": 4.676328182220459, "l_mix": 2.8957497775554657, "l_a": 2.9909094870090485, "l_c": 2.9387601912021637, "z_loss": 16.77444624900818, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17576730251312256, "tokens_per_second": 1349701.1389123008, "eta_seconds": 17821.58172540665, "data_wait_seconds": 1.8298625946044922e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.32261962, "gate_a2b/L1_absmax": 0.65900129, "gate_a2b/L1_rms": 0.3364912, "gate_a2b/L3_absmean": 0.33907148, "gate_a2b/L3_absmax": 0.54430592, "gate_a2b/L3_rms": 0.34347084, "gate_a2b/L5_absmean": 0.5157631, "gate_a2b/L5_absmax": 0.6741029, "gate_a2b/L5_rms": 0.52123225, "gate_c2b/L1_absmean": 0.01530715, "gate_c2b/L1_absmax": 0.42757311, "gate_c2b/L1_rms": 0.05340412, "gate_c2b/L3_absmean": 0.21071994, "gate_c2b/L3_absmax": 0.42877755, "gate_c2b/L3_rms": 0.21458489, "gate_c2b/L5_absmean": 0.53026527, "gate_c2b/L5_absmax": 0.70653141, "gate_c2b/L5_rms": 0.54200977, "gate_b2a/L7_absmean": 0.0909581, "gate_b2a/L7_absmax": 0.28821522, "gate_b2a/L7_rms": 0.09419502, "gate_b2a/L11_absmean": 0.13754618, "gate_b2a/L11_absmax": 0.4916144, "gate_b2a/L11_rms": 0.14064954, "gate_b2a/L14_absmean": 0.32797021, "gate_b2a/L14_absmax": 0.57687074, "gate_b2a/L14_rms": 0.32954094, "gate_b2c/L3_absmean": 0.1076992, "gate_b2c/L3_absmax": 0.63898987, "gate_b2c/L3_rms": 0.12277117, "gate_b2c/L5_absmean": 0.12830459, "gate_b2c/L5_absmax": 0.80215621, "gate_b2c/L5_rms": 0.13689172, "gate_b2c/L6_absmean": 0.18713763, "gate_b2c/L6_absmax": 0.54677749, "gate_b2c/L6_rms": 0.1923372}, "streams": {"cos_a2b/round0": 0.389807, "cos_c2b/round0": 0.298014, "cos_b2a/round0": 0.113909, "cos_b2c/round0": 0.148136, "rms_a/round0": 0.828949, "rms_b/round0": 1.53832, "rms_c/round0": 1.250618, "cos_a2b/round1": 0.419798, "cos_c2b/round1": 0.275492, "cos_b2a/round1": 0.232753, "cos_b2c/round1": 0.121261, "rms_a/round1": 1.456756, "rms_b/round1": 3.077756, "rms_c/round1": 2.045313, "cos_a2b/round2": 0.534847, "cos_c2b/round2": 0.510915, "cos_b2a/round2": 0.06925, "cos_b2c/round2": 0.134677, "rms_a/round2": 3.324323, "rms_b/round2": 6.972662, "rms_c/round2": 3.359421}, "alpha_mean": 0.4375231120400116, "alpha_hist": [0.01132, 0.06358, 0.15831, 0.2065, 0.19762, 0.16972, 0.1219, 0.06352, 0.00746, 6e-05], "alpha_by_type": {"special": {"share": 0.00092, "mean": 0.52991, "std": 0.11751}, "whitespace": {"share": 0.02235, "mean": 0.51878, "std": 0.11855}, "digit": {"share": 0.02597, "mean": 0.5567, "std": 0.13816}, "word_start": {"share": 0.57893, "mean": 0.45259, "std": 0.1574}, "word_cont": {"share": 0.26674, "mean": 0.33035, "std": 0.13661}, "punct": {"share": 0.10509, "mean": 0.57903, "std": 0.15435}}, "alpha_p05": 0.17414, "alpha_p50": 0.42441, "alpha_p95": 0.7237}
{"step": 26610, "total_steps": 72479, "tokens": 13951303680, "loss": 4.648891925811768, "l_mix": 2.8779939711093903, "l_a": 2.976684957742691, "l_c": 2.9207016229629517, "z_loss": 16.8185133934021, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16142047941684723, "tokens_per_second": 1463048.7334795834, "eta_seconds": 16437.296804738045, "data_wait_seconds": 1.9073486328125e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08234}}
{"step": 26620, "total_steps": 72479, "tokens": 13956546560, "loss": 4.568629205226898, "l_mix": 2.8271422684192657, "l_a": 2.927416354417801, "l_c": 2.871892988681793, "z_loss": 16.9408221244812, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17587345838546753, "tokens_per_second": 1387671.9840331695, "eta_seconds": 17326.373717021943, "data_wait_seconds": 1.7434358596801758e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20041, "finemath": 0.08234}}
{"step": 26630, "total_steps": 72479, "tokens": 13961789440, "loss": 4.627446115016937, "l_mix": 2.864610731601715, "l_a": 2.9606873691082, "l_c": 2.9097816348075867, "z_loss": 16.945931434631348, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1654912531375885, "tokens_per_second": 1375707.4624690453, "eta_seconds": 17473.250067901612, "data_wait_seconds": 1.817941665649414e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 26640, "total_steps": 72479, "tokens": 13967032320, "loss": 4.66373473405838, "l_mix": 2.8870854675769806, "l_a": 2.9861139357089996, "l_c": 2.9303290247917175, "z_loss": 17.162724018096924, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20986664295196533, "tokens_per_second": 1476256.1945055386, "eta_seconds": 16279.58461508751, "data_wait_seconds": 1.570582389831543e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20041, "finemath": 0.08236}}
{"step": 26650, "total_steps": 72479, "tokens": 13972275200, "loss": 3.431126147508621, "l_mix": 2.1087766587734222, "l_a": 2.23777174949646, "l_c": 2.164803296327591, "z_loss": 15.768596053123474, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19794292747974396, "tokens_per_second": 1453128.6742419312, "eta_seconds": 16535.0771600008, "data_wait_seconds": 2.911686897277832e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20045, "finemath": 0.08233}}
{"step": 26660, "total_steps": 72479, "tokens": 13977518080, "loss": 3.460373640060425, "l_mix": 2.1280263662338257, "l_a": 2.252496123313904, "l_c": 2.1833964586257935, "z_loss": 15.794728636741638, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17698445916175842, "tokens_per_second": 1387911.2601722279, "eta_seconds": 17308.276516914368, "data_wait_seconds": 2.6911497116088867e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20045, "finemath": 0.08233}}
{"step": 26670, "total_steps": 72479, "tokens": 13982760960, "loss": 3.4052943289279938, "l_mix": 2.093205213546753, "l_a": 2.218762904405594, "l_c": 2.1496417820453644, "z_loss": 15.676915764808655, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19842295348644257, "tokens_per_second": 1451503.2771693093, "eta_seconds": 16546.369112467768, "data_wait_seconds": 2.2798776626586914e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20045, "finemath": 0.08234}}
{"step": 26680, "total_steps": 72479, "tokens": 13988003840, "loss": 3.4241145253181458, "l_mix": 2.1047695875167847, "l_a": 2.229325830936432, "l_c": 2.163356453180313, "z_loss": 15.402809381484985, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22144857048988342, "tokens_per_second": 1426173.4468976515, "eta_seconds": 16836.567925333977, "data_wait_seconds": 1.633167266845703e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20045, "finemath": 0.08235}}
{"step": 26690, "total_steps": 72479, "tokens": 13993246720, "loss": 3.4593039751052856, "l_mix": 2.1266273856163025, "l_a": 2.2519510686397552, "l_c": 2.1851204335689545, "z_loss": 15.551280617713928, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21088887751102448, "tokens_per_second": 1424192.9387019982, "eta_seconds": 16856.299859118462, "data_wait_seconds": 1.8924474716186523e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20764, "cosmopedia_v2": 0.20045, "finemath": 0.08235}}
{"step": 26700, "total_steps": 72479, "tokens": 13998489600, "loss": 4.888435482978821, "l_mix": 3.028585910797119, "l_a": 3.120943546295166, "l_c": 3.0729065537452698, "z_loss": 16.943610668182373, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19399695098400116, "tokens_per_second": 1446069.9571865234, "eta_seconds": 16597.66198220253, "data_wait_seconds": 1.8090009689331055e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20768, "cosmopedia_v2": 0.20041, "finemath": 0.08236}, "gates": {"gate_a2b/L1_absmean": 0.32077077, "gate_a2b/L1_absmax": 0.6452564, "gate_a2b/L1_rms": 0.33459616, "gate_a2b/L3_absmean": 0.33802146, "gate_a2b/L3_absmax": 0.54260772, "gate_a2b/L3_rms": 0.34246719, "gate_a2b/L5_absmean": 0.51374292, "gate_a2b/L5_absmax": 0.67178541, "gate_a2b/L5_rms": 0.5191986, "gate_c2b/L1_absmean": 0.01505859, "gate_c2b/L1_absmax": 0.43222529, "gate_c2b/L1_rms": 0.0540976, "gate_c2b/L3_absmean": 0.21143846, "gate_c2b/L3_absmax": 0.43693775, "gate_c2b/L3_rms": 0.21542805, "gate_c2b/L5_absmean": 0.52789098, "gate_c2b/L5_absmax": 0.70242554, "gate_c2b/L5_rms": 0.53946441, "gate_b2a/L7_absmean": 0.09139171, "gate_b2a/L7_absmax": 0.28667465, "gate_b2a/L7_rms": 0.09461604, "gate_b2a/L11_absmean": 0.13755056, "gate_b2a/L11_absmax": 0.49427643, "gate_b2a/L11_rms": 0.14079539, "gate_b2a/L14_absmean": 0.32673526, "gate_b2a/L14_absmax": 0.57484537, "gate_b2a/L14_rms": 0.3284713, "gate_b2c/L3_absmean": 0.10729825, "gate_b2c/L3_absmax": 0.65309393, "gate_b2c/L3_rms": 0.12238096, "gate_b2c/L5_absmean": 0.12788652, "gate_b2c/L5_absmax": 0.80177486, "gate_b2c/L5_rms": 0.13661554, "gate_b2c/L6_absmean": 0.18638173, "gate_b2c/L6_absmax": 0.55451787, "gate_b2c/L6_rms": 0.19163994}, "streams": {"cos_a2b/round0": 0.383347, "cos_c2b/round0": 0.29521, "cos_b2a/round0": 0.111403, "cos_b2c/round0": 0.150451, "rms_a/round0": 0.823877, "rms_b/round0": 1.547692, "rms_c/round0": 1.245939, "cos_a2b/round1": 0.420684, "cos_c2b/round1": 0.281417, "cos_b2a/round1": 0.232989, "cos_b2c/round1": 0.127993, "rms_a/round1": 1.45422, "rms_b/round1": 3.055074, "rms_c/round1": 2.051487, "cos_a2b/round2": 0.553884, "cos_c2b/round2": 0.499823, "cos_b2a/round2": 0.067862, "cos_b2c/round2": 0.136723, "rms_a/round2": 3.299831, "rms_b/round2": 6.758037, "rms_c/round2": 3.331098}, "alpha_mean": 0.47171301682722344, "alpha_hist": [0.00655, 0.04213, 0.12444, 0.19196, 0.20474, 0.18094, 0.14001, 0.0881, 0.02087, 0.00026], "alpha_by_type": {"special": {"share": 0.00067, "mean": 0.52474, "std": 0.1279}, "whitespace": {"share": 0.04347, "mean": 0.51694, "std": 0.12026}, "digit": {"share": 0.0177, "mean": 0.4988, "std": 0.12612}, "word_start": {"share": 0.57518, "mean": 0.49818, "std": 0.16126}, "word_cont": {"share": 0.2625, "mean": 0.35463, "std": 0.13243}, "punct": {"share": 0.10048, "mean": 0.60139, "std": 0.16843}}, "alpha_p05": 0.2053, "alpha_p50": 0.4632, "alpha_p95": 0.75419}
{"step": 26710, "total_steps": 72479, "tokens": 14003732480, "loss": 4.533426463603973, "l_mix": 2.805535137653351, "l_a": 2.904811978340149, "l_c": 2.8490668535232544, "z_loss": 17.27650213241577, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15699553489685059, "tokens_per_second": 1155529.7454340768, "eta_seconds": 20766.352027559282, "data_wait_seconds": 1.5944242477416992e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20768, "cosmopedia_v2": 0.20041, "finemath": 0.08233}}
{"step": 26720, "total_steps": 72479, "tokens": 14008975360, "loss": 4.594143271446228, "l_mix": 2.8431951999664307, "l_a": 2.9414535462856293, "l_c": 2.8893780410289764, "z_loss": 16.984407424926758, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.14300014078617096, "tokens_per_second": 1449212.4648563103, "eta_seconds": 16554.43571856022, "data_wait_seconds": 2.5451183319091797e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08234}}
{"step": 26730, "total_steps": 72479, "tokens": 14014218240, "loss": 4.568160533905029, "l_mix": 2.827232211828232, "l_a": 2.927007555961609, "l_c": 2.8704177737236023, "z_loss": 17.006351947784424, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.14717291295528412, "tokens_per_second": 1452222.679946582, "eta_seconds": 16516.51089272499, "data_wait_seconds": 1.767277717590332e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08234}}
{"step": 26740, "total_steps": 72479, "tokens": 14019461120, "loss": 4.649775147438049, "l_mix": 2.8781309127807617, "l_a": 2.9753156304359436, "l_c": 2.9243513345718384, "z_loss": 17.439147472381592, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15419742465019226, "tokens_per_second": 1398015.2941840636, "eta_seconds": 17153.180606651305, "data_wait_seconds": 1.7434358596801758e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 26750, "total_steps": 72479, "tokens": 14024704000, "loss": 4.656563103199005, "l_mix": 2.882069945335388, "l_a": 2.980952113866806, "l_c": 2.928422123193741, "z_loss": 16.807564973831177, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20092451572418213, "tokens_per_second": 1423365.311065464, "eta_seconds": 16844.00045835972, "data_wait_seconds": 2.014636993408203e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20041, "finemath": 0.08236}}
{"step": 26760, "total_steps": 72479, "tokens": 14029946880, "loss": 3.4134433567523956, "l_mix": 2.097496211528778, "l_a": 2.2279273867607117, "l_c": 2.1533798575401306, "z_loss": 15.548851490020752, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.23666344583034515, "tokens_per_second": 1057777.6454416856, "eta_seconds": 22660.644394683837, "data_wait_seconds": 1.6927719116210938e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20045, "finemath": 0.08232}}
{"step": 26770, "total_steps": 72479, "tokens": 14035189760, "loss": 3.3674547374248505, "l_mix": 2.0701981484889984, "l_a": 2.193657338619232, "l_c": 2.125205874443054, "z_loss": 15.975419521331787, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20078282058238983, "tokens_per_second": 1384368.4082991222, "eta_seconds": 17310.91236143112, "data_wait_seconds": 1.7821788787841797e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20045, "finemath": 0.08233}}
{"step": 26780, "total_steps": 72479, "tokens": 14040432640, "loss": 3.4012422263622284, "l_mix": 2.0918449461460114, "l_a": 2.212797462940216, "l_c": 2.146575152873993, "z_loss": 15.854644179344177, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18404324352741241, "tokens_per_second": 1336504.3963795519, "eta_seconds": 17926.942385601997, "data_wait_seconds": 3.5762786865234375e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20045, "finemath": 0.08234}}
{"step": 26790, "total_steps": 72479, "tokens": 14045675520, "loss": 3.3930524289608, "l_mix": 2.0862768590450287, "l_a": 2.2090387642383575, "l_c": 2.1414990425109863, "z_loss": 16.141648411750793, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19774216413497925, "tokens_per_second": 1350774.1196332173, "eta_seconds": 17733.67884669304, "data_wait_seconds": 2.4884939193725586e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20045, "finemath": 0.08234}}
{"step": 26800, "total_steps": 72479, "tokens": 14050918400, "loss": 3.4698129892349243, "l_mix": 2.1332054138183594, "l_a": 2.2594220340251923, "l_c": 2.1906652748584747, "z_loss": 15.813547015190125, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20022159814834595, "tokens_per_second": 1465104.8921740174, "eta_seconds": 16346.236832547189, "data_wait_seconds": 2.232193946838379e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20765, "cosmopedia_v2": 0.20045, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.32115051, "gate_a2b/L1_absmax": 0.64962447, "gate_a2b/L1_rms": 0.33509383, "gate_a2b/L3_absmean": 0.3385044, "gate_a2b/L3_absmax": 0.54085106, "gate_a2b/L3_rms": 0.34285718, "gate_a2b/L5_absmean": 0.51425153, "gate_a2b/L5_absmax": 0.66686201, "gate_a2b/L5_rms": 0.51969457, "gate_c2b/L1_absmean": 0.01500574, "gate_c2b/L1_absmax": 0.43241554, "gate_c2b/L1_rms": 0.05406493, "gate_c2b/L3_absmean": 0.21119939, "gate_c2b/L3_absmax": 0.43378714, "gate_c2b/L3_rms": 0.21509945, "gate_c2b/L5_absmean": 0.5290634, "gate_c2b/L5_absmax": 0.7044552, "gate_c2b/L5_rms": 0.54072595, "gate_b2a/L7_absmean": 0.09119035, "gate_b2a/L7_absmax": 0.28503254, "gate_b2a/L7_rms": 0.09442604, "gate_b2a/L11_absmean": 0.13744159, "gate_b2a/L11_absmax": 0.49333304, "gate_b2a/L11_rms": 0.14053455, "gate_b2a/L14_absmean": 0.32798481, "gate_b2a/L14_absmax": 0.56997573, "gate_b2a/L14_rms": 0.32958716, "gate_b2c/L3_absmean": 0.10814047, "gate_b2c/L3_absmax": 0.64750445, "gate_b2c/L3_rms": 0.12315854, "gate_b2c/L5_absmean": 0.12800805, "gate_b2c/L5_absmax": 0.80025768, "gate_b2c/L5_rms": 0.13670343, "gate_b2c/L6_absmean": 0.18672022, "gate_b2c/L6_absmax": 0.5514136, "gate_b2c/L6_rms": 0.19191101}, "streams": {"cos_a2b/round0": 0.379761, "cos_c2b/round0": 0.26039, "cos_b2a/round0": 0.095247, "cos_b2c/round0": 0.109112, "rms_a/round0": 0.853198, "rms_b/round0": 1.546852, "rms_c/round0": 1.263861, "cos_a2b/round1": 0.413931, "cos_c2b/round1": 0.281009, "cos_b2a/round1": 0.218114, "cos_b2c/round1": 0.126848, "rms_a/round1": 1.517632, "rms_b/round1": 3.199012, "rms_c/round1": 2.112486, "cos_a2b/round2": 0.510026, "cos_c2b/round2": 0.479005, "cos_b2a/round2": 0.02545, "cos_b2c/round2": 0.104648, "rms_a/round2": 3.371762, "rms_b/round2": 7.095454, "rms_c/round2": 3.420686}, "alpha_mean": 0.4121231952745035, "alpha_hist": [0.06116, 0.15779, 0.16667, 0.14174, 0.11525, 0.10857, 0.10283, 0.08997, 0.05083, 0.0052], "alpha_by_type": {"special": {"share": 0.00116, "mean": 0.61602, "std": 0.08621}, "whitespace": {"share": 0.03223, "mean": 0.63557, "std": 0.18164}, "digit": {"share": 0.01025, "mean": 0.34981, "std": 0.16067}, "word_start": {"share": 0.575, "mean": 0.43158, "std": 0.21444}, "word_cont": {"share": 0.26398, "mean": 0.25252, "std": 0.15347}, "punct": {"share": 0.11739, "mean": 0.61783, "std": 0.2006}}, "alpha_p05": 0.09043, "alpha_p50": 0.39443, "alpha_p95": 0.80807}
{"step": 26810, "total_steps": 72479, "tokens": 14056161280, "loss": 4.749912858009338, "l_mix": 2.9409687519073486, "l_a": 3.038221925497055, "l_c": 2.9858561158180237, "z_loss": 17.205642223358154, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1988012194633484, "tokens_per_second": 1059512.5395533915, "eta_seconds": 22598.796878886224, "data_wait_seconds": 0.03690934181213379, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20765, "cosmopedia_v2": 0.20041, "finemath": 0.08236}}
{"step": 26820, "total_steps": 72479, "tokens": 14061404160, "loss": 4.6128286719322205, "l_mix": 2.8558470010757446, "l_a": 2.954988569021225, "l_c": 2.8958782255649567, "z_loss": 17.215188026428223, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1941022276878357, "tokens_per_second": 1335555.0982937508, "eta_seconds": 17923.9821873188, "data_wait_seconds": 0.024260401725769043, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20045, "finemath": 0.08233}}
{"step": 26830, "total_steps": 72479, "tokens": 14066647040, "loss": 4.6201131939888, "l_mix": 2.8600529730319977, "l_a": 2.9567056000232697, "l_c": 2.904336988925934, "z_loss": 17.473783493041992, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22635093331336975, "tokens_per_second": 1428477.3103180507, "eta_seconds": 16754.359862160683, "data_wait_seconds": 1.704692840576172e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20045, "finemath": 0.08233}}
{"step": 26840, "total_steps": 72479, "tokens": 14071889920, "loss": 4.676525235176086, "l_mix": 2.8957234025001526, "l_a": 2.9918573200702667, "l_c": 2.938569188117981, "z_loss": 16.73931312561035, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16767176985740662, "tokens_per_second": 1416978.232475277, "eta_seconds": 16886.624990844724, "data_wait_seconds": 2.1964311599731445e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20768, "cosmopedia_v2": 0.20041, "finemath": 0.08234}}
{"step": 26850, "total_steps": 72479, "tokens": 14077132800, "loss": 4.760861277580261, "l_mix": 2.948137253522873, "l_a": 3.042072832584381, "l_c": 2.9946100413799286, "z_loss": 17.191150426864624, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17554965615272522, "tokens_per_second": 1352844.6383267476, "eta_seconds": 17683.284890413284, "data_wait_seconds": 1.722574234008789e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 26860, "total_steps": 72479, "tokens": 14082375680, "loss": 4.920703291893005, "l_mix": 3.0497496128082275, "l_a": 3.135493725538254, "l_c": 3.095424383878708, "z_loss": 16.780247688293457, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.185477152466774, "tokens_per_second": 1459493.1262197988, "eta_seconds": 16387.534714841844, "data_wait_seconds": 1.65402889251709e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 26870, "total_steps": 72479, "tokens": 14087618560, "loss": 3.460353285074234, "l_mix": 2.12770876288414, "l_a": 2.251015305519104, "l_c": 2.18591770529747, "z_loss": 15.645761251449585, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1949167400598526, "tokens_per_second": 1438344.911954142, "eta_seconds": 16624.83816869259, "data_wait_seconds": 2.0176172256469727e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20045, "finemath": 0.08232}}
{"step": 26880, "total_steps": 72479, "tokens": 14092861440, "loss": 3.463564872741699, "l_mix": 2.1302999556064606, "l_a": 2.248131185770035, "l_c": 2.1908912360668182, "z_loss": 15.581747174263, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1999543160200119, "tokens_per_second": 1473281.6304841056, "eta_seconds": 16227.045812106133, "data_wait_seconds": 1.9073486328125e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20045, "finemath": 0.08233}}
{"step": 26890, "total_steps": 72479, "tokens": 14098104320, "loss": 3.4381931722164154, "l_mix": 2.1148292422294617, "l_a": 2.2374194264411926, "l_c": 2.1686072945594788, "z_loss": 15.558476328849792, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21149004995822906, "tokens_per_second": 1392512.03446453, "eta_seconds": 17164.49484128952, "data_wait_seconds": 2.911686897277832e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20045, "finemath": 0.08234}}
{"step": 26900, "total_steps": 72479, "tokens": 14103347200, "loss": 3.488375872373581, "l_mix": 2.1457120180130005, "l_a": 2.264711380004883, "l_c": 2.2055411338806152, "z_loss": 15.880657434463501, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.24987557530403137, "tokens_per_second": 1313978.7066905487, "eta_seconds": 18186.385083961486, "data_wait_seconds": 2.0951032638549805e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20766, "cosmopedia_v2": 0.20045, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.32032925, "gate_a2b/L1_absmax": 0.6525833, "gate_a2b/L1_rms": 0.33423114, "gate_a2b/L3_absmean": 0.33812669, "gate_a2b/L3_absmax": 0.54320073, "gate_a2b/L3_rms": 0.34249473, "gate_a2b/L5_absmean": 0.51356125, "gate_a2b/L5_absmax": 0.66612154, "gate_a2b/L5_rms": 0.5190565, "gate_c2b/L1_absmean": 0.01542121, "gate_c2b/L1_absmax": 0.43522108, "gate_c2b/L1_rms": 0.05430016, "gate_c2b/L3_absmean": 0.21080092, "gate_c2b/L3_absmax": 0.43299961, "gate_c2b/L3_rms": 0.2147018, "gate_c2b/L5_absmean": 0.52914512, "gate_c2b/L5_absmax": 0.70554209, "gate_c2b/L5_rms": 0.54088354, "gate_b2a/L7_absmean": 0.09076585, "gate_b2a/L7_absmax": 0.2898939, "gate_b2a/L7_rms": 0.09400959, "gate_b2a/L11_absmean": 0.13729189, "gate_b2a/L11_absmax": 0.49235851, "gate_b2a/L11_rms": 0.14043221, "gate_b2a/L14_absmean": 0.32832691, "gate_b2a/L14_absmax": 0.57373607, "gate_b2a/L14_rms": 0.32995418, "gate_b2c/L3_absmean": 0.10785936, "gate_b2c/L3_absmax": 0.64887065, "gate_b2c/L3_rms": 0.12275288, "gate_b2c/L5_absmean": 0.12751506, "gate_b2c/L5_absmax": 0.795219, "gate_b2c/L5_rms": 0.13619785, "gate_b2c/L6_absmean": 0.18629631, "gate_b2c/L6_absmax": 0.54467338, "gate_b2c/L6_rms": 0.1914667}, "streams": {"cos_a2b/round0": 0.383864, "cos_c2b/round0": 0.266483, "cos_b2a/round0": 0.100783, "cos_b2c/round0": 0.117006, "rms_a/round0": 0.865593, "rms_b/round0": 1.537625, "rms_c/round0": 1.252613, "cos_a2b/round1": 0.418088, "cos_c2b/round1": 0.277618, "cos_b2a/round1": 0.219986, "cos_b2c/round1": 0.130314, "rms_a/round1": 1.522517, "rms_b/round1": 3.191373, "rms_c/round1": 2.106249, "cos_a2b/round2": 0.508241, "cos_c2b/round2": 0.486934, "cos_b2a/round2": 0.03123, "cos_b2c/round2": 0.110488, "rms_a/round2": 3.37146, "rms_b/round2": 7.121235, "rms_c/round2": 3.455374}, "alpha_mean": 0.42405445143526066, "alpha_hist": [0.05286, 0.15488, 0.16983, 0.14195, 0.10933, 0.10219, 0.09911, 0.0934, 0.0665, 0.00996], "alpha_by_type": {"special": {"share": 0.00114, "mean": 0.70752, "std": 0.10628}, "whitespace": {"share": 0.03763, "mean": 0.65272, "std": 0.19397}, "digit": {"share": 0.00964, "mean": 0.40311, "std": 0.19895}, "word_start": {"share": 0.57288, "mean": 0.4334, "std": 0.22183}, "word_cont": {"share": 0.25378, "mean": 0.26462, "std": 0.1563}, "punct": {"share": 0.12492, "mean": 0.63524, "std": 0.20076}}, "alpha_p05": 0.10485, "alpha_p50": 0.378, "alpha_p95": 0.83059}
{"step": 26910, "total_steps": 72479, "tokens": 14108590080, "loss": 3.452763706445694, "l_mix": 2.122690796852112, "l_a": 2.2480164766311646, "l_c": 2.1802625358104706, "z_loss": 15.890989303588867, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2298736274242401, "tokens_per_second": 1433628.4044171455, "eta_seconds": 16664.904098153114, "data_wait_seconds": 2.7418136596679688e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20766, "cosmopedia_v2": 0.20045, "finemath": 0.08235}}
{"step": 26920, "total_steps": 72479, "tokens": 14113832960, "loss": 4.529732972383499, "l_mix": 2.8036397993564606, "l_a": 2.9005014896392822, "l_c": 2.8473659455776215, "z_loss": 17.327584505081177, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1956876963376999, "tokens_per_second": 1409492.5050929317, "eta_seconds": 16946.55126273632, "data_wait_seconds": 1.7404556274414062e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20041, "finemath": 0.08236}}
{"step": 26930, "total_steps": 72479, "tokens": 14119075840, "loss": 4.680893838405609, "l_mix": 2.8987546265125275, "l_a": 2.993333548307419, "l_c": 2.9414927065372467, "z_loss": 16.91360092163086, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2006940096616745, "tokens_per_second": 1344428.1139334224, "eta_seconds": 17762.7898914814, "data_wait_seconds": 1.5497207641601562e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20045, "finemath": 0.08232}}
{"step": 26940, "total_steps": 72479, "tokens": 14124318720, "loss": 4.570214033126831, "l_mix": 2.8291980624198914, "l_a": 2.9203526973724365, "l_c": 2.877202957868576, "z_loss": 17.491511344909668, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21258744597434998, "tokens_per_second": 1475386.074094192, "eta_seconds": 16182.578683114052, "data_wait_seconds": 1.671910285949707e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20045, "finemath": 0.08233}}
{"step": 26950, "total_steps": 72479, "tokens": 14129561600, "loss": 3.4508348405361176, "l_mix": 2.122838705778122, "l_a": 2.2412809431552887, "l_c": 2.180168032646179, "z_loss": 15.61430311203003, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2161625474691391, "tokens_per_second": 1466940.360700743, "eta_seconds": 16272.173696684838, "data_wait_seconds": 1.996755599975586e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20764, "cosmopedia_v2": 0.20045, "finemath": 0.08234}}
{"step": 26960, "total_steps": 72479, "tokens": 14134804480, "loss": 3.423040270805359, "l_mix": 2.1062194406986237, "l_a": 2.219307839870453, "l_c": 2.164794474840164, "z_loss": 15.901163816452026, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.27661797404289246, "tokens_per_second": 1375263.084640575, "eta_seconds": 17353.09101111889, "data_wait_seconds": 1.6987323760986328e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20764, "cosmopedia_v2": 0.20045, "finemath": 0.08234}}
{"step": 26970, "total_steps": 72479, "tokens": 14140047360, "loss": 3.461402475833893, "l_mix": 2.1287390291690826, "l_a": 2.248124897480011, "l_c": 2.1888653934001923, "z_loss": 15.663719773292542, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21726281940937042, "tokens_per_second": 1399245.379836063, "eta_seconds": 17051.921654224396, "data_wait_seconds": 1.6629695892333984e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 26980, "total_steps": 72479, "tokens": 14145290240, "loss": 4.934627652168274, "l_mix": 3.0573784708976746, "l_a": 3.148686170578003, "l_c": 3.1032987236976624, "z_loss": 16.53608989715576, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1830340474843979, "tokens_per_second": 1459103.051631, "eta_seconds": 16348.797081422805, "data_wait_seconds": 2.73287296295166e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08236}}
{"step": 26990, "total_steps": 72479, "tokens": 14150533120, "loss": 3.406525284051895, "l_mix": 2.094628542661667, "l_a": 2.2179099917411804, "l_c": 2.149742841720581, "z_loss": 16.008001685142517, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20859214663505554, "tokens_per_second": 1365253.5141880873, "eta_seconds": 17468.797248387335, "data_wait_seconds": 4.2945146560668945e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 27000, "total_steps": 72479, "tokens": 14155776000, "loss": 3.4680133759975433, "l_mix": 2.133269488811493, "l_a": 2.2535473108291626, "l_c": 2.1902511715888977, "z_loss": 16.042507886886597, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2159297913312912, "tokens_per_second": 1443903.0904728028, "eta_seconds": 16513.63869869709, "data_wait_seconds": 3.123283386230469e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.32135484, "gate_a2b/L1_absmax": 0.65293276, "gate_a2b/L1_rms": 0.33523157, "gate_a2b/L3_absmean": 0.33873114, "gate_a2b/L3_absmax": 0.53851134, "gate_a2b/L3_rms": 0.34312648, "gate_a2b/L5_absmean": 0.51385707, "gate_a2b/L5_absmax": 0.66415364, "gate_a2b/L5_rms": 0.51930803, "gate_c2b/L1_absmean": 0.01423614, "gate_c2b/L1_absmax": 0.43014345, "gate_c2b/L1_rms": 0.05384558, "gate_c2b/L3_absmean": 0.21077338, "gate_c2b/L3_absmax": 0.43306544, "gate_c2b/L3_rms": 0.21465018, "gate_c2b/L5_absmean": 0.52930826, "gate_c2b/L5_absmax": 0.70926595, "gate_c2b/L5_rms": 0.54105645, "gate_b2a/L7_absmean": 0.09093603, "gate_b2a/L7_absmax": 0.28694409, "gate_b2a/L7_rms": 0.09415972, "gate_b2a/L11_absmean": 0.13713565, "gate_b2a/L11_absmax": 0.49489254, "gate_b2a/L11_rms": 0.14027938, "gate_b2a/L14_absmean": 0.32808596, "gate_b2a/L14_absmax": 0.5737291, "gate_b2a/L14_rms": 0.32971492, "gate_b2c/L3_absmean": 0.10811758, "gate_b2c/L3_absmax": 0.64588517, "gate_b2c/L3_rms": 0.12303976, "gate_b2c/L5_absmean": 0.12789655, "gate_b2c/L5_absmax": 0.79555327, "gate_b2c/L5_rms": 0.13644606, "gate_b2c/L6_absmean": 0.18674269, "gate_b2c/L6_absmax": 0.54252839, "gate_b2c/L6_rms": 0.19188601}, "streams": {"cos_a2b/round0": 0.380092, "cos_c2b/round0": 0.267564, "cos_b2a/round0": 0.099554, "cos_b2c/round0": 0.117706, "rms_a/round0": 0.853187, "rms_b/round0": 1.528979, "rms_c/round0": 1.258855, "cos_a2b/round1": 0.415508, "cos_c2b/round1": 0.277952, "cos_b2a/round1": 0.217772, "cos_b2c/round1": 0.127149, "rms_a/round1": 1.506865, "rms_b/round1": 3.182937, "rms_c/round1": 2.126519, "cos_a2b/round2": 0.5131, "cos_c2b/round2": 0.472504, "cos_b2a/round2": 0.026793, "cos_b2c/round2": 0.10873, "rms_a/round2": 3.353252, "rms_b/round2": 7.058531, "rms_c/round2": 3.433021}, "alpha_mean": 0.4452686286458345, "alpha_hist": [0.0477, 0.13377, 0.16193, 0.14076, 0.11589, 0.10164, 0.1004, 0.10007, 0.08459, 0.01324], "alpha_by_type": {"special": {"share": 0.00114, "mean": 0.65149, "std": 0.10305}, "whitespace": {"share": 0.03278, "mean": 0.66466, "std": 0.19073}, "digit": {"share": 0.0078, "mean": 0.38995, "std": 0.17918}, "word_start": {"share": 0.5768, "mean": 0.47086, "std": 0.22064}, "word_cont": {"share": 0.26498, "mean": 0.26398, "std": 0.15759}, "punct": {"share": 0.1165, "mean": 0.67088, "std": 0.20054}}, "alpha_p05": 0.09774, "alpha_p50": 0.38769, "alpha_p95": 0.85047}
{"step": 27010, "total_steps": 72479, "tokens": 14161018880, "loss": 3.4586620032787323, "l_mix": 2.1276732683181763, "l_a": 2.248615175485611, "l_c": 2.1827784180641174, "z_loss": 15.706148505210876, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2390684187412262, "tokens_per_second": 1455614.1700469016, "eta_seconds": 16377.17711365223, "data_wait_seconds": 1.9371509552001953e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 27020, "total_steps": 72479, "tokens": 14166261760, "loss": 3.4946822226047516, "l_mix": 2.1490080654621124, "l_a": 2.27550807595253, "l_c": 2.204801470041275, "z_loss": 15.812809586524963, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21908318996429443, "tokens_per_second": 1256987.5280901445, "eta_seconds": 18960.894725990296, "data_wait_seconds": 2.467632293701172e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 27030, "total_steps": 72479, "tokens": 14171504640, "loss": 4.535400748252869, "l_mix": 2.8076544404029846, "l_a": 2.901256799697876, "l_c": 2.852283298969269, "z_loss": 16.84263038635254, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18673531711101532, "tokens_per_second": 1318429.5347347949, "eta_seconds": 18073.294540381434, "data_wait_seconds": 2.002716064453125e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 27040, "total_steps": 72479, "tokens": 14176747520, "loss": 4.619523465633392, "l_mix": 2.859067440032959, "l_a": 2.958099216222763, "l_c": 2.9043148458004, "z_loss": 17.317927837371826, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18979115784168243, "tokens_per_second": 1423202.0747283858, "eta_seconds": 16739.100409579278, "data_wait_seconds": 2.372264862060547e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08232}}
{"step": 27050, "total_steps": 72479, "tokens": 14181990400, "loss": 4.632466793060303, "l_mix": 2.866930365562439, "l_a": 2.9680064022541046, "l_c": 2.911490708589554, "z_loss": 16.870376110076904, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1859697699546814, "tokens_per_second": 1346684.2831309154, "eta_seconds": 17686.312857699395, "data_wait_seconds": 1.7911195755004883e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 27060, "total_steps": 72479, "tokens": 14187233280, "loss": 3.476482629776001, "l_mix": 2.1383552849292755, "l_a": 2.2607769072055817, "l_c": 2.1943163573741913, "z_loss": 15.993366241455078, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21355536580085754, "tokens_per_second": 1446333.6031847154, "eta_seconds": 16464.138439130784, "data_wait_seconds": 2.250075340270996e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 27070, "total_steps": 72479, "tokens": 14192476160, "loss": 3.4624069035053253, "l_mix": 2.1292661130428314, "l_a": 2.250632733106613, "l_c": 2.1879984736442566, "z_loss": 15.514852166175842, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21526190638542175, "tokens_per_second": 1487402.9701696, "eta_seconds": 16006.014691019058, "data_wait_seconds": 1.8537044525146484e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 27080, "total_steps": 72479, "tokens": 14197719040, "loss": 3.4560621082782745, "l_mix": 2.1254842877388, "l_a": 2.2477082908153534, "l_c": 2.1823593080043793, "z_loss": 15.575044751167297, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21652516722679138, "tokens_per_second": 1412345.9798417522, "eta_seconds": 16852.917947673795, "data_wait_seconds": 2.664327621459961e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 27090, "total_steps": 72479, "tokens": 14202961920, "loss": 3.472174733877182, "l_mix": 2.135515332221985, "l_a": 2.258524090051651, "l_c": 2.1918462216854095, "z_loss": 15.482832908630371, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2391856610774994, "tokens_per_second": 1466223.0231332073, "eta_seconds": 16230.073908638955, "data_wait_seconds": 2.7120113372802734e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08236}}
{"step": 27100, "total_steps": 72479, "tokens": 14208204800, "loss": 4.967081665992737, "l_mix": 3.078562378883362, "l_a": 3.167132943868637, "l_c": 3.1224530935287476, "z_loss": 16.433067083358765, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2228727489709854, "tokens_per_second": 1407095.2655821035, "eta_seconds": 16908.354205965996, "data_wait_seconds": 2.0623207092285156e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.20044, "finemath": 0.08232}, "gates": {"gate_a2b/L1_absmean": 0.32286027, "gate_a2b/L1_absmax": 0.6550734, "gate_a2b/L1_rms": 0.33650351, "gate_a2b/L3_absmean": 0.33855462, "gate_a2b/L3_absmax": 0.54395741, "gate_a2b/L3_rms": 0.3430554, "gate_a2b/L5_absmean": 0.51426727, "gate_a2b/L5_absmax": 0.67063266, "gate_a2b/L5_rms": 0.51975465, "gate_c2b/L1_absmean": 0.01447782, "gate_c2b/L1_absmax": 0.42862681, "gate_c2b/L1_rms": 0.05377445, "gate_c2b/L3_absmean": 0.21042733, "gate_c2b/L3_absmax": 0.43119341, "gate_c2b/L3_rms": 0.21433324, "gate_c2b/L5_absmean": 0.529082, "gate_c2b/L5_absmax": 0.70958, "gate_c2b/L5_rms": 0.54090285, "gate_b2a/L7_absmean": 0.09138568, "gate_b2a/L7_absmax": 0.28839052, "gate_b2a/L7_rms": 0.09465013, "gate_b2a/L11_absmean": 0.13694075, "gate_b2a/L11_absmax": 0.49495944, "gate_b2a/L11_rms": 0.14008953, "gate_b2a/L14_absmean": 0.3287105, "gate_b2a/L14_absmax": 0.57591945, "gate_b2a/L14_rms": 0.33033159, "gate_b2c/L3_absmean": 0.10849184, "gate_b2c/L3_absmax": 0.64581722, "gate_b2c/L3_rms": 0.12330574, "gate_b2c/L5_absmean": 0.12754476, "gate_b2c/L5_absmax": 0.79302996, "gate_b2c/L5_rms": 0.1362239, "gate_b2c/L6_absmean": 0.18723595, "gate_b2c/L6_absmax": 0.53995681, "gate_b2c/L6_rms": 0.19235313}, "streams": {"cos_a2b/round0": 0.388042, "cos_c2b/round0": 0.293954, "cos_b2a/round0": 0.110993, "cos_b2c/round0": 0.151174, "rms_a/round0": 0.830047, "rms_b/round0": 1.53229, "rms_c/round0": 1.239164, "cos_a2b/round1": 0.416517, "cos_c2b/round1": 0.282314, "cos_b2a/round1": 0.240613, "cos_b2c/round1": 0.127142, "rms_a/round1": 1.44196, "rms_b/round1": 3.070001, "rms_c/round1": 2.069703, "cos_a2b/round2": 0.549162, "cos_c2b/round2": 0.502779, "cos_b2a/round2": 0.074886, "cos_b2c/round2": 0.141339, "rms_a/round2": 3.311032, "rms_b/round2": 6.808643, "rms_c/round2": 3.362068}, "alpha_mean": 0.46826512274849463, "alpha_hist": [0.00758, 0.04597, 0.13268, 0.18732, 0.19261, 0.18854, 0.13937, 0.08676, 0.01909, 8e-05], "alpha_by_type": {"special": {"share": 0.00067, "mean": 0.56242, "std": 0.10173}, "whitespace": {"share": 0.04648, "mean": 0.54596, "std": 0.12013}, "digit": {"share": 0.03172, "mean": 0.49067, "std": 0.11953}, "word_start": {"share": 0.56592, "mean": 0.49035, "std": 0.16791}, "word_cont": {"share": 0.25375, "mean": 0.3559, "std": 0.13481}, "punct": {"share": 0.10146, "mean": 0.58289, "std": 0.16872}}, "alpha_p05": 0.21436, "alpha_p50": 0.47329, "alpha_p95": 0.74055}
{"step": 27110, "total_steps": 72479, "tokens": 14213447680, "loss": 4.9084137082099915, "l_mix": 3.0421775579452515, "l_a": 3.129411369562149, "l_c": 3.0860299170017242, "z_loss": 16.03683590888977, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20944909751415253, "tokens_per_second": 1397198.9827634867, "eta_seconds": 17024.362718153, "data_wait_seconds": 2.6106834411621094e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 27120, "total_steps": 72479, "tokens": 14218690560, "loss": 3.50051349401474, "l_mix": 2.153143346309662, "l_a": 2.272069990634918, "l_c": 2.2136110067367554, "z_loss": 16.657778024673462, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.34388428926467896, "tokens_per_second": 1433343.116259649, "eta_seconds": 16591.407264757156, "data_wait_seconds": 2.9653310775756836e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 27130, "total_steps": 72479, "tokens": 14223933440, "loss": 3.454080641269684, "l_mix": 2.1252203285694122, "l_a": 2.244857430458069, "l_c": 2.17953622341156, "z_loss": 15.421871662139893, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2155780792236328, "tokens_per_second": 1461131.5254643164, "eta_seconds": 16272.276723647117, "data_wait_seconds": 1.990795135498047e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 27140, "total_steps": 72479, "tokens": 14229176320, "loss": 4.672049462795258, "l_mix": 2.891843855381012, "l_a": 2.9908087849617004, "l_c": 2.937551498413086, "z_loss": 16.974806308746338, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15383043885231018, "tokens_per_second": 1324278.2657714665, "eta_seconds": 17949.92355186939, "data_wait_seconds": 1.677870750427246e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 27150, "total_steps": 72479, "tokens": 14234419200, "loss": 4.683989644050598, "l_mix": 2.900141626596451, "l_a": 2.9984095692634583, "l_c": 2.94197815656662, "z_loss": 17.314782857894897, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19693408906459808, "tokens_per_second": 1366950.8445273654, "eta_seconds": 17385.739104771616, "data_wait_seconds": 1.946091651916504e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08232}}
{"step": 27160, "total_steps": 72479, "tokens": 14239662080, "loss": 4.5486661195755005, "l_mix": 2.815696269273758, "l_a": 2.911534905433655, "l_c": 2.8593221604824066, "z_loss": 17.12632155418396, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18288485705852509, "tokens_per_second": 1389251.7499408673, "eta_seconds": 17102.881369781495, "data_wait_seconds": 3.0219554901123047e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 27170, "total_steps": 72479, "tokens": 14244904960, "loss": 3.4721776843070984, "l_mix": 2.1360277235507965, "l_a": 2.255088150501251, "l_c": 2.193428874015808, "z_loss": 15.948113918304443, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.27052173018455505, "tokens_per_second": 1367454.319208345, "eta_seconds": 17371.669867372515, "data_wait_seconds": 7.516145706176758e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 27180, "total_steps": 72479, "tokens": 14250147840, "loss": 3.444318562746048, "l_mix": 2.1189554929733276, "l_a": 2.2397399842739105, "l_c": 2.1728328466415405, "z_loss": 15.91192102432251, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.26269301772117615, "tokens_per_second": 1435617.171465847, "eta_seconds": 16543.214015579222, "data_wait_seconds": 2.2470951080322266e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 27190, "total_steps": 72479, "tokens": 14255390720, "loss": 3.4469343423843384, "l_mix": 2.1186427772045135, "l_a": 2.2430756390094757, "l_c": 2.179409086704254, "z_loss": 15.46156370639801, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2352125495672226, "tokens_per_second": 1425280.0496943684, "eta_seconds": 16659.518413305283, "data_wait_seconds": 5.0902366638183594e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 27200, "total_steps": 72479, "tokens": 14260633600, "loss": 3.5152446627616882, "l_mix": 2.1626191437244415, "l_a": 2.2869637310504913, "l_c": 2.216606557369232, "z_loss": 15.544857621192932, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2549097239971161, "tokens_per_second": 1444909.043019067, "eta_seconds": 16429.571443748475, "data_wait_seconds": 2.7000904083251953e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.32292777, "gate_a2b/L1_absmax": 0.65843701, "gate_a2b/L1_rms": 0.33678633, "gate_a2b/L3_absmean": 0.3389996, "gate_a2b/L3_absmax": 0.53786844, "gate_a2b/L3_rms": 0.3433587, "gate_a2b/L5_absmean": 0.51428139, "gate_a2b/L5_absmax": 0.66390634, "gate_a2b/L5_rms": 0.5197525, "gate_c2b/L1_absmean": 0.01509444, "gate_c2b/L1_absmax": 0.42589793, "gate_c2b/L1_rms": 0.05389288, "gate_c2b/L3_absmean": 0.21057926, "gate_c2b/L3_absmax": 0.42818683, "gate_c2b/L3_rms": 0.21446082, "gate_c2b/L5_absmean": 0.53000778, "gate_c2b/L5_absmax": 0.70893931, "gate_c2b/L5_rms": 0.54178786, "gate_b2a/L7_absmean": 0.09070462, "gate_b2a/L7_absmax": 0.28824803, "gate_b2a/L7_rms": 0.09398817, "gate_b2a/L11_absmean": 0.13713935, "gate_b2a/L11_absmax": 0.49754432, "gate_b2a/L11_rms": 0.14029838, "gate_b2a/L14_absmean": 0.32894734, "gate_b2a/L14_absmax": 0.57762176, "gate_b2a/L14_rms": 0.3305741, "gate_b2c/L3_absmean": 0.10834152, "gate_b2c/L3_absmax": 0.64716917, "gate_b2c/L3_rms": 0.12332845, "gate_b2c/L5_absmean": 0.12806146, "gate_b2c/L5_absmax": 0.79337925, "gate_b2c/L5_rms": 0.13666715, "gate_b2c/L6_absmean": 0.18752089, "gate_b2c/L6_absmax": 0.54255056, "gate_b2c/L6_rms": 0.19266616}, "streams": {"cos_a2b/round0": 0.379954, "cos_c2b/round0": 0.263541, "cos_b2a/round0": 0.100248, "cos_b2c/round0": 0.115635, "rms_a/round0": 0.855497, "rms_b/round0": 1.513669, "rms_c/round0": 1.250714, "cos_a2b/round1": 0.411323, "cos_c2b/round1": 0.280866, "cos_b2a/round1": 0.220493, "cos_b2c/round1": 0.127779, "rms_a/round1": 1.499994, "rms_b/round1": 3.147341, "rms_c/round1": 2.122131, "cos_a2b/round2": 0.509812, "cos_c2b/round2": 0.48596, "cos_b2a/round2": 0.034973, "cos_b2c/round2": 0.11142, "rms_a/round2": 3.331917, "rms_b/round2": 6.989207, "rms_c/round2": 3.436742}, "alpha_mean": 0.4374384152084332, "alpha_hist": [0.04225, 0.12311, 0.16174, 0.15451, 0.12903, 0.11555, 0.11241, 0.10213, 0.05305, 0.00621], "alpha_by_type": {"special": {"share": 0.00114, "mean": 0.67158, "std": 0.12184}, "whitespace": {"share": 0.03255, "mean": 0.65833, "std": 0.17507}, "digit": {"share": 0.01012, "mean": 0.40097, "std": 0.18366}, "word_start": {"share": 0.57213, "mean": 0.45413, "std": 0.20526}, "word_cont": {"share": 0.25537, "mean": 0.27677, "std": 0.15815}, "punct": {"share": 0.12869, "mean": 0.62695, "std": 0.19079}}, "alpha_p05": 0.12065, "alpha_p50": 0.40451, "alpha_p95": 0.82018}
{"step": 27210, "total_steps": 72479, "tokens": 14265876480, "loss": 3.0562265813350677, "l_mix": 1.8892806470394135, "l_a": 1.963609755039215, "l_c": 1.919948324561119, "z_loss": 18.784139394760132, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21309340000152588, "tokens_per_second": 1460101.2177088817, "eta_seconds": 16255.032996439933, "data_wait_seconds": 3.057718276977539e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08236}}
{"step": 27220, "total_steps": 72479, "tokens": 14271119360, "loss": 4.8204843401908875, "l_mix": 2.9871340095996857, "l_a": 3.0705592036247253, "l_c": 3.0349816977977753, "z_loss": 16.87994146347046, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1946091651916504, "tokens_per_second": 1469253.8371085764, "eta_seconds": 16150.204949402809, "data_wait_seconds": 2.6941299438476562e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20768, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 27230, "total_steps": 72479, "tokens": 14276362240, "loss": 4.525968790054321, "l_mix": 2.8007437586784363, "l_a": 2.902037888765335, "l_c": 2.8430885672569275, "z_loss": 16.870105504989624, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20946067571640015, "tokens_per_second": 1443334.4648098235, "eta_seconds": 16436.59753882885, "data_wait_seconds": 2.053380012512207e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 27240, "total_steps": 72479, "tokens": 14281605120, "loss": 4.595952391624451, "l_mix": 2.8447989225387573, "l_a": 2.9410825073719025, "l_c": 2.890411525964737, "z_loss": 17.050861835479736, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18701791763305664, "tokens_per_second": 1258959.042271493, "eta_seconds": 18839.584160900118, "data_wait_seconds": 1.6629695892333984e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 27250, "total_steps": 72479, "tokens": 14286848000, "loss": 4.597331702709198, "l_mix": 2.846434533596039, "l_a": 2.939641684293747, "l_c": 2.8910268545150757, "z_loss": 16.96635913848877, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.14972326159477234, "tokens_per_second": 1329982.9822959304, "eta_seconds": 17829.56794760227, "data_wait_seconds": 2.0265579223632812e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.2004, "finemath": 0.08235}}
{"step": 27260, "total_steps": 72479, "tokens": 14292090880, "loss": 4.527559757232666, "l_mix": 2.802179664373398, "l_a": 2.8993141651153564, "l_c": 2.8462055027484894, "z_loss": 17.24174666404724, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2551979720592499, "tokens_per_second": 1357839.5917212798, "eta_seconds": 17459.926206707954, "data_wait_seconds": 2.5510787963867188e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.2004, "finemath": 0.08236}}
{"step": 27270, "total_steps": 72479, "tokens": 14297333760, "loss": 4.648827850818634, "l_mix": 2.8777097165584564, "l_a": 2.975536346435547, "l_c": 2.9225369691848755, "z_loss": 16.960895538330078, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19128020107746124, "tokens_per_second": 1241678.5439211612, "eta_seconds": 19089.10829460621, "data_wait_seconds": 2.4527311325073242e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08232}}
{"step": 27280, "total_steps": 72479, "tokens": 14302576640, "loss": 3.468910276889801, "l_mix": 2.1333329379558563, "l_a": 2.258479356765747, "l_c": 2.187976598739624, "z_loss": 16.40480375289917, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2612890303134918, "tokens_per_second": 1449604.6263665461, "eta_seconds": 16347.418379449844, "data_wait_seconds": 1.7017126083374023e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 27290, "total_steps": 72479, "tokens": 14307819520, "loss": 3.4276320040225983, "l_mix": 2.1073594093322754, "l_a": 2.2285046875476837, "l_c": 2.167158305644989, "z_loss": 15.735831618309021, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20042234659194946, "tokens_per_second": 1441212.6188542172, "eta_seconds": 16438.96960244179, "data_wait_seconds": 2.2798776626586914e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 27300, "total_steps": 72479, "tokens": 14313062400, "loss": 3.4802366197109222, "l_mix": 2.1403363049030304, "l_a": 2.262548506259918, "l_c": 2.198502629995346, "z_loss": 15.849116802215576, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18503643572330475, "tokens_per_second": 1445358.2508655298, "eta_seconds": 16388.191327524186, "data_wait_seconds": 2.4199485778808594e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.32369184, "gate_a2b/L1_absmax": 0.65292877, "gate_a2b/L1_rms": 0.33731306, "gate_a2b/L3_absmean": 0.33891797, "gate_a2b/L3_absmax": 0.53932506, "gate_a2b/L3_rms": 0.34337243, "gate_a2b/L5_absmean": 0.51381463, "gate_a2b/L5_absmax": 0.6633873, "gate_a2b/L5_rms": 0.51928794, "gate_c2b/L1_absmean": 0.01522129, "gate_c2b/L1_absmax": 0.43275204, "gate_c2b/L1_rms": 0.05441213, "gate_c2b/L3_absmean": 0.21052575, "gate_c2b/L3_absmax": 0.43757671, "gate_c2b/L3_rms": 0.21444519, "gate_c2b/L5_absmean": 0.52908421, "gate_c2b/L5_absmax": 0.70967132, "gate_c2b/L5_rms": 0.54092759, "gate_b2a/L7_absmean": 0.09113133, "gate_b2a/L7_absmax": 0.28449473, "gate_b2a/L7_rms": 0.09429562, "gate_b2a/L11_absmean": 0.13731238, "gate_b2a/L11_absmax": 0.50087202, "gate_b2a/L11_rms": 0.14050286, "gate_b2a/L14_absmean": 0.32873917, "gate_b2a/L14_absmax": 0.57777387, "gate_b2a/L14_rms": 0.33038372, "gate_b2c/L3_absmean": 0.10856029, "gate_b2c/L3_absmax": 0.64425611, "gate_b2c/L3_rms": 0.12350526, "gate_b2c/L5_absmean": 0.1276294, "gate_b2c/L5_absmax": 0.79747105, "gate_b2c/L5_rms": 0.13632685, "gate_b2c/L6_absmean": 0.18695542, "gate_b2c/L6_absmax": 0.54617065, "gate_b2c/L6_rms": 0.19220424}, "streams": {"cos_a2b/round0": 0.383812, "cos_c2b/round0": 0.267018, "cos_b2a/round0": 0.095287, "cos_b2c/round0": 0.112093, "rms_a/round0": 0.854394, "rms_b/round0": 1.536515, "rms_c/round0": 1.255552, "cos_a2b/round1": 0.416885, "cos_c2b/round1": 0.278772, "cos_b2a/round1": 0.219178, "cos_b2c/round1": 0.12943, "rms_a/round1": 1.520274, "rms_b/round1": 3.198077, "rms_c/round1": 2.125678, "cos_a2b/round2": 0.513916, "cos_c2b/round2": 0.47942, "cos_b2a/round2": 0.029628, "cos_b2c/round2": 0.115737, "rms_a/round2": 3.37456, "rms_b/round2": 7.068131, "rms_c/round2": 3.449139}, "alpha_mean": 0.41399720031927956, "alpha_hist": [0.04619, 0.15286, 0.18527, 0.14989, 0.11285, 0.10121, 0.11014, 0.09186, 0.04335, 0.00638], "alpha_by_type": {"special": {"share": 0.00104, "mean": 0.54061, "std": 0.11283}, "whitespace": {"share": 0.03874, "mean": 0.6643, "std": 0.17372}, "digit": {"share": 0.00999, "mean": 0.42009, "std": 0.19459}, "word_start": {"share": 0.5634, "mean": 0.4212, "std": 0.20824}, "word_cont": {"share": 0.26239, "mean": 0.26254, "std": 0.15109}, "punct": {"share": 0.12444, "mean": 0.62127, "std": 0.18261}}, "alpha_p05": 0.1016, "alpha_p50": 0.39001, "alpha_p95": 0.80007}
{"step": 27310, "total_steps": 72479, "tokens": 14318305280, "loss": 3.4292171895504, "l_mix": 2.107708975672722, "l_a": 2.2335234582424164, "l_c": 2.166284739971161, "z_loss": 15.657528519630432, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22234071791172028, "tokens_per_second": 1456985.6891939847, "eta_seconds": 16253.807328128814, "data_wait_seconds": 3.644824028015137e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 27320, "total_steps": 72479, "tokens": 14323548160, "loss": 3.2936988174915314, "l_mix": 2.0368432849645615, "l_a": 2.113010957837105, "l_c": 2.070093423128128, "z_loss": 19.241990566253662, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.32420387864112854, "tokens_per_second": 1460347.6025861758, "eta_seconds": 16212.798754262925, "data_wait_seconds": 2.7507543563842773e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08236}}
{"step": 27330, "total_steps": 72479, "tokens": 14328791040, "loss": 4.674879491329193, "l_mix": 2.8943856358528137, "l_a": 2.9906713664531708, "l_c": 2.9383876025676727, "z_loss": 17.7611141204834, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.23283621668815613, "tokens_per_second": 1435902.896870483, "eta_seconds": 16485.152974891666, "data_wait_seconds": 1.913309097290039e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 27340, "total_steps": 72479, "tokens": 14334033920, "loss": 4.616980671882629, "l_mix": 2.858429491519928, "l_a": 2.9515009224414825, "l_c": 2.9045483767986298, "z_loss": 17.363247632980347, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17926697432994843, "tokens_per_second": 1426862.7697949, "eta_seconds": 16585.92300043106, "data_wait_seconds": 1.8149614334106445e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 27350, "total_steps": 72479, "tokens": 14339276800, "loss": 4.562688052654266, "l_mix": 2.8238630890846252, "l_a": 2.918612539768219, "l_c": 2.871594876050949, "z_loss": 17.62787938117981, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19902099668979645, "tokens_per_second": 1426300.8228948761, "eta_seconds": 16588.781813907623, "data_wait_seconds": 2.1785497665405273e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 27360, "total_steps": 72479, "tokens": 14344519680, "loss": 4.535201370716095, "l_mix": 2.806661367416382, "l_a": 2.9050504863262177, "l_c": 2.85101655125618, "z_loss": 17.19788146018982, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16605085134506226, "tokens_per_second": 1396469.2851461058, "eta_seconds": 16939.398899507523, "data_wait_seconds": 2.3484230041503906e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 27370, "total_steps": 72479, "tokens": 14349762560, "loss": 4.787886261940002, "l_mix": 2.965008467435837, "l_a": 3.056939125061035, "l_c": 3.013645261526108, "z_loss": 17.025136947631836, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21416054666042328, "tokens_per_second": 1382993.4829256511, "eta_seconds": 17100.664380550385, "data_wait_seconds": 3.212690353393555e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 27380, "total_steps": 72479, "tokens": 14355005440, "loss": 4.406573951244354, "l_mix": 2.726252257823944, "l_a": 2.8261460065841675, "l_c": 2.7692735493183136, "z_loss": 16.95779061317444, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1852831393480301, "tokens_per_second": 1407294.7240457984, "eta_seconds": 16801.64368414879, "data_wait_seconds": 2.086162567138672e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08232}}
{"step": 27390, "total_steps": 72479, "tokens": 14360248320, "loss": 3.4579063653945923, "l_mix": 2.125555545091629, "l_a": 2.2501377761363983, "l_c": 2.1857753694057465, "z_loss": 15.76809847354889, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1998206228017807, "tokens_per_second": 1468157.0554595636, "eta_seconds": 16101.561848640442, "data_wait_seconds": 2.0623207092285156e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 27400, "total_steps": 72479, "tokens": 14365491200, "loss": 3.442343771457672, "l_mix": 2.1169434189796448, "l_a": 2.239916503429413, "l_c": 2.1728382110595703, "z_loss": 15.738749742507935, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18056051433086395, "tokens_per_second": 1445082.8049003757, "eta_seconds": 16355.034238767625, "data_wait_seconds": 1.8537044525146484e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.32406941, "gate_a2b/L1_absmax": 0.64959484, "gate_a2b/L1_rms": 0.33759734, "gate_a2b/L3_absmean": 0.33885321, "gate_a2b/L3_absmax": 0.53868461, "gate_a2b/L3_rms": 0.34328085, "gate_a2b/L5_absmean": 0.51439285, "gate_a2b/L5_absmax": 0.67246789, "gate_a2b/L5_rms": 0.51985192, "gate_c2b/L1_absmean": 0.0145032, "gate_c2b/L1_absmax": 0.43094888, "gate_c2b/L1_rms": 0.05397484, "gate_c2b/L3_absmean": 0.21045013, "gate_c2b/L3_absmax": 0.42945749, "gate_c2b/L3_rms": 0.21438739, "gate_c2b/L5_absmean": 0.52916235, "gate_c2b/L5_absmax": 0.71235722, "gate_c2b/L5_rms": 0.54092962, "gate_b2a/L7_absmean": 0.09116735, "gate_b2a/L7_absmax": 0.28770974, "gate_b2a/L7_rms": 0.09431924, "gate_b2a/L11_absmean": 0.13731682, "gate_b2a/L11_absmax": 0.50043392, "gate_b2a/L11_rms": 0.14053339, "gate_b2a/L14_absmean": 0.3292132, "gate_b2a/L14_absmax": 0.57832175, "gate_b2a/L14_rms": 0.33084095, "gate_b2c/L3_absmean": 0.10873969, "gate_b2c/L3_absmax": 0.64369756, "gate_b2c/L3_rms": 0.12349989, "gate_b2c/L5_absmean": 0.12835915, "gate_b2c/L5_absmax": 0.79905528, "gate_b2c/L5_rms": 0.13711022, "gate_b2c/L6_absmean": 0.18732676, "gate_b2c/L6_absmax": 0.54423684, "gate_b2c/L6_rms": 0.19261225}, "streams": {"cos_a2b/round0": 0.383536, "cos_c2b/round0": 0.269876, "cos_b2a/round0": 0.099351, "cos_b2c/round0": 0.117216, "rms_a/round0": 0.856279, "rms_b/round0": 1.522204, "rms_c/round0": 1.25181, "cos_a2b/round1": 0.419402, "cos_c2b/round1": 0.279819, "cos_b2a/round1": 0.220499, "cos_b2c/round1": 0.129289, "rms_a/round1": 1.520458, "rms_b/round1": 3.180079, "rms_c/round1": 2.115965, "cos_a2b/round2": 0.513967, "cos_c2b/round2": 0.479364, "cos_b2a/round2": 0.027202, "cos_b2c/round2": 0.10823, "rms_a/round2": 3.360369, "rms_b/round2": 7.043853, "rms_c/round2": 3.429605}, "alpha_mean": 0.4250086494298344, "alpha_hist": [0.05321, 0.15279, 0.16769, 0.13861, 0.11104, 0.09886, 0.10825, 0.10518, 0.0575, 0.00688], "alpha_by_type": {"special": {"share": 0.00107, "mean": 0.65976, "std": 0.10736}, "whitespace": {"share": 0.03722, "mean": 0.64227, "std": 0.19263}, "digit": {"share": 0.00993, "mean": 0.35501, "std": 0.17072}, "word_start": {"share": 0.57249, "mean": 0.4472, "std": 0.21923}, "word_cont": {"share": 0.25926, "mean": 0.25326, "std": 0.15351}, "punct": {"share": 0.12003, "mean": 0.62649, "std": 0.19396}}, "alpha_p05": 0.09106, "alpha_p50": 0.38145, "alpha_p95": 0.83113}
{"step": 27410, "total_steps": 72479, "tokens": 14370734080, "loss": 3.4031492471694946, "l_mix": 2.0931117832660675, "l_a": 2.2142173051834106, "l_c": 2.1472418308258057, "z_loss": 15.996684193611145, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2361942082643509, "tokens_per_second": 1460041.888319463, "eta_seconds": 16183.875312781334, "data_wait_seconds": 1.65402889251709e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 27420, "total_steps": 72479, "tokens": 14375976960, "loss": 3.4572232365608215, "l_mix": 2.126662403345108, "l_a": 2.246423989534378, "l_c": 2.1834614872932434, "z_loss": 15.951857447624207, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2512185573577881, "tokens_per_second": 1450426.4172636778, "eta_seconds": 16287.550137543678, "data_wait_seconds": 3.3229589462280273e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 27430, "total_steps": 72479, "tokens": 14381219840, "loss": 3.1516418755054474, "l_mix": 1.9474036246538162, "l_a": 2.0230567902326584, "l_c": 1.984606072306633, "z_loss": 19.39326572418213, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.26523515582084656, "tokens_per_second": 1475756.9754965864, "eta_seconds": 16004.430610299109, "data_wait_seconds": 2.473592758178711e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08236}}
{"step": 27440, "total_steps": 72479, "tokens": 14386462720, "loss": 4.6520790457725525, "l_mix": 2.880495399236679, "l_a": 2.9761553406715393, "l_c": 2.923314183950424, "z_loss": 17.427769422531128, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2049952745437622, "tokens_per_second": 1463307.4094555676, "eta_seconds": 16137.010637283325, "data_wait_seconds": 1.7464160919189453e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 27450, "total_steps": 72479, "tokens": 14391705600, "loss": 4.580181539058685, "l_mix": 2.8348465263843536, "l_a": 2.9320189356803894, "l_c": 2.8799746930599213, "z_loss": 17.369468927383423, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1945003867149353, "tokens_per_second": 1425416.5057026104, "eta_seconds": 16562.29197399616, "data_wait_seconds": 2.008676528930664e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 27460, "total_steps": 72479, "tokens": 14396948480, "loss": 4.684071660041809, "l_mix": 2.89994677901268, "l_a": 2.9960145354270935, "l_c": 2.9454135298728943, "z_loss": 16.962157487869263, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1944609433412552, "tokens_per_second": 1358615.4153535017, "eta_seconds": 17372.776140522958, "data_wait_seconds": 1.5735626220703125e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 27470, "total_steps": 72479, "tokens": 14402191360, "loss": 4.765119850635529, "l_mix": 2.950649380683899, "l_a": 3.0470891296863556, "l_c": 2.995285004377365, "z_loss": 17.582561016082764, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.25695738196372986, "tokens_per_second": 1291494.0981293737, "eta_seconds": 18271.611636614798, "data_wait_seconds": 1.6182661056518555e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 27480, "total_steps": 72479, "tokens": 14407434240, "loss": 4.457265764474869, "l_mix": 2.7588694393634796, "l_a": 2.849417418241501, "l_c": 2.806195229291916, "z_loss": 17.124228954315186, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2870135009288788, "tokens_per_second": 1226142.6776453692, "eta_seconds": 19241.18305490017, "data_wait_seconds": 1.6242265701293945e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 27490, "total_steps": 72479, "tokens": 14412677120, "loss": 4.964576363563538, "l_mix": 3.0771697759628296, "l_a": 3.164028376340866, "l_c": 3.121754378080368, "z_loss": 16.716530323028564, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18293270468711853, "tokens_per_second": 1470621.7852571157, "eta_seconds": 16038.925214123725, "data_wait_seconds": 3.4242868423461914e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20768, "cosmopedia_v2": 0.20044, "finemath": 0.08232}}
{"step": 27500, "total_steps": 72479, "tokens": 14417920000, "loss": 4.8753180503845215, "l_mix": 3.021074950695038, "l_a": 3.1098415553569794, "l_c": 3.065264642238617, "z_loss": 17.112237453460693, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17693881690502167, "tokens_per_second": 1389806.5721620831, "eta_seconds": 16967.792802500728, "data_wait_seconds": 4.1544437408447266e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.32389075, "gate_a2b/L1_absmax": 0.65724844, "gate_a2b/L1_rms": 0.33753794, "gate_a2b/L3_absmean": 0.33890334, "gate_a2b/L3_absmax": 0.54191595, "gate_a2b/L3_rms": 0.34330928, "gate_a2b/L5_absmean": 0.51453108, "gate_a2b/L5_absmax": 0.66504943, "gate_a2b/L5_rms": 0.52001148, "gate_c2b/L1_absmean": 0.01441007, "gate_c2b/L1_absmax": 0.42815062, "gate_c2b/L1_rms": 0.05367635, "gate_c2b/L3_absmean": 0.21021946, "gate_c2b/L3_absmax": 0.42975429, "gate_c2b/L3_rms": 0.21422063, "gate_c2b/L5_absmean": 0.52946377, "gate_c2b/L5_absmax": 0.71601605, "gate_c2b/L5_rms": 0.54130882, "gate_b2a/L7_absmean": 0.09119718, "gate_b2a/L7_absmax": 0.28540426, "gate_b2a/L7_rms": 0.09441013, "gate_b2a/L11_absmean": 0.13733646, "gate_b2a/L11_absmax": 0.50202435, "gate_b2a/L11_rms": 0.14054413, "gate_b2a/L14_absmean": 0.32953736, "gate_b2a/L14_absmax": 0.57729036, "gate_b2a/L14_rms": 0.33112842, "gate_b2c/L3_absmean": 0.10851406, "gate_b2c/L3_absmax": 0.64391422, "gate_b2c/L3_rms": 0.12322602, "gate_b2c/L5_absmean": 0.12800477, "gate_b2c/L5_absmax": 0.80084908, "gate_b2c/L5_rms": 0.13677429, "gate_b2c/L6_absmean": 0.18768904, "gate_b2c/L6_absmax": 0.55127364, "gate_b2c/L6_rms": 0.19293214}, "streams": {"cos_a2b/round0": 0.388688, "cos_c2b/round0": 0.307548, "cos_b2a/round0": 0.110671, "cos_b2c/round0": 0.155418, "rms_a/round0": 0.81502, "rms_b/round0": 1.505651, "rms_c/round0": 1.216969, "cos_a2b/round1": 0.412791, "cos_c2b/round1": 0.275154, "cos_b2a/round1": 0.23209, "cos_b2c/round1": 0.127359, "rms_a/round1": 1.4302, "rms_b/round1": 3.016395, "rms_c/round1": 2.012386, "cos_a2b/round2": 0.549607, "cos_c2b/round2": 0.495988, "cos_b2a/round2": 0.06879, "cos_b2c/round2": 0.143362, "rms_a/round2": 3.280599, "rms_b/round2": 6.757193, "rms_c/round2": 3.226715}, "alpha_mean": 0.4716053157624458, "alpha_hist": [0.01076, 0.04871, 0.12041, 0.17433, 0.1897, 0.20651, 0.15007, 0.08366, 0.01572, 0.00014], "alpha_by_type": {"special": {"share": 0.00066, "mean": 0.53442, "std": 0.15697}, "whitespace": {"share": 0.0672, "mean": 0.55593, "std": 0.1155}, "digit": {"share": 0.03543, "mean": 0.46512, "std": 0.17757}, "word_start": {"share": 0.54028, "mean": 0.49862, "std": 0.15311}, "word_cont": {"share": 0.24097, "mean": 0.32954, "std": 0.13597}, "punct": {"share": 0.11546, "mean": 0.59423, "std": 0.15621}}, "alpha_p05": 0.20181, "alpha_p50": 0.47682, "alpha_p95": 0.75707}
{"step": 27510, "total_steps": 72479, "tokens": 14423162880, "loss": 4.91938453912735, "l_mix": 3.048578381538391, "l_a": 3.1393107175827026, "l_c": 3.091277301311493, "z_loss": 16.29614567756653, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1771318018436432, "tokens_per_second": 1323860.5095334107, "eta_seconds": 17809.056847167016, "data_wait_seconds": 2.0682811737060547e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 27520, "total_steps": 72479, "tokens": 14428405760, "loss": 4.950486898422241, "l_mix": 3.06799578666687, "l_a": 3.156311810016632, "l_c": 3.1130136251449585, "z_loss": 16.934076070785522, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18152879178524017, "tokens_per_second": 1444898.2198910771, "eta_seconds": 16313.581031179428, "data_wait_seconds": 6.178021430969238e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 27530, "total_steps": 72479, "tokens": 14433648640, "loss": 3.466548651456833, "l_mix": 2.131660044193268, "l_a": 2.2602169513702393, "l_c": 2.1841583847999573, "z_loss": 15.759016752243042, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.24855749309062958, "tokens_per_second": 1382908.162569669, "eta_seconds": 17041.060245251654, "data_wait_seconds": 3.108382225036621e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 27540, "total_steps": 72479, "tokens": 14438891520, "loss": 3.410104423761368, "l_mix": 2.0972633063793182, "l_a": 2.2194476425647736, "l_c": 2.1514039635658264, "z_loss": 15.85521686077118, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1965734213590622, "tokens_per_second": 1445154.6006966373, "eta_seconds": 16303.431079721451, "data_wait_seconds": 3.528594970703125e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 27550, "total_steps": 72479, "tokens": 14444134400, "loss": 4.586468040943146, "l_mix": 2.8389178216457367, "l_a": 2.933222860097885, "l_c": 2.8861992359161377, "z_loss": 17.235325574874878, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20923596620559692, "tokens_per_second": 1398807.6455332963, "eta_seconds": 16839.86760239601, "data_wait_seconds": 1.8417835235595703e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08232}}
{"step": 27560, "total_steps": 72479, "tokens": 14449377280, "loss": 4.6075462102890015, "l_mix": 2.851901024580002, "l_a": 2.9465406239032745, "l_c": 2.900020331144333, "z_loss": 16.76755118370056, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19764910638332367, "tokens_per_second": 1453454.939904669, "eta_seconds": 16203.111651706697, "data_wait_seconds": 1.5139579772949219e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 27570, "total_steps": 72479, "tokens": 14454620160, "loss": 4.4436299204826355, "l_mix": 2.7494978606700897, "l_a": 2.8485116362571716, "l_c": 2.7928467392921448, "z_loss": 17.244085788726807, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1821328103542328, "tokens_per_second": 1345331.0565637513, "eta_seconds": 17501.454141807557, "data_wait_seconds": 1.5437602996826172e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 27580, "total_steps": 72479, "tokens": 14459863040, "loss": 4.527382075786591, "l_mix": 2.8027320206165314, "l_a": 2.8988941609859467, "l_c": 2.844271808862686, "z_loss": 17.003916263580322, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19548465311527252, "tokens_per_second": 1377029.213658257, "eta_seconds": 17094.7766964674, "data_wait_seconds": 1.755356788635254e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 27590, "total_steps": 72479, "tokens": 14465105920, "loss": 4.583988189697266, "l_mix": 2.8381054997444153, "l_a": 2.9333657026290894, "l_c": 2.880490630865097, "z_loss": 17.25704288482666, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1720045655965805, "tokens_per_second": 1256201.3248139524, "eta_seconds": 18734.866431927683, "data_wait_seconds": 2.0503997802734375e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 27600, "total_steps": 72479, "tokens": 14470348800, "loss": 3.382763087749481, "l_mix": 2.0919660329818726, "l_a": 2.173076719045639, "l_c": 2.1231181770563126, "z_loss": 19.384814023971558, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.24898380041122437, "tokens_per_second": 1240373.5315503355, "eta_seconds": 18969.705942201614, "data_wait_seconds": 2.5600194931030273e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20764, "cosmopedia_v2": 0.20043, "finemath": 0.08236}, "gates": {"gate_a2b/L1_absmean": 0.32194793, "gate_a2b/L1_absmax": 0.64856851, "gate_a2b/L1_rms": 0.33561176, "gate_a2b/L3_absmean": 0.33878106, "gate_a2b/L3_absmax": 0.54014891, "gate_a2b/L3_rms": 0.34317183, "gate_a2b/L5_absmean": 0.51433116, "gate_a2b/L5_absmax": 0.66138899, "gate_a2b/L5_rms": 0.51976514, "gate_c2b/L1_absmean": 0.01400166, "gate_c2b/L1_absmax": 0.43505129, "gate_c2b/L1_rms": 0.05389409, "gate_c2b/L3_absmean": 0.21071707, "gate_c2b/L3_absmax": 0.43329886, "gate_c2b/L3_rms": 0.21460688, "gate_c2b/L5_absmean": 0.52963585, "gate_c2b/L5_absmax": 0.71599758, "gate_c2b/L5_rms": 0.54137373, "gate_b2a/L7_absmean": 0.09154999, "gate_b2a/L7_absmax": 0.28559762, "gate_b2a/L7_rms": 0.09472771, "gate_b2a/L11_absmean": 0.13789815, "gate_b2a/L11_absmax": 0.49918544, "gate_b2a/L11_rms": 0.14108001, "gate_b2a/L14_absmean": 0.32951668, "gate_b2a/L14_absmax": 0.57862419, "gate_b2a/L14_rms": 0.33112419, "gate_b2c/L3_absmean": 0.10871746, "gate_b2c/L3_absmax": 0.64594597, "gate_b2c/L3_rms": 0.12357563, "gate_b2c/L5_absmean": 0.12748462, "gate_b2c/L5_absmax": 0.80006129, "gate_b2c/L5_rms": 0.13631953, "gate_b2c/L6_absmean": 0.18728551, "gate_b2c/L6_absmax": 0.54960334, "gate_b2c/L6_rms": 0.19261484}, "streams": {"cos_a2b/round0": 0.418478, "cos_c2b/round0": 0.326432, "cos_b2a/round0": 0.112669, "cos_b2c/round0": 0.169036, "rms_a/round0": 0.852882, "rms_b/round0": 1.491079, "rms_c/round0": 1.19713, "cos_a2b/round1": 0.398829, "cos_c2b/round1": 0.250272, "cos_b2a/round1": 0.230791, "cos_b2c/round1": 0.107322, "rms_a/round1": 1.465384, "rms_b/round1": 3.029671, "rms_c/round1": 2.065676, "cos_a2b/round2": 0.499146, "cos_c2b/round2": 0.54693, "cos_b2a/round2": 0.06768, "cos_b2c/round2": 0.12634, "rms_a/round2": 3.220169, "rms_b/round2": 7.084079, "rms_c/round2": 3.376088}, "alpha_mean": 0.4486813007165651, "alpha_hist": [0.01523, 0.04689, 0.11958, 0.20212, 0.24597, 0.18472, 0.12035, 0.05801, 0.00679, 0.00034], "alpha_by_type": {"special": {"share": 0.00061, "mean": 0.51718, "std": 0.15636}, "whitespace": {"share": 0.08414, "mean": 0.50071, "std": 0.1366}, "digit": {"share": 0.09047, "mean": 0.46318, "std": 0.12116}, "word_start": {"share": 0.37305, "mean": 0.42068, "std": 0.15537}, "word_cont": {"share": 0.23883, "mean": 0.37105, "std": 0.13572}, "punct": {"share": 0.21291, "mean": 0.55791, "std": 0.15108}}, "alpha_p05": 0.18953, "alpha_p50": 0.44565, "alpha_p95": 0.70171}
{"step": 27610, "total_steps": 72479, "tokens": 14475591680, "loss": 5.066450595855713, "l_mix": 3.1415258646011353, "l_a": 3.2256633639335632, "l_c": 3.185387820005417, "z_loss": 16.09419047832489, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2484356164932251, "tokens_per_second": 1454602.3061136175, "eta_seconds": 16172.309209966661, "data_wait_seconds": 3.55839729309082e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 27620, "total_steps": 72479, "tokens": 14480834560, "loss": 4.941757261753082, "l_mix": 3.063258558511734, "l_a": 3.1472997069358826, "l_c": 3.108766257762909, "z_loss": 16.788339614868164, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1885402649641037, "tokens_per_second": 1396499.8809285285, "eta_seconds": 16841.4159665823, "data_wait_seconds": 2.3871660232543945e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 27630, "total_steps": 72479, "tokens": 14486077440, "loss": 4.944931924343109, "l_mix": 3.0640942752361298, "l_a": 3.1550023555755615, "l_c": 3.1087414622306824, "z_loss": 17.14482808113098, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1953684687614441, "tokens_per_second": 1462142.4832156263, "eta_seconds": 16081.738121914863, "data_wait_seconds": 2.1398067474365234e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 27640, "total_steps": 72479, "tokens": 14491320320, "loss": 4.946558833122253, "l_mix": 3.0664963126182556, "l_a": 3.147325724363327, "l_c": 3.113895922899246, "z_loss": 16.959514617919922, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18195714056491852, "tokens_per_second": 1444371.5953450871, "eta_seconds": 16275.970607399942, "data_wait_seconds": 2.9981136322021484e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 27650, "total_steps": 72479, "tokens": 14496563200, "loss": 4.7664185762405396, "l_mix": 2.95351779460907, "l_a": 3.0372975170612335, "l_c": 3.0000825822353363, "z_loss": 16.865474462509155, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22637127339839935, "tokens_per_second": 1355902.9839489667, "eta_seconds": 17334.062267160418, "data_wait_seconds": 2.1725893020629883e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 27660, "total_steps": 72479, "tokens": 14501806080, "loss": 4.599643707275391, "l_mix": 2.8466455042362213, "l_a": 2.943668842315674, "l_c": 2.894023209810257, "z_loss": 16.905057430267334, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18629887700080872, "tokens_per_second": 1457716.722400706, "eta_seconds": 16119.773829102518, "data_wait_seconds": 1.5497207641601562e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08232}}
{"step": 27670, "total_steps": 72479, "tokens": 14507048960, "loss": 4.546509146690369, "l_mix": 2.814801514148712, "l_a": 2.9069854617118835, "l_c": 2.859715014696121, "z_loss": 16.975713968276978, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22141125798225403, "tokens_per_second": 1446641.311712842, "eta_seconds": 16239.561805534364, "data_wait_seconds": 1.5020370483398438e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 27680, "total_steps": 72479, "tokens": 14512291840, "loss": 4.526011765003204, "l_mix": 2.8012001514434814, "l_a": 2.8971742391586304, "l_c": 2.8464381992816925, "z_loss": 17.27881932258606, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18331849575042725, "tokens_per_second": 1261017.373672082, "eta_seconds": 18625.895727038383, "data_wait_seconds": 2.524256706237793e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 27690, "total_steps": 72479, "tokens": 14517534720, "loss": 4.526745617389679, "l_mix": 2.802055835723877, "l_a": 2.897297888994217, "l_c": 2.8459526002407074, "z_loss": 17.145410537719727, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20110231637954712, "tokens_per_second": 1429287.8559537465, "eta_seconds": 16429.3953343153, "data_wait_seconds": 3.3468008041381836e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 27700, "total_steps": 72479, "tokens": 14522777600, "loss": 4.638586461544037, "l_mix": 2.871969521045685, "l_a": 2.96656796336174, "l_c": 2.916312873363495, "z_loss": 17.526007890701294, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2833906412124634, "tokens_per_second": 1347545.1715785568, "eta_seconds": 17422.119011044502, "data_wait_seconds": 3.0547380447387695e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.32322675, "gate_a2b/L1_absmax": 0.64623052, "gate_a2b/L1_rms": 0.33685669, "gate_a2b/L3_absmean": 0.33930954, "gate_a2b/L3_absmax": 0.53936476, "gate_a2b/L3_rms": 0.34372982, "gate_a2b/L5_absmean": 0.51458114, "gate_a2b/L5_absmax": 0.67215616, "gate_a2b/L5_rms": 0.52008241, "gate_c2b/L1_absmean": 0.01513022, "gate_c2b/L1_absmax": 0.4319118, "gate_c2b/L1_rms": 0.05382532, "gate_c2b/L3_absmean": 0.21040609, "gate_c2b/L3_absmax": 0.43799162, "gate_c2b/L3_rms": 0.21437697, "gate_c2b/L5_absmean": 0.52984661, "gate_c2b/L5_absmax": 0.71859962, "gate_c2b/L5_rms": 0.541686, "gate_b2a/L7_absmean": 0.09181888, "gate_b2a/L7_absmax": 0.28535235, "gate_b2a/L7_rms": 0.09497735, "gate_b2a/L11_absmean": 0.13765195, "gate_b2a/L11_absmax": 0.49845499, "gate_b2a/L11_rms": 0.14078978, "gate_b2a/L14_absmean": 0.32994613, "gate_b2a/L14_absmax": 0.58162034, "gate_b2a/L14_rms": 0.33157328, "gate_b2c/L3_absmean": 0.10970136, "gate_b2c/L3_absmax": 0.64547735, "gate_b2c/L3_rms": 0.12440024, "gate_b2c/L5_absmean": 0.12795365, "gate_b2c/L5_absmax": 0.80518728, "gate_b2c/L5_rms": 0.13670315, "gate_b2c/L6_absmean": 0.18759362, "gate_b2c/L6_absmax": 0.55017632, "gate_b2c/L6_rms": 0.19289604}, "streams": {"cos_a2b/round0": 0.390442, "cos_c2b/round0": 0.303764, "cos_b2a/round0": 0.106898, "cos_b2c/round0": 0.156241, "rms_a/round0": 0.832327, "rms_b/round0": 1.530465, "rms_c/round0": 1.241329, "cos_a2b/round1": 0.411356, "cos_c2b/round1": 0.265609, "cos_b2a/round1": 0.222201, "cos_b2c/round1": 0.114615, "rms_a/round1": 1.452964, "rms_b/round1": 3.075155, "rms_c/round1": 2.037986, "cos_a2b/round2": 0.539259, "cos_c2b/round2": 0.496664, "cos_b2a/round2": 0.05812, "cos_b2c/round2": 0.125553, "rms_a/round2": 3.306767, "rms_b/round2": 6.940444, "rms_c/round2": 3.312489}, "alpha_mean": 0.42606500229150157, "alpha_hist": [0.02133, 0.07719, 0.16528, 0.20346, 0.19472, 0.15262, 0.10944, 0.06367, 0.01196, 0.00032], "alpha_by_type": {"special": {"share": 0.00107, "mean": 0.50178, "std": 0.1136}, "whitespace": {"share": 0.02766, "mean": 0.48239, "std": 0.13176}, "digit": {"share": 0.04791, "mean": 0.39507, "std": 0.12578}, "word_start": {"share": 0.56645, "mean": 0.4522, "std": 0.16909}, "word_cont": {"share": 0.26125, "mean": 0.31258, "std": 0.13644}, "punct": {"share": 0.09566, "mean": 0.57961, "std": 0.1715}}, "alpha_p05": 0.15406, "alpha_p50": 0.42179, "alpha_p95": 0.74167}
{"step": 27710, "total_steps": 72479, "tokens": 14528020480, "loss": 3.205706387758255, "l_mix": 1.9814426749944687, "l_a": 2.056392729282379, "l_c": 2.0179241746664047, "z_loss": 19.686418056488037, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.24503932893276215, "tokens_per_second": 1389038.0697590408, "eta_seconds": 16897.9166107893, "data_wait_seconds": 3.2842159271240234e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 27720, "total_steps": 72479, "tokens": 14533263360, "loss": 3.1956430673599243, "l_mix": 1.9749325513839722, "l_a": 2.0510261952877045, "l_c": 2.011541798710823, "z_loss": 19.401407957077026, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2709137201309204, "tokens_per_second": 1247615.5026711114, "eta_seconds": 18809.165597701074, "data_wait_seconds": 2.613663673400879e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08236}}
{"step": 27730, "total_steps": 72479, "tokens": 14538506240, "loss": 4.978626370429993, "l_mix": 3.0867180824279785, "l_a": 3.1714457273483276, "l_c": 3.129382222890854, "z_loss": 16.59981679916382, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18795230984687805, "tokens_per_second": 1368943.8759813462, "eta_seconds": 17138.294800567626, "data_wait_seconds": 4.0024518966674805e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 27740, "total_steps": 72479, "tokens": 14543749120, "loss": 4.806890547275543, "l_mix": 2.9784536361694336, "l_a": 3.067168116569519, "l_c": 3.0220045149326324, "z_loss": 16.84944498538971, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1707375943660736, "tokens_per_second": 1396296.909620606, "eta_seconds": 16798.80594906807, "data_wait_seconds": 1.6689300537109375e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20768, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 27750, "total_steps": 72479, "tokens": 14548992000, "loss": 4.573699355125427, "l_mix": 2.831657886505127, "l_a": 2.9278274178504944, "l_c": 2.8732117414474487, "z_loss": 17.296767950057983, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2875343859195709, "tokens_per_second": 1481274.6071881813, "eta_seconds": 15831.553338050842, "data_wait_seconds": 1.7881393432617188e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20764, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 27760, "total_steps": 72479, "tokens": 14554234880, "loss": 4.585791707038879, "l_mix": 2.8381398022174835, "l_a": 2.936789035797119, "l_c": 2.882979780435562, "z_loss": 17.21143412590027, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19335679709911346, "tokens_per_second": 1383400.486715281, "eta_seconds": 16947.829133462903, "data_wait_seconds": 1.5527009963989258e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 27770, "total_steps": 72479, "tokens": 14559477760, "loss": 4.577265441417694, "l_mix": 2.8338789343833923, "l_a": 2.9278766810894012, "l_c": 2.877649188041687, "z_loss": 17.286211013793945, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17412567138671875, "tokens_per_second": 1399229.0867266613, "eta_seconds": 16752.362007308006, "data_wait_seconds": 1.8209218978881836e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08232}}
{"step": 27780, "total_steps": 72479, "tokens": 14564720640, "loss": 4.547562479972839, "l_mix": 2.8141327500343323, "l_a": 2.909802585840225, "l_c": 2.862561374902725, "z_loss": 17.20502018928528, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16086602210998535, "tokens_per_second": 1480752.1464968899, "eta_seconds": 15826.517197656633, "data_wait_seconds": 1.5556812286376953e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 27790, "total_steps": 72479, "tokens": 14569963520, "loss": 4.505913436412811, "l_mix": 2.788195103406906, "l_a": 2.8873073160648346, "l_c": 2.8325918316841125, "z_loss": 17.484211444854736, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17325179278850555, "tokens_per_second": 1406771.0299022365, "eta_seconds": 16655.09591395855, "data_wait_seconds": 2.4944543838500977e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 27800, "total_steps": 72479, "tokens": 14575206400, "loss": 4.439235687255859, "l_mix": 2.747049778699875, "l_a": 2.840075135231018, "l_c": 2.7948183119297028, "z_loss": 17.1771981716156, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1882859319448471, "tokens_per_second": 1461600.3015105303, "eta_seconds": 16026.723262023927, "data_wait_seconds": 2.104043960571289e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.32425168, "gate_a2b/L1_absmax": 0.65346956, "gate_a2b/L1_rms": 0.33763412, "gate_a2b/L3_absmean": 0.33967134, "gate_a2b/L3_absmax": 0.54050738, "gate_a2b/L3_rms": 0.34401539, "gate_a2b/L5_absmean": 0.51421481, "gate_a2b/L5_absmax": 0.67141569, "gate_a2b/L5_rms": 0.51971275, "gate_c2b/L1_absmean": 0.01470994, "gate_c2b/L1_absmax": 0.43039754, "gate_c2b/L1_rms": 0.0535843, "gate_c2b/L3_absmean": 0.21023691, "gate_c2b/L3_absmax": 0.43743467, "gate_c2b/L3_rms": 0.21416354, "gate_c2b/L5_absmean": 0.52986133, "gate_c2b/L5_absmax": 0.72140568, "gate_c2b/L5_rms": 0.54166281, "gate_b2a/L7_absmean": 0.09155409, "gate_b2a/L7_absmax": 0.2856271, "gate_b2a/L7_rms": 0.0947876, "gate_b2a/L11_absmean": 0.13795537, "gate_b2a/L11_absmax": 0.50132477, "gate_b2a/L11_rms": 0.14113009, "gate_b2a/L14_absmean": 0.32967857, "gate_b2a/L14_absmax": 0.57564098, "gate_b2a/L14_rms": 0.33128777, "gate_b2c/L3_absmean": 0.10956717, "gate_b2c/L3_absmax": 0.64321971, "gate_b2c/L3_rms": 0.12423127, "gate_b2c/L5_absmean": 0.12795258, "gate_b2c/L5_absmax": 0.80794966, "gate_b2c/L5_rms": 0.13682279, "gate_b2c/L6_absmean": 0.18758333, "gate_b2c/L6_absmax": 0.54623145, "gate_b2c/L6_rms": 0.19292869}, "streams": {"cos_a2b/round0": 0.392908, "cos_c2b/round0": 0.31257, "cos_b2a/round0": 0.103127, "cos_b2c/round0": 0.155468, "rms_a/round0": 0.825184, "rms_b/round0": 1.52067, "rms_c/round0": 1.235931, "cos_a2b/round1": 0.413606, "cos_c2b/round1": 0.264052, "cos_b2a/round1": 0.223021, "cos_b2c/round1": 0.110808, "rms_a/round1": 1.452211, "rms_b/round1": 3.075794, "rms_c/round1": 2.036006, "cos_a2b/round2": 0.53042, "cos_c2b/round2": 0.506309, "cos_b2a/round2": 0.055712, "cos_b2c/round2": 0.124485, "rms_a/round2": 3.266337, "rms_b/round2": 6.925223, "rms_c/round2": 3.310665}, "alpha_mean": 0.4416562062811664, "alpha_hist": [0.0175, 0.08026, 0.15535, 0.18553, 0.18045, 0.16136, 0.12012, 0.07822, 0.02045, 0.00076], "alpha_by_type": {"special": {"share": 0.00107, "mean": 0.58234, "std": 0.09956}, "whitespace": {"share": 0.02983, "mean": 0.55232, "std": 0.13337}, "digit": {"share": 0.04509, "mean": 0.41725, "std": 0.14706}, "word_start": {"share": 0.5638, "mean": 0.4497, "std": 0.18022}, "word_cont": {"share": 0.25571, "mean": 0.34637, "std": 0.14878}, "punct": {"share": 0.10451, "mean": 0.60892, "std": 0.16462}}, "alpha_p05": 0.15078, "alpha_p50": 0.44915, "alpha_p95": 0.74316}
{"step": 27810, "total_steps": 72479, "tokens": 14580449280, "loss": 4.60810786485672, "l_mix": 2.852536231279373, "l_a": 2.9478999376296997, "l_c": 2.898423820734024, "z_loss": 16.74515151977539, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21017619967460632, "tokens_per_second": 1397747.8227412628, "eta_seconds": 16755.111537981033, "data_wait_seconds": 2.8848648071289062e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 27820, "total_steps": 72479, "tokens": 14585692160, "loss": 4.557275652885437, "l_mix": 2.821050614118576, "l_a": 2.920828640460968, "l_c": 2.860853224992752, "z_loss": 17.20456290245056, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20899598300457, "tokens_per_second": 1403362.9271632987, "eta_seconds": 16684.33541944027, "data_wait_seconds": 1.963973045349121e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 27830, "total_steps": 72479, "tokens": 14590935040, "loss": 2.9627539813518524, "l_mix": 1.8322084993124008, "l_a": 1.8974206782877445, "l_c": 1.8647452741861343, "z_loss": 18.95689105987549, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.24042414128780365, "tokens_per_second": 1422429.0533346392, "eta_seconds": 16457.014047288892, "data_wait_seconds": 2.4259090423583984e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08236}}
{"step": 27840, "total_steps": 72479, "tokens": 14596177920, "loss": 4.580610066652298, "l_mix": 2.8355852663517, "l_a": 2.9311296343803406, "l_c": 2.8798471689224243, "z_loss": 17.317126989364624, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18353767693042755, "tokens_per_second": 1276751.0365593415, "eta_seconds": 18330.662252736092, "data_wait_seconds": 2.8371810913085938e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 27850, "total_steps": 72479, "tokens": 14601420800, "loss": 4.566168129444122, "l_mix": 2.825972944498062, "l_a": 2.9228223860263824, "l_c": 2.8720635771751404, "z_loss": 17.293614864349365, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16709177196025848, "tokens_per_second": 1403462.7927707944, "eta_seconds": 16671.941196107862, "data_wait_seconds": 1.5020370483398438e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 27860, "total_steps": 72479, "tokens": 14606663680, "loss": 4.503136515617371, "l_mix": 2.7869352996349335, "l_a": 2.8859823048114777, "l_c": 2.8288411498069763, "z_loss": 17.54207968711853, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2280571460723877, "tokens_per_second": 1384025.378912749, "eta_seconds": 16902.29574429989, "data_wait_seconds": 1.8894672393798828e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 27870, "total_steps": 72479, "tokens": 14611906560, "loss": 4.55015367269516, "l_mix": 2.817177027463913, "l_a": 2.905812382698059, "l_c": 2.8649809658527374, "z_loss": 17.38574743270874, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21627865731716156, "tokens_per_second": 1447855.2483684148, "eta_seconds": 16153.523232626914, "data_wait_seconds": 1.8268823623657227e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 27880, "total_steps": 72479, "tokens": 14617149440, "loss": 4.519934117794037, "l_mix": 2.797890782356262, "l_a": 2.8880759477615356, "l_c": 2.8463769257068634, "z_loss": 17.073673486709595, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22842532396316528, "tokens_per_second": 1399381.5267122816, "eta_seconds": 16709.324845051768, "data_wait_seconds": 1.8417835235595703e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 27890, "total_steps": 72479, "tokens": 14622392320, "loss": 4.556436538696289, "l_mix": 2.8205411434173584, "l_a": 2.914488673210144, "l_c": 2.866172879934311, "z_loss": 16.968321561813354, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17133867740631104, "tokens_per_second": 1447729.8075935424, "eta_seconds": 16147.679981017112, "data_wait_seconds": 1.9371509552001953e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08232}}
{"step": 27900, "total_steps": 72479, "tokens": 14627635200, "loss": 4.5365577936172485, "l_mix": 2.8070539832115173, "l_a": 2.905550569295883, "l_c": 2.8537266850471497, "z_loss": 17.20540976524353, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17163939774036407, "tokens_per_second": 1450408.6234435397, "eta_seconds": 16114.241444945335, "data_wait_seconds": 2.3066997528076172e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.32590029, "gate_a2b/L1_absmax": 0.65705937, "gate_a2b/L1_rms": 0.3392939, "gate_a2b/L3_absmean": 0.34014714, "gate_a2b/L3_absmax": 0.53774387, "gate_a2b/L3_rms": 0.34445536, "gate_a2b/L5_absmean": 0.51507348, "gate_a2b/L5_absmax": 0.67097205, "gate_a2b/L5_rms": 0.52056843, "gate_c2b/L1_absmean": 0.01474966, "gate_c2b/L1_absmax": 0.42963871, "gate_c2b/L1_rms": 0.05343833, "gate_c2b/L3_absmean": 0.21024708, "gate_c2b/L3_absmax": 0.43859568, "gate_c2b/L3_rms": 0.21414787, "gate_c2b/L5_absmean": 0.529594, "gate_c2b/L5_absmax": 0.71372461, "gate_c2b/L5_rms": 0.54132241, "gate_b2a/L7_absmean": 0.09174137, "gate_b2a/L7_absmax": 0.28850308, "gate_b2a/L7_rms": 0.09495195, "gate_b2a/L11_absmean": 0.13758811, "gate_b2a/L11_absmax": 0.50233138, "gate_b2a/L11_rms": 0.14081855, "gate_b2a/L14_absmean": 0.32973933, "gate_b2a/L14_absmax": 0.5745427, "gate_b2a/L14_rms": 0.33132955, "gate_b2c/L3_absmean": 0.11018814, "gate_b2c/L3_absmax": 0.64439684, "gate_b2c/L3_rms": 0.12484562, "gate_b2c/L5_absmean": 0.12856224, "gate_b2c/L5_absmax": 0.81108505, "gate_b2c/L5_rms": 0.13738213, "gate_b2c/L6_absmean": 0.18839052, "gate_b2c/L6_absmax": 0.54626405, "gate_b2c/L6_rms": 0.1936935}, "streams": {"cos_a2b/round0": 0.401312, "cos_c2b/round0": 0.305546, "cos_b2a/round0": 0.098395, "cos_b2c/round0": 0.157777, "rms_a/round0": 0.832355, "rms_b/round0": 1.51623, "rms_c/round0": 1.248122, "cos_a2b/round1": 0.417933, "cos_c2b/round1": 0.261591, "cos_b2a/round1": 0.22441, "cos_b2c/round1": 0.106376, "rms_a/round1": 1.460742, "rms_b/round1": 3.100204, "rms_c/round1": 2.047214, "cos_a2b/round2": 0.526921, "cos_c2b/round2": 0.502695, "cos_b2a/round2": 0.054443, "cos_b2c/round2": 0.119672, "rms_a/round2": 3.297489, "rms_b/round2": 6.958657, "rms_c/round2": 3.353087}, "alpha_mean": 0.43810860372583704, "alpha_hist": [0.01617, 0.08244, 0.17485, 0.19014, 0.16698, 0.1456, 0.11185, 0.08423, 0.02725, 0.00049], "alpha_by_type": {"special": {"share": 0.00092, "mean": 0.60246, "std": 0.10846}, "whitespace": {"share": 0.03189, "mean": 0.5215, "std": 0.12464}, "digit": {"share": 0.05917, "mean": 0.39322, "std": 0.14118}, "word_start": {"share": 0.56329, "mean": 0.46182, "std": 0.18405}, "word_cont": {"share": 0.24995, "mean": 0.32233, "std": 0.14618}, "punct": {"share": 0.09477, "mean": 0.60091, "std": 0.18116}}, "alpha_p05": 0.17189, "alpha_p50": 0.43063, "alpha_p95": 0.75491}
{"step": 27910, "total_steps": 72479, "tokens": 14632878080, "loss": 4.929402768611908, "l_mix": 3.055829495191574, "l_a": 3.139902025461197, "l_c": 3.0998732447624207, "z_loss": 16.405210733413696, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20748603343963623, "tokens_per_second": 1423688.1170314935, "eta_seconds": 16412.99916214943, "data_wait_seconds": 4.270672798156738e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 27920, "total_steps": 72479, "tokens": 14638120960, "loss": 4.861775636672974, "l_mix": 3.013103872537613, "l_a": 3.098023384809494, "l_c": 3.0584511160850525, "z_loss": 17.29389977455139, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1812373846769333, "tokens_per_second": 1372184.0062145113, "eta_seconds": 17025.23049838543, "data_wait_seconds": 1.7136335372924805e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 27930, "total_steps": 72479, "tokens": 14643363840, "loss": 4.768521666526794, "l_mix": 2.955121338367462, "l_a": 3.0405077040195465, "l_c": 2.998553693294525, "z_loss": 16.81835126876831, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18460465967655182, "tokens_per_second": 1396769.1816346676, "eta_seconds": 16721.808026051523, "data_wait_seconds": 2.3543834686279297e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 27940, "total_steps": 72479, "tokens": 14648606720, "loss": 3.1366134881973267, "l_mix": 1.9388263523578644, "l_a": 2.0147350430488586, "l_c": 1.9713515788316727, "z_loss": 19.610042572021484, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.24283598363399506, "tokens_per_second": 1415357.5909491188, "eta_seconds": 16498.49012103081, "data_wait_seconds": 6.186962127685547e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08236}}
{"step": 27950, "total_steps": 72479, "tokens": 14653849600, "loss": 4.6799450516700745, "l_mix": 2.8972554802894592, "l_a": 2.993214428424835, "l_c": 2.9433634877204895, "z_loss": 17.161745309829712, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21230553090572357, "tokens_per_second": 1384042.539411123, "eta_seconds": 16867.99335079193, "data_wait_seconds": 2.378225326538086e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 27960, "total_steps": 72479, "tokens": 14659092480, "loss": 4.580997705459595, "l_mix": 2.835820436477661, "l_a": 2.930282950401306, "l_c": 2.881231665611267, "z_loss": 17.228163480758667, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15847118198871613, "tokens_per_second": 1462285.311357747, "eta_seconds": 15961.849093818666, "data_wait_seconds": 1.659989356994629e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 27970, "total_steps": 72479, "tokens": 14664335360, "loss": 4.656600892543793, "l_mix": 2.8833464682102203, "l_a": 2.976445347070694, "l_c": 2.9287168979644775, "z_loss": 17.05650758743286, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1958618313074112, "tokens_per_second": 1289676.9434780278, "eta_seconds": 18094.093028497697, "data_wait_seconds": 2.7477741241455078e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 27980, "total_steps": 72479, "tokens": 14669578240, "loss": 4.626345098018646, "l_mix": 2.863736569881439, "l_a": 2.9595242738723755, "l_c": 2.909927248954773, "z_loss": 17.730942010879517, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19479332864284515, "tokens_per_second": 1461792.5798438417, "eta_seconds": 15960.056189703942, "data_wait_seconds": 1.9282102584838867e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 27990, "total_steps": 72479, "tokens": 14674821120, "loss": 4.6437103152275085, "l_mix": 2.8762556314468384, "l_a": 2.9644605815410614, "l_c": 2.9213484823703766, "z_loss": 17.119582653045654, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2044145166873932, "tokens_per_second": 1391244.1973121497, "eta_seconds": 16765.603678393363, "data_wait_seconds": 1.6391277313232422e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 28000, "total_steps": 72479, "tokens": 14680064000, "loss": 2.9283443689346313, "l_mix": 1.8096130639314651, "l_a": 1.8849510699510574, "l_c": 1.8376003205776215, "z_loss": 19.657756328582764, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2585034668445587, "tokens_per_second": 1426691.510485066, "eta_seconds": 16345.373741006852, "data_wait_seconds": 2.6702880859375e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20764, "cosmopedia_v2": 0.20043, "finemath": 0.08236}, "gates": {"gate_a2b/L1_absmean": 0.32554874, "gate_a2b/L1_absmax": 0.6569038, "gate_a2b/L1_rms": 0.33894899, "gate_a2b/L3_absmean": 0.34038189, "gate_a2b/L3_absmax": 0.54043394, "gate_a2b/L3_rms": 0.34471729, "gate_a2b/L5_absmean": 0.51547825, "gate_a2b/L5_absmax": 0.67196399, "gate_a2b/L5_rms": 0.520859, "gate_c2b/L1_absmean": 0.01506248, "gate_c2b/L1_absmax": 0.42863995, "gate_c2b/L1_rms": 0.05357498, "gate_c2b/L3_absmean": 0.21022534, "gate_c2b/L3_absmax": 0.4341346, "gate_c2b/L3_rms": 0.21414006, "gate_c2b/L5_absmean": 0.53015745, "gate_c2b/L5_absmax": 0.71485758, "gate_c2b/L5_rms": 0.54192275, "gate_b2a/L7_absmean": 0.09187365, "gate_b2a/L7_absmax": 0.28857195, "gate_b2a/L7_rms": 0.09504855, "gate_b2a/L11_absmean": 0.13777742, "gate_b2a/L11_absmax": 0.50269353, "gate_b2a/L11_rms": 0.1409995, "gate_b2a/L14_absmean": 0.3298656, "gate_b2a/L14_absmax": 0.57354599, "gate_b2a/L14_rms": 0.3314245, "gate_b2c/L3_absmean": 0.11026759, "gate_b2c/L3_absmax": 0.64433771, "gate_b2c/L3_rms": 0.125026, "gate_b2c/L5_absmean": 0.1284609, "gate_b2c/L5_absmax": 0.81713033, "gate_b2c/L5_rms": 0.13754688, "gate_b2c/L6_absmean": 0.18833452, "gate_b2c/L6_absmax": 0.54596806, "gate_b2c/L6_rms": 0.19369145}, "streams": {"cos_a2b/round0": 0.418713, "cos_c2b/round0": 0.31733, "cos_b2a/round0": 0.113083, "cos_b2c/round0": 0.160926, "rms_a/round0": 0.878466, "rms_b/round0": 1.485956, "rms_c/round0": 1.15911, "cos_a2b/round1": 0.389349, "cos_c2b/round1": 0.235405, "cos_b2a/round1": 0.219841, "cos_b2c/round1": 0.104011, "rms_a/round1": 1.462201, "rms_b/round1": 3.040495, "rms_c/round1": 2.023266, "cos_a2b/round2": 0.470895, "cos_c2b/round2": 0.572144, "cos_b2a/round2": 0.064102, "cos_b2c/round2": 0.115141, "rms_a/round2": 3.088912, "rms_b/round2": 6.936433, "rms_c/round2": 3.403587}, "alpha_mean": 0.4005093971644129, "alpha_hist": [0.02242, 0.06456, 0.18623, 0.25397, 0.20604, 0.15651, 0.07677, 0.0302, 0.00313, 0.00018], "alpha_by_type": {"special": {"share": 0.00018, "mean": 0.58924, "std": 0.09741}, "whitespace": {"share": 0.07332, "mean": 0.50126, "std": 0.14471}, "digit": {"share": 0.09192, "mean": 0.37112, "std": 0.12951}, "word_start": {"share": 0.20412, "mean": 0.3923, "std": 0.17825}, "word_cont": {"share": 0.28311, "mean": 0.30657, "std": 0.10929}, "punct": {"share": 0.34735, "mean": 0.46831, "std": 0.13352}}, "alpha_p05": 0.15482, "alpha_p50": 0.38862, "alpha_p95": 0.67918}
{"step": 28010, "total_steps": 72479, "tokens": 14685306880, "loss": 4.819208085536957, "l_mix": 2.9867328107357025, "l_a": 3.072344958782196, "l_c": 3.0303589701652527, "z_loss": 16.64076566696167, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1876446008682251, "tokens_per_second": 1327499.674196752, "eta_seconds": 17562.76368663311, "data_wait_seconds": 2.8848648071289062e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20768, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 28020, "total_steps": 72479, "tokens": 14690549760, "loss": 5.013817608356476, "l_mix": 3.108306050300598, "l_a": 3.193817973136902, "l_c": 3.1522265672683716, "z_loss": 16.981977939605713, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1845003366470337, "tokens_per_second": 1446294.5067447978, "eta_seconds": 16116.579357314109, "data_wait_seconds": 2.8818845748901367e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 28030, "total_steps": 72479, "tokens": 14695792640, "loss": 4.954301118850708, "l_mix": 3.0707353353500366, "l_a": 3.1527853310108185, "l_c": 3.1201968789100647, "z_loss": 16.710991621017456, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1858530044555664, "tokens_per_second": 1356495.4939731895, "eta_seconds": 17179.620142889024, "data_wait_seconds": 4.455447196960449e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 28040, "total_steps": 72479, "tokens": 14701035520, "loss": 5.134808540344238, "l_mix": 3.1845248341560364, "l_a": 3.268056273460388, "l_c": 3.227309077978134, "z_loss": 16.74035620689392, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22482767701148987, "tokens_per_second": 1277877.441606006, "eta_seconds": 18232.4483345747, "data_wait_seconds": 2.8908252716064453e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 28050, "total_steps": 72479, "tokens": 14706278400, "loss": 5.029846787452698, "l_mix": 3.1182225048542023, "l_a": 3.2041035890579224, "l_c": 3.162273496389389, "z_loss": 17.110130786895752, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19184958934783936, "tokens_per_second": 1409532.3475932826, "eta_seconds": 16525.7587679863, "data_wait_seconds": 2.1010637283325195e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 28060, "total_steps": 72479, "tokens": 14711521280, "loss": 4.698697865009308, "l_mix": 2.9093097448349, "l_a": 3.0069609582424164, "l_c": 2.952071249485016, "z_loss": 16.78312087059021, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15193262696266174, "tokens_per_second": 1427754.6303021386, "eta_seconds": 16311.17012526989, "data_wait_seconds": 2.4765729904174805e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08232}}
{"step": 28070, "total_steps": 72479, "tokens": 14716764160, "loss": 4.624308109283447, "l_mix": 2.86214342713356, "l_a": 2.958655059337616, "l_c": 2.9094816744327545, "z_loss": 17.23612642288208, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1544613093137741, "tokens_per_second": 1417266.176950417, "eta_seconds": 16428.181361174586, "data_wait_seconds": 1.52587890625e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 28080, "total_steps": 72479, "tokens": 14722007040, "loss": 4.5870601534843445, "l_mix": 2.8389990031719208, "l_a": 2.937237113714218, "l_c": 2.883894205093384, "z_loss": 17.217442274093628, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16610203683376312, "tokens_per_second": 1365436.9619392352, "eta_seconds": 17047.922065138817, "data_wait_seconds": 1.9103288650512695e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 28090, "total_steps": 72479, "tokens": 14727249920, "loss": 4.6238017082214355, "l_mix": 2.8627160489559174, "l_a": 2.959936261177063, "l_c": 2.90476855635643, "z_loss": 16.740834712982178, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20924100279808044, "tokens_per_second": 1416128.141531463, "eta_seconds": 16433.978924274445, "data_wait_seconds": 1.9222497940063477e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 28100, "total_steps": 72479, "tokens": 14732492800, "loss": 4.637070000171661, "l_mix": 2.8715358078479767, "l_a": 2.9660018384456635, "l_c": 2.913444757461548, "z_loss": 17.001314878463745, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.24403738975524902, "tokens_per_second": 1463756.2447294681, "eta_seconds": 15895.663800430299, "data_wait_seconds": 1.4662742614746094e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.32608148, "gate_a2b/L1_absmax": 0.65975428, "gate_a2b/L1_rms": 0.33954021, "gate_a2b/L3_absmean": 0.34057915, "gate_a2b/L3_absmax": 0.53662407, "gate_a2b/L3_rms": 0.34492075, "gate_a2b/L5_absmean": 0.51584017, "gate_a2b/L5_absmax": 0.67411357, "gate_a2b/L5_rms": 0.52125371, "gate_c2b/L1_absmean": 0.01498951, "gate_c2b/L1_absmax": 0.42878541, "gate_c2b/L1_rms": 0.05356807, "gate_c2b/L3_absmean": 0.21039152, "gate_c2b/L3_absmax": 0.4368937, "gate_c2b/L3_rms": 0.21436459, "gate_c2b/L5_absmean": 0.52951217, "gate_c2b/L5_absmax": 0.71758401, "gate_c2b/L5_rms": 0.5412972, "gate_b2a/L7_absmean": 0.09205977, "gate_b2a/L7_absmax": 0.28890309, "gate_b2a/L7_rms": 0.09533594, "gate_b2a/L11_absmean": 0.13825448, "gate_b2a/L11_absmax": 0.50471282, "gate_b2a/L11_rms": 0.14141445, "gate_b2a/L14_absmean": 0.33032566, "gate_b2a/L14_absmax": 0.58054322, "gate_b2a/L14_rms": 0.33196887, "gate_b2c/L3_absmean": 0.1102955, "gate_b2c/L3_absmax": 0.64521849, "gate_b2c/L3_rms": 0.12498531, "gate_b2c/L5_absmean": 0.12844145, "gate_b2c/L5_absmax": 0.81200868, "gate_b2c/L5_rms": 0.13745376, "gate_b2c/L6_absmean": 0.18821308, "gate_b2c/L6_absmax": 0.54789263, "gate_b2c/L6_rms": 0.19353242}, "streams": {"cos_a2b/round0": 0.389924, "cos_c2b/round0": 0.302315, "cos_b2a/round0": 0.113357, "cos_b2c/round0": 0.156014, "rms_a/round0": 0.822222, "rms_b/round0": 1.524826, "rms_c/round0": 1.236747, "cos_a2b/round1": 0.419599, "cos_c2b/round1": 0.275193, "cos_b2a/round1": 0.235941, "cos_b2c/round1": 0.128057, "rms_a/round1": 1.444108, "rms_b/round1": 3.073505, "rms_c/round1": 2.033237, "cos_a2b/round2": 0.54766, "cos_c2b/round2": 0.497564, "cos_b2a/round2": 0.069314, "cos_b2c/round2": 0.142105, "rms_a/round2": 3.31134, "rms_b/round2": 6.784405, "rms_c/round2": 3.315599}, "alpha_mean": 0.43107914275466186, "alpha_hist": [0.01924, 0.07832, 0.16904, 0.18523, 0.18486, 0.16571, 0.12234, 0.06593, 0.00923, 9e-05], "alpha_by_type": {"special": {"share": 0.00095, "mean": 0.53111, "std": 0.10204}, "whitespace": {"share": 0.02766, "mean": 0.50936, "std": 0.11356}, "digit": {"share": 0.02942, "mean": 0.44204, "std": 0.13335}, "word_start": {"share": 0.59471, "mean": 0.45216, "std": 0.16537}, "word_cont": {"share": 0.24214, "mean": 0.30405, "std": 0.13971}, "punct": {"share": 0.10512, "mean": 0.57989, "std": 0.16731}}, "alpha_p05": 0.15817, "alpha_p50": 0.44203, "alpha_p95": 0.71463}
{"step": 28110, "total_steps": 72479, "tokens": 14737735680, "loss": 3.3142068684101105, "l_mix": 2.048809126019478, "l_a": 2.1285672336816788, "l_c": 2.0832448452711105, "z_loss": 18.539631843566895, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.24951866269111633, "tokens_per_second": 1430270.3771386023, "eta_seconds": 16264.151620435716, "data_wait_seconds": 3.6716461181640625e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08236}}
{"step": 28120, "total_steps": 72479, "tokens": 14742978560, "loss": 3.075802192091942, "l_mix": 1.9012254178524017, "l_a": 1.9740401655435562, "l_c": 1.9348085820674896, "z_loss": 19.22092866897583, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2839125394821167, "tokens_per_second": 1455445.9581237084, "eta_seconds": 15979.220157361031, "data_wait_seconds": 1.6689300537109375e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08236}}
{"step": 28130, "total_steps": 72479, "tokens": 14748221440, "loss": 4.908635318279266, "l_mix": 3.042170226573944, "l_a": 3.1316984593868256, "l_c": 3.0842165648937225, "z_loss": 16.905652046203613, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18266449868679047, "tokens_per_second": 1459232.2141129554, "eta_seconds": 15934.166123199464, "data_wait_seconds": 3.4302473068237305e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 28140, "total_steps": 72479, "tokens": 14753464320, "loss": 4.947788774967194, "l_mix": 3.0667344331741333, "l_a": 3.1543952226638794, "l_c": 3.1103300750255585, "z_loss": 16.367851972579956, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1977282166481018, "tokens_per_second": 1454341.9853654664, "eta_seconds": 15984.139814376831, "data_wait_seconds": 2.3663043975830078e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 28150, "total_steps": 72479, "tokens": 14758707200, "loss": 4.776602149009705, "l_mix": 2.959889739751816, "l_a": 3.047780603170395, "l_c": 3.0021307468414307, "z_loss": 17.388838052749634, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21089749038219452, "tokens_per_second": 1366070.676576155, "eta_seconds": 17013.14811196327, "data_wait_seconds": 2.0563602447509766e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 28160, "total_steps": 72479, "tokens": 14763950080, "loss": 4.917948067188263, "l_mix": 3.04824697971344, "l_a": 3.1351753771305084, "l_c": 3.091423988342285, "z_loss": 17.211498975753784, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.181190624833107, "tokens_per_second": 1344260.6214122865, "eta_seconds": 17285.27898674011, "data_wait_seconds": 2.0712614059448242e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 28170, "total_steps": 72479, "tokens": 14769192960, "loss": 4.701445817947388, "l_mix": 2.9110875129699707, "l_a": 3.006392180919647, "l_c": 2.955851763486862, "z_loss": 16.851495504379272, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20213262736797333, "tokens_per_second": 1436207.6834613455, "eta_seconds": 16175.012332487106, "data_wait_seconds": 2.956390380859375e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08232}}
{"step": 28180, "total_steps": 72479, "tokens": 14774435840, "loss": 4.7145607471466064, "l_mix": 2.918885976076126, "l_a": 3.0133028626441956, "l_c": 2.966576188802719, "z_loss": 17.11070489883423, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1506841629743576, "tokens_per_second": 1384453.647453457, "eta_seconds": 16775.88423037529, "data_wait_seconds": 1.8298625946044922e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 28190, "total_steps": 72479, "tokens": 14779678720, "loss": 4.652238190174103, "l_mix": 2.880146324634552, "l_a": 2.974024146795273, "l_c": 2.9272562861442566, "z_loss": 17.07695245742798, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.14098364114761353, "tokens_per_second": 1445722.2855056336, "eta_seconds": 16061.30822274685, "data_wait_seconds": 1.9937753677368164e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 28200, "total_steps": 72479, "tokens": 14784921600, "loss": 4.679544031620026, "l_mix": 2.8979637920856476, "l_a": 2.9915446639060974, "l_c": 2.9413710236549377, "z_loss": 17.054545164108276, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.14141248166561127, "tokens_per_second": 1304600.269455543, "eta_seconds": 17794.683088397982, "data_wait_seconds": 1.9311904907226562e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.32661295, "gate_a2b/L1_absmax": 0.65586007, "gate_a2b/L1_rms": 0.34017581, "gate_a2b/L3_absmean": 0.34126845, "gate_a2b/L3_absmax": 0.54223377, "gate_a2b/L3_rms": 0.34556061, "gate_a2b/L5_absmean": 0.51623964, "gate_a2b/L5_absmax": 0.67893159, "gate_a2b/L5_rms": 0.52167422, "gate_c2b/L1_absmean": 0.01461462, "gate_c2b/L1_absmax": 0.42769438, "gate_c2b/L1_rms": 0.05347712, "gate_c2b/L3_absmean": 0.21068697, "gate_c2b/L3_absmax": 0.43777624, "gate_c2b/L3_rms": 0.2146159, "gate_c2b/L5_absmean": 0.52960634, "gate_c2b/L5_absmax": 0.71163738, "gate_c2b/L5_rms": 0.54130447, "gate_b2a/L7_absmean": 0.09198118, "gate_b2a/L7_absmax": 0.28952071, "gate_b2a/L7_rms": 0.0952526, "gate_b2a/L11_absmean": 0.13825533, "gate_b2a/L11_absmax": 0.50581187, "gate_b2a/L11_rms": 0.14146286, "gate_b2a/L14_absmean": 0.33041772, "gate_b2a/L14_absmax": 0.57480955, "gate_b2a/L14_rms": 0.33199486, "gate_b2c/L3_absmean": 0.11084895, "gate_b2c/L3_absmax": 0.64672065, "gate_b2c/L3_rms": 0.12549227, "gate_b2c/L5_absmean": 0.12874363, "gate_b2c/L5_absmax": 0.81450701, "gate_b2c/L5_rms": 0.13785243, "gate_b2c/L6_absmean": 0.18893765, "gate_b2c/L6_absmax": 0.54912966, "gate_b2c/L6_rms": 0.19425817}, "streams": {"cos_a2b/round0": 0.390064, "cos_c2b/round0": 0.303195, "cos_b2a/round0": 0.105232, "cos_b2c/round0": 0.154585, "rms_a/round0": 0.819195, "rms_b/round0": 1.517455, "rms_c/round0": 1.232766, "cos_a2b/round1": 0.41033, "cos_c2b/round1": 0.270236, "cos_b2a/round1": 0.228247, "cos_b2c/round1": 0.120155, "rms_a/round1": 1.43655, "rms_b/round1": 3.063474, "rms_c/round1": 2.019502, "cos_a2b/round2": 0.546315, "cos_c2b/round2": 0.491678, "cos_b2a/round2": 0.063259, "cos_b2c/round2": 0.138429, "rms_a/round2": 3.330307, "rms_b/round2": 6.89573, "rms_c/round2": 3.281029}, "alpha_mean": 0.42865100996806405, "alpha_hist": [0.01682, 0.07956, 0.16029, 0.1937, 0.19814, 0.16879, 0.1163, 0.05748, 0.00883, 9e-05], "alpha_by_type": {"special": {"share": 0.00116, "mean": 0.55686, "std": 0.11}, "whitespace": {"share": 0.02986, "mean": 0.51778, "std": 0.11751}, "digit": {"share": 0.0455, "mean": 0.44086, "std": 0.13129}, "word_start": {"share": 0.5388, "mean": 0.4525, "std": 0.1625}, "word_cont": {"share": 0.27184, "mean": 0.31067, "std": 0.13566}, "punct": {"share": 0.11284, "mean": 0.56915, "std": 0.15736}}, "alpha_p05": 0.15844, "alpha_p50": 0.43254, "alpha_p95": 0.73144}
{"step": 28210, "total_steps": 72479, "tokens": 14790164480, "loss": 4.678355693817139, "l_mix": 2.8965795040130615, "l_a": 2.99119171500206, "l_c": 2.9424789547920227, "z_loss": 16.74839210510254, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2164798527956009, "tokens_per_second": 1463498.2866482981, "eta_seconds": 15859.058861732485, "data_wait_seconds": 1.4156103134155273e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 28220, "total_steps": 72479, "tokens": 14795407360, "loss": 3.2323279678821564, "l_mix": 1.9982011914253235, "l_a": 2.0773457884788513, "l_c": 2.03028804063797, "z_loss": 18.36558485031128, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.369651734828949, "tokens_per_second": 1422348.3658153343, "eta_seconds": 16314.190777516365, "data_wait_seconds": 2.6226043701171875e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 28230, "total_steps": 72479, "tokens": 14800650240, "loss": 3.0311182737350464, "l_mix": 1.8739879727363586, "l_a": 1.9385842084884644, "l_c": 1.9119183272123337, "z_loss": 19.79401445388794, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.6276255249977112, "tokens_per_second": 1441183.9050178754, "eta_seconds": 16097.334719896317, "data_wait_seconds": 2.3573637008666992e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08236}}
{"step": 28240, "total_steps": 72479, "tokens": 14805893120, "loss": 4.999694108963013, "l_mix": 3.0988996028900146, "l_a": 3.1879594027996063, "l_c": 3.1424417197704315, "z_loss": 16.741538763046265, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16211260855197906, "tokens_per_second": 1432793.6049422582, "eta_seconds": 16187.939946126937, "data_wait_seconds": 3.7163496017456055e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 28250, "total_steps": 72479, "tokens": 14811136000, "loss": 5.077457547187805, "l_mix": 3.1468398571014404, "l_a": 3.236403375864029, "l_c": 3.193534880876541, "z_loss": 16.360581874847412, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2304341197013855, "tokens_per_second": 1443229.507589607, "eta_seconds": 16067.25321929455, "data_wait_seconds": 3.528594970703125e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 28260, "total_steps": 72479, "tokens": 14816378880, "loss": 4.964231967926025, "l_mix": 3.0770739316940308, "l_a": 3.1675931811332703, "l_c": 3.117181360721588, "z_loss": 17.255632162094116, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1964588463306427, "tokens_per_second": 1374921.629390574, "eta_seconds": 16861.68184165955, "data_wait_seconds": 1.570582389831543e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20768, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 28270, "total_steps": 72479, "tokens": 14821621760, "loss": 4.798412919044495, "l_mix": 2.9733774662017822, "l_a": 3.056662827730179, "l_c": 3.0211983919143677, "z_loss": 16.77074408531189, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22459502518177032, "tokens_per_second": 1398825.0855806004, "eta_seconds": 16569.797347021104, "data_wait_seconds": 3.725290298461914e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20768, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 28280, "total_steps": 72479, "tokens": 14826864640, "loss": 4.734375894069672, "l_mix": 2.9308409988880157, "l_a": 3.0294324159622192, "l_c": 2.9767560362815857, "z_loss": 16.78291916847229, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.23681922256946564, "tokens_per_second": 1326553.1173047235, "eta_seconds": 17468.58456680775, "data_wait_seconds": 2.390146255493164e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20042, "finemath": 0.08236}}
{"step": 28290, "total_steps": 72479, "tokens": 14832107520, "loss": 4.575797617435455, "l_mix": 2.832272380590439, "l_a": 2.9289805591106415, "l_c": 2.877031981945038, "z_loss": 17.21442937850952, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15652547776699066, "tokens_per_second": 1346220.5429686524, "eta_seconds": 17209.485141944886, "data_wait_seconds": 1.570582389831543e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 28300, "total_steps": 72479, "tokens": 14837350400, "loss": 4.693325459957123, "l_mix": 2.906253933906555, "l_a": 3.0031523406505585, "l_c": 2.9479976296424866, "z_loss": 17.2646701335907, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1566500961780548, "tokens_per_second": 1456131.573804662, "eta_seconds": 15906.886416506766, "data_wait_seconds": 1.9162893295288086e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.3270888, "gate_a2b/L1_absmax": 0.65578145, "gate_a2b/L1_rms": 0.34053692, "gate_a2b/L3_absmean": 0.34158283, "gate_a2b/L3_absmax": 0.54352152, "gate_a2b/L3_rms": 0.34589833, "gate_a2b/L5_absmean": 0.5155822, "gate_a2b/L5_absmax": 0.67832875, "gate_a2b/L5_rms": 0.52101952, "gate_c2b/L1_absmean": 0.01453026, "gate_c2b/L1_absmax": 0.42974266, "gate_c2b/L1_rms": 0.05344385, "gate_c2b/L3_absmean": 0.21015862, "gate_c2b/L3_absmax": 0.433236, "gate_c2b/L3_rms": 0.21408086, "gate_c2b/L5_absmean": 0.52855402, "gate_c2b/L5_absmax": 0.70863247, "gate_c2b/L5_rms": 0.54018617, "gate_b2a/L7_absmean": 0.09178963, "gate_b2a/L7_absmax": 0.28932682, "gate_b2a/L7_rms": 0.09504366, "gate_b2a/L11_absmean": 0.1380679, "gate_b2a/L11_absmax": 0.50747997, "gate_b2a/L11_rms": 0.1412662, "gate_b2a/L14_absmean": 0.32950178, "gate_b2a/L14_absmax": 0.5716691, "gate_b2a/L14_rms": 0.33115, "gate_b2c/L3_absmean": 0.11087444, "gate_b2c/L3_absmax": 0.64639556, "gate_b2c/L3_rms": 0.12561275, "gate_b2c/L5_absmean": 0.12822676, "gate_b2c/L5_absmax": 0.8091625, "gate_b2c/L5_rms": 0.13722356, "gate_b2c/L6_absmean": 0.18849441, "gate_b2c/L6_absmax": 0.5455246, "gate_b2c/L6_rms": 0.19382685}, "streams": {"cos_a2b/round0": 0.389071, "cos_c2b/round0": 0.303826, "cos_b2a/round0": 0.10814, "cos_b2c/round0": 0.154556, "rms_a/round0": 0.818631, "rms_b/round0": 1.515007, "rms_c/round0": 1.237587, "cos_a2b/round1": 0.420027, "cos_c2b/round1": 0.279419, "cos_b2a/round1": 0.232917, "cos_b2c/round1": 0.128211, "rms_a/round1": 1.438183, "rms_b/round1": 3.052395, "rms_c/round1": 2.05013, "cos_a2b/round2": 0.547193, "cos_c2b/round2": 0.500099, "cos_b2a/round2": 0.065474, "cos_b2c/round2": 0.136617, "rms_a/round2": 3.327262, "rms_b/round2": 6.865272, "rms_c/round2": 3.360312}, "alpha_mean": 0.4691230481236204, "alpha_hist": [0.01051, 0.05244, 0.13928, 0.18176, 0.19029, 0.16281, 0.13277, 0.10022, 0.02966, 0.00024], "alpha_by_type": {"special": {"share": 0.00124, "mean": 0.56252, "std": 0.08535}, "whitespace": {"share": 0.02704, "mean": 0.52861, "std": 0.1121}, "digit": {"share": 0.03363, "mean": 0.38054, "std": 0.11175}, "word_start": {"share": 0.58832, "mean": 0.4993, "std": 0.17079}, "word_cont": {"share": 0.25186, "mean": 0.34245, "std": 0.14396}, "punct": {"share": 0.09792, "mean": 0.62646, "std": 0.1683}}, "alpha_p05": 0.19163, "alpha_p50": 0.44976, "alpha_p95": 0.76348}
{"step": 28310, "total_steps": 72479, "tokens": 14842593280, "loss": 4.62976211309433, "l_mix": 2.865971177816391, "l_a": 2.9628793597221375, "l_c": 2.9106417298316956, "z_loss": 17.344074726104736, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16466374695301056, "tokens_per_second": 1365924.47390382, "eta_seconds": 16953.555715870858, "data_wait_seconds": 2.1159648895263672e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 28320, "total_steps": 72479, "tokens": 14847836160, "loss": 4.626763880252838, "l_mix": 2.864284574985504, "l_a": 2.961307018995285, "l_c": 2.9080038964748383, "z_loss": 16.859471797943115, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19806483387947083, "tokens_per_second": 1354825.0147414985, "eta_seconds": 17088.57862830162, "data_wait_seconds": 1.5556812286376953e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 28330, "total_steps": 72479, "tokens": 14853079040, "loss": 4.612235128879547, "l_mix": 2.855267196893692, "l_a": 2.9532638788223267, "l_c": 2.897646814584732, "z_loss": 16.945478677749634, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20160244405269623, "tokens_per_second": 1310098.4979614452, "eta_seconds": 17667.977597117424, "data_wait_seconds": 2.7865171432495117e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 28340, "total_steps": 72479, "tokens": 14858321920, "loss": 3.191638946533203, "l_mix": 1.9733860492706299, "l_a": 2.051379933953285, "l_c": 2.0028497129678726, "z_loss": 19.839550971984863, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2380460798740387, "tokens_per_second": 1391757.6260751989, "eta_seconds": 16627.57048959732, "data_wait_seconds": 2.4318695068359375e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08236}}
{"step": 28350, "total_steps": 72479, "tokens": 14863564800, "loss": 3.308488756418228, "l_mix": 2.0453885197639465, "l_a": 2.1187489330768585, "l_c": 2.085048884153366, "z_loss": 19.60777997970581, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.28440964221954346, "tokens_per_second": 1404985.1948148864, "eta_seconds": 16467.294628715514, "data_wait_seconds": 1.704692840576172e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08236}}
{"step": 28360, "total_steps": 72479, "tokens": 14868807680, "loss": 4.615359365940094, "l_mix": 2.85572150349617, "l_a": 2.958859443664551, "l_c": 2.900914281606674, "z_loss": 17.057619094848633, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20434994995594025, "tokens_per_second": 1315141.1911153852, "eta_seconds": 17588.272976517677, "data_wait_seconds": 1.6957521438598633e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 28370, "total_steps": 72479, "tokens": 14874050560, "loss": 4.653794348239899, "l_mix": 2.880705416202545, "l_a": 2.97856941819191, "l_c": 2.9261142015457153, "z_loss": 16.839369297027588, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15290318429470062, "tokens_per_second": 1434724.8248104686, "eta_seconds": 16118.64448993206, "data_wait_seconds": 2.047419548034668e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 28380, "total_steps": 72479, "tokens": 14879293440, "loss": 4.703948795795441, "l_mix": 2.911669284105301, "l_a": 3.0117647647857666, "l_c": 2.9567078351974487, "z_loss": 17.377236366271973, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18743234872817993, "tokens_per_second": 1431998.5686365154, "eta_seconds": 16145.670127320289, "data_wait_seconds": 2.5391578674316406e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 28390, "total_steps": 72479, "tokens": 14884536320, "loss": 4.629420876502991, "l_mix": 2.865576833486557, "l_a": 2.9609738290309906, "l_c": 2.9128569066524506, "z_loss": 16.948476791381836, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18410679697990417, "tokens_per_second": 1360498.6732042504, "eta_seconds": 16990.338974428178, "data_wait_seconds": 1.8209218978881836e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 28400, "total_steps": 72479, "tokens": 14889779200, "loss": 4.69939661026001, "l_mix": 2.910018652677536, "l_a": 3.002895027399063, "l_c": 2.955906003713608, "z_loss": 17.375840187072754, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15922294557094574, "tokens_per_second": 1390352.1679603527, "eta_seconds": 16621.753311538698, "data_wait_seconds": 1.6301870346069336e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20042, "finemath": 0.08236}, "gates": {"gate_a2b/L1_absmean": 0.32661623, "gate_a2b/L1_absmax": 0.65777218, "gate_a2b/L1_rms": 0.34006655, "gate_a2b/L3_absmean": 0.34199995, "gate_a2b/L3_absmax": 0.54505092, "gate_a2b/L3_rms": 0.34630412, "gate_a2b/L5_absmean": 0.51601475, "gate_a2b/L5_absmax": 0.6800378, "gate_a2b/L5_rms": 0.52145106, "gate_c2b/L1_absmean": 0.01486352, "gate_c2b/L1_absmax": 0.42939278, "gate_c2b/L1_rms": 0.05369086, "gate_c2b/L3_absmean": 0.21030612, "gate_c2b/L3_absmax": 0.43037951, "gate_c2b/L3_rms": 0.2141979, "gate_c2b/L5_absmean": 0.52954131, "gate_c2b/L5_absmax": 0.71210825, "gate_c2b/L5_rms": 0.54126948, "gate_b2a/L7_absmean": 0.09231886, "gate_b2a/L7_absmax": 0.28996864, "gate_b2a/L7_rms": 0.09553849, "gate_b2a/L11_absmean": 0.13871226, "gate_b2a/L11_absmax": 0.505279, "gate_b2a/L11_rms": 0.14187495, "gate_b2a/L14_absmean": 0.33042401, "gate_b2a/L14_absmax": 0.57199925, "gate_b2a/L14_rms": 0.33200586, "gate_b2c/L3_absmean": 0.11107886, "gate_b2c/L3_absmax": 0.64655322, "gate_b2c/L3_rms": 0.12574758, "gate_b2c/L5_absmean": 0.12869099, "gate_b2c/L5_absmax": 0.81878871, "gate_b2c/L5_rms": 0.13783047, "gate_b2c/L6_absmean": 0.18895382, "gate_b2c/L6_absmax": 0.55212885, "gate_b2c/L6_rms": 0.19437787}, "streams": {"cos_a2b/round0": 0.388592, "cos_c2b/round0": 0.300214, "cos_b2a/round0": 0.108593, "cos_b2c/round0": 0.155335, "rms_a/round0": 0.824871, "rms_b/round0": 1.531889, "rms_c/round0": 1.233486, "cos_a2b/round1": 0.420131, "cos_c2b/round1": 0.275749, "cos_b2a/round1": 0.228508, "cos_b2c/round1": 0.125975, "rms_a/round1": 1.454259, "rms_b/round1": 3.063413, "rms_c/round1": 2.055882, "cos_a2b/round2": 0.54743, "cos_c2b/round2": 0.495182, "cos_b2a/round2": 0.06231, "cos_b2c/round2": 0.129605, "rms_a/round2": 3.328124, "rms_b/round2": 6.854835, "rms_c/round2": 3.353473}, "alpha_mean": 0.4529597117774955, "alpha_hist": [0.01654, 0.06522, 0.14493, 0.18076, 0.18517, 0.17242, 0.13304, 0.08244, 0.01939, 9e-05], "alpha_by_type": {"special": {"share": 0.00148, "mean": 0.58205, "std": 0.11582}, "whitespace": {"share": 0.03075, "mean": 0.53742, "std": 0.13554}, "digit": {"share": 0.03517, "mean": 0.50654, "std": 0.11612}, "word_start": {"share": 0.56595, "mean": 0.46771, "std": 0.1702}, "word_cont": {"share": 0.2587, "mean": 0.33842, "std": 0.15123}, "punct": {"share": 0.10796, "mean": 0.60686, "std": 0.16719}}, "alpha_p05": 0.16238, "alpha_p50": 0.44915, "alpha_p95": 0.76419}
{"step": 28410, "total_steps": 72479, "tokens": 14895022080, "loss": 4.73527067899704, "l_mix": 2.9322427213191986, "l_a": 3.026743561029434, "l_c": 2.977663993835449, "z_loss": 17.055960655212402, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.14781008660793304, "tokens_per_second": 1443913.1402204745, "eta_seconds": 16001.54969742298, "data_wait_seconds": 2.0742416381835938e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 28420, "total_steps": 72479, "tokens": 14900264960, "loss": 4.70538330078125, "l_mix": 2.9137193262577057, "l_a": 3.010036677122116, "l_c": 2.9565468430519104, "z_loss": 16.887628078460693, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1599072813987732, "tokens_per_second": 1395072.4427967665, "eta_seconds": 16557.99676301479, "data_wait_seconds": 1.570582389831543e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 28430, "total_steps": 72479, "tokens": 14905507840, "loss": 4.728199005126953, "l_mix": 2.9276382327079773, "l_a": 3.0223280787467957, "l_c": 2.9737906754016876, "z_loss": 17.25159788131714, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21264928579330444, "tokens_per_second": 1361516.5657801197, "eta_seconds": 16962.233653593063, "data_wait_seconds": 1.6927719116210938e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 28440, "total_steps": 72479, "tokens": 14910750720, "loss": 4.7415183782577515, "l_mix": 2.9377334117889404, "l_a": 3.0266230404376984, "l_c": 2.9802418649196625, "z_loss": 17.25454020500183, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.2170519083738327, "tokens_per_second": 1404887.894806378, "eta_seconds": 16434.848159313202, "data_wait_seconds": 1.9103288650512695e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 28450, "total_steps": 72479, "tokens": 14915993600, "loss": 3.2921278178691864, "l_mix": 2.0358404964208603, "l_a": 2.1058112382888794, "l_c": 2.0755430459976196, "z_loss": 18.809715270996094, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.29279130697250366, "tokens_per_second": 1447688.9201237033, "eta_seconds": 15945.3291595459, "data_wait_seconds": 2.0384788513183594e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08236}}
{"step": 28460, "total_steps": 72479, "tokens": 14921236480, "loss": 3.0123235285282135, "l_mix": 1.8621100187301636, "l_a": 1.936070367693901, "l_c": 1.8914298564195633, "z_loss": 19.63351535797119, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.24056664109230042, "tokens_per_second": 1469635.1170188764, "eta_seconds": 15703.648616409302, "data_wait_seconds": 2.0623207092285156e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08236}}
{"step": 28470, "total_steps": 72479, "tokens": 14926479360, "loss": 4.673862934112549, "l_mix": 2.894016593694687, "l_a": 2.9862439334392548, "l_c": 2.94079452753067, "z_loss": 17.347253799438477, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15921975672245026, "tokens_per_second": 1464796.304910213, "eta_seconds": 15751.944836735725, "data_wait_seconds": 1.5795230865478516e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 28480, "total_steps": 72479, "tokens": 14931722240, "loss": 4.647058725357056, "l_mix": 2.8761729896068573, "l_a": 2.9742141366004944, "l_c": 2.922960013151169, "z_loss": 17.33443570137024, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16539019346237183, "tokens_per_second": 1473681.595360398, "eta_seconds": 15653.413725614548, "data_wait_seconds": 2.3990869522094727e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 28490, "total_steps": 72479, "tokens": 14936965120, "loss": 4.6036006808280945, "l_mix": 2.8496806919574738, "l_a": 2.946265459060669, "l_c": 2.8944396674633026, "z_loss": 17.08361840248108, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16113673150539398, "tokens_per_second": 1417076.027323824, "eta_seconds": 16274.994698452949, "data_wait_seconds": 2.485513687133789e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 28500, "total_steps": 72479, "tokens": 14942208000, "loss": 4.660101771354675, "l_mix": 2.8848880529403687, "l_a": 2.980158358812332, "l_c": 2.9314652383327484, "z_loss": 17.265052318572998, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18646441400051117, "tokens_per_second": 1423183.284686522, "eta_seconds": 16201.470464205742, "data_wait_seconds": 1.659989356994629e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.32709429, "gate_a2b/L1_absmax": 0.65369046, "gate_a2b/L1_rms": 0.34053627, "gate_a2b/L3_absmean": 0.34201226, "gate_a2b/L3_absmax": 0.54219604, "gate_a2b/L3_rms": 0.34631425, "gate_a2b/L5_absmean": 0.51565236, "gate_a2b/L5_absmax": 0.67834711, "gate_a2b/L5_rms": 0.52115476, "gate_c2b/L1_absmean": 0.01482858, "gate_c2b/L1_absmax": 0.42346665, "gate_c2b/L1_rms": 0.05340065, "gate_c2b/L3_absmean": 0.2110313, "gate_c2b/L3_absmax": 0.43210581, "gate_c2b/L3_rms": 0.2149668, "gate_c2b/L5_absmean": 0.52900195, "gate_c2b/L5_absmax": 0.72217178, "gate_c2b/L5_rms": 0.54067475, "gate_b2a/L7_absmean": 0.09170192, "gate_b2a/L7_absmax": 0.29428199, "gate_b2a/L7_rms": 0.09500653, "gate_b2a/L11_absmean": 0.13890955, "gate_b2a/L11_absmax": 0.51264518, "gate_b2a/L11_rms": 0.14216439, "gate_b2a/L14_absmean": 0.33063722, "gate_b2a/L14_absmax": 0.57149887, "gate_b2a/L14_rms": 0.33227411, "gate_b2c/L3_absmean": 0.11130795, "gate_b2c/L3_absmax": 0.64596128, "gate_b2c/L3_rms": 0.12603219, "gate_b2c/L5_absmean": 0.1293354, "gate_b2c/L5_absmax": 0.82451153, "gate_b2c/L5_rms": 0.13859294, "gate_b2c/L6_absmean": 0.18891546, "gate_b2c/L6_absmax": 0.55281723, "gate_b2c/L6_rms": 0.19432864}, "streams": {"cos_a2b/round0": 0.395076, "cos_c2b/round0": 0.302504, "cos_b2a/round0": 0.106773, "cos_b2c/round0": 0.157802, "rms_a/round0": 0.829194, "rms_b/round0": 1.521892, "rms_c/round0": 1.238504, "cos_a2b/round1": 0.418387, "cos_c2b/round1": 0.283997, "cos_b2a/round1": 0.228405, "cos_b2c/round1": 0.117746, "rms_a/round1": 1.450027, "rms_b/round1": 3.057978, "rms_c/round1": 2.038699, "cos_a2b/round2": 0.551154, "cos_c2b/round2": 0.496523, "cos_b2a/round2": 0.06681, "cos_b2c/round2": 0.130512, "rms_a/round2": 3.340104, "rms_b/round2": 6.843894, "rms_c/round2": 3.313976}, "alpha_mean": 0.46597307317628633, "alpha_hist": [0.01416, 0.06017, 0.13615, 0.17259, 0.1761, 0.17242, 0.1563, 0.0925, 0.0195, 0.00011], "alpha_by_type": {"special": {"share": 0.00117, "mean": 0.58021, "std": 0.11176}, "whitespace": {"share": 0.02701, "mean": 0.55649, "std": 0.12192}, "digit": {"share": 0.03789, "mean": 0.45628, "std": 0.11709}, "word_start": {"share": 0.57899, "mean": 0.49398, "std": 0.16855}, "word_cont": {"share": 0.24821, "mean": 0.32953, "std": 0.14674}, "punct": {"share": 0.10672, "mean": 0.61062, "std": 0.16385}}, "alpha_p05": 0.16858, "alpha_p50": 0.46922, "alpha_p95": 0.74651}
{"step": 28510, "total_steps": 72479, "tokens": 14947450880, "loss": 4.65627521276474, "l_mix": 2.8816975355148315, "l_a": 2.980515331029892, "l_c": 2.929119110107422, "z_loss": 16.87249755859375, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21463976800441742, "tokens_per_second": 1380391.9101176634, "eta_seconds": 16699.90884692669, "data_wait_seconds": 1.5109777450561523e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08236}}
{"step": 28520, "total_steps": 72479, "tokens": 14952693760, "loss": 4.829526126384735, "l_mix": 2.9930694699287415, "l_a": 3.0821183919906616, "l_c": 3.033746600151062, "z_loss": 16.97139263153076, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15096695721149445, "tokens_per_second": 1432804.9009827608, "eta_seconds": 16085.355498290062, "data_wait_seconds": 2.2858381271362305e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20768, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 28530, "total_steps": 72479, "tokens": 14957936640, "loss": 4.90767365694046, "l_mix": 3.0412018597126007, "l_a": 3.1277571618556976, "l_c": 3.088060885667801, "z_loss": 17.26178550720215, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1653708815574646, "tokens_per_second": 1422633.2511302829, "eta_seconds": 16196.67844378948, "data_wait_seconds": 2.1696090698242188e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20768, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 28540, "total_steps": 72479, "tokens": 14963179520, "loss": 4.8189563155174255, "l_mix": 2.9855892062187195, "l_a": 3.0748276710510254, "l_c": 3.030727446079254, "z_loss": 17.005618572235107, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17950139939785004, "tokens_per_second": 1417507.9105013406, "eta_seconds": 16251.542768359184, "data_wait_seconds": 2.3305416107177734e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 28550, "total_steps": 72479, "tokens": 14968422400, "loss": 4.820051431655884, "l_mix": 2.9857148826122284, "l_a": 3.071189731359482, "l_c": 3.0375536382198334, "z_loss": 17.13415813446045, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1971510648727417, "tokens_per_second": 1458907.995727747, "eta_seconds": 15786.771763157843, "data_wait_seconds": 2.5719404220581055e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 28560, "total_steps": 72479, "tokens": 14973665280, "loss": 4.922722578048706, "l_mix": 3.0508081316947937, "l_a": 3.1387106478214264, "l_c": 3.0953910052776337, "z_loss": 16.840057134628296, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16321133077144623, "tokens_per_second": 1392034.617523068, "eta_seconds": 16541.40233449936, "data_wait_seconds": 3.793835639953613e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 28570, "total_steps": 72479, "tokens": 14978908160, "loss": 3.033120572566986, "l_mix": 1.8756247013807297, "l_a": 1.9457421600818634, "l_c": 1.9059405773878098, "z_loss": 19.910093069076538, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.24095477163791656, "tokens_per_second": 1461303.1905925232, "eta_seconds": 15753.720336890221, "data_wait_seconds": 2.5063753128051758e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08236}}
{"step": 28580, "total_steps": 72479, "tokens": 14984151040, "loss": 4.599678337574005, "l_mix": 2.8475159406661987, "l_a": 2.9416074454784393, "l_c": 2.8932068049907684, "z_loss": 17.1808123588562, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16215765476226807, "tokens_per_second": 1458375.1733929394, "eta_seconds": 15781.75447025299, "data_wait_seconds": 3.382563591003418e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 28590, "total_steps": 72479, "tokens": 14989393920, "loss": 4.683860003948212, "l_mix": 2.899853438138962, "l_a": 2.9964540600776672, "l_c": 2.9446552097797394, "z_loss": 16.735291242599487, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17673613131046295, "tokens_per_second": 1437097.743866667, "eta_seconds": 16011.768253207207, "data_wait_seconds": 1.6361474990844727e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 28600, "total_steps": 72479, "tokens": 14994636800, "loss": 4.700704514980316, "l_mix": 2.9106280505657196, "l_a": 3.0049880743026733, "l_c": 2.9561479687690735, "z_loss": 17.35531258583069, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17133072018623352, "tokens_per_second": 1419021.119090556, "eta_seconds": 16212.044234228133, "data_wait_seconds": 2.0205974578857422e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.32829294, "gate_a2b/L1_absmax": 0.6563018, "gate_a2b/L1_rms": 0.34162205, "gate_a2b/L3_absmean": 0.34277508, "gate_a2b/L3_absmax": 0.54217803, "gate_a2b/L3_rms": 0.34703293, "gate_a2b/L5_absmean": 0.51582545, "gate_a2b/L5_absmax": 0.67196316, "gate_a2b/L5_rms": 0.5212869, "gate_c2b/L1_absmean": 0.01448465, "gate_c2b/L1_absmax": 0.42306194, "gate_c2b/L1_rms": 0.05358634, "gate_c2b/L3_absmean": 0.21059489, "gate_c2b/L3_absmax": 0.43077698, "gate_c2b/L3_rms": 0.21443611, "gate_c2b/L5_absmean": 0.52893901, "gate_c2b/L5_absmax": 0.71442711, "gate_c2b/L5_rms": 0.54060888, "gate_b2a/L7_absmean": 0.09201312, "gate_b2a/L7_absmax": 0.29031631, "gate_b2a/L7_rms": 0.09525238, "gate_b2a/L11_absmean": 0.13888851, "gate_b2a/L11_absmax": 0.51039797, "gate_b2a/L11_rms": 0.14211701, "gate_b2a/L14_absmean": 0.33075529, "gate_b2a/L14_absmax": 0.57533753, "gate_b2a/L14_rms": 0.33235553, "gate_b2c/L3_absmean": 0.11174121, "gate_b2c/L3_absmax": 0.64396948, "gate_b2c/L3_rms": 0.12635462, "gate_b2c/L5_absmean": 0.12993117, "gate_b2c/L5_absmax": 0.8231892, "gate_b2c/L5_rms": 0.13906394, "gate_b2c/L6_absmean": 0.18936762, "gate_b2c/L6_absmax": 0.55131263, "gate_b2c/L6_rms": 0.19479124}, "streams": {"cos_a2b/round0": 0.392647, "cos_c2b/round0": 0.303715, "cos_b2a/round0": 0.103888, "cos_b2c/round0": 0.155216, "rms_a/round0": 0.81542, "rms_b/round0": 1.50569, "rms_c/round0": 1.228112, "cos_a2b/round1": 0.421811, "cos_c2b/round1": 0.275268, "cos_b2a/round1": 0.225754, "cos_b2c/round1": 0.116223, "rms_a/round1": 1.444044, "rms_b/round1": 3.038905, "rms_c/round1": 2.019314, "cos_a2b/round2": 0.539847, "cos_c2b/round2": 0.505944, "cos_b2a/round2": 0.068058, "cos_b2c/round2": 0.133451, "rms_a/round2": 3.300621, "rms_b/round2": 6.890381, "rms_c/round2": 3.304302}, "alpha_mean": 0.4566403491607005, "alpha_hist": [0.01141, 0.05563, 0.14796, 0.18741, 0.1875, 0.17464, 0.13672, 0.0826, 0.0159, 0.00023], "alpha_by_type": {"special": {"share": 0.00099, "mean": 0.5972, "std": 0.09707}, "whitespace": {"share": 0.03044, "mean": 0.5706, "std": 0.11988}, "digit": {"share": 0.0426, "mean": 0.52267, "std": 0.12693}, "word_start": {"share": 0.56013, "mean": 0.46421, "std": 0.17051}, "word_cont": {"share": 0.26869, "mean": 0.35898, "std": 0.14275}, "punct": {"share": 0.09714, "mean": 0.61702, "std": 0.15519}}, "alpha_p05": 0.17217, "alpha_p50": 0.4436, "alpha_p95": 0.73564}
{"step": 28610, "total_steps": 72479, "tokens": 14999879680, "loss": 4.612772464752197, "l_mix": 2.8553219735622406, "l_a": 2.950502723455429, "l_c": 2.901959151029587, "z_loss": 17.1184663772583, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15413910150527954, "tokens_per_second": 1475582.8880307472, "eta_seconds": 15587.054077792167, "data_wait_seconds": 1.6510486602783203e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 28620, "total_steps": 72479, "tokens": 15005122560, "loss": 4.60019850730896, "l_mix": 2.8468528985977173, "l_a": 2.9434301257133484, "l_c": 2.8952101469039917, "z_loss": 17.534581184387207, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17964833974838257, "tokens_per_second": 1413905.8163200812, "eta_seconds": 16263.280854058266, "data_wait_seconds": 1.9073486328125e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 28630, "total_steps": 72479, "tokens": 15010365440, "loss": 3.213015615940094, "l_mix": 1.9858218729496002, "l_a": 2.064409762620926, "l_c": 2.0195843130350113, "z_loss": 19.954774379730225, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.24743689596652985, "tokens_per_second": 1460582.332169972, "eta_seconds": 15739.957964468003, "data_wait_seconds": 1.6003847122192383e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08236}}
{"step": 28640, "total_steps": 72479, "tokens": 15015608320, "loss": 4.907301664352417, "l_mix": 3.0416463911533356, "l_a": 3.1281480193138123, "l_c": 3.0851518511772156, "z_loss": 16.652480602264404, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18278570473194122, "tokens_per_second": 1471351.4086297762, "eta_seconds": 15621.191169691085, "data_wait_seconds": 1.6242265701293945e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20768, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 28650, "total_steps": 72479, "tokens": 15020851200, "loss": 4.988885343074799, "l_mix": 3.0923138856887817, "l_a": 3.1796433329582214, "l_c": 3.136636346578598, "z_loss": 16.873204708099365, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20157316327095032, "tokens_per_second": 1371872.1493259869, "eta_seconds": 16750.11681175232, "data_wait_seconds": 1.77919864654541e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 28660, "total_steps": 72479, "tokens": 15026094080, "loss": 4.896666765213013, "l_mix": 3.0338985919952393, "l_a": 3.123341679573059, "l_c": 3.080227851867676, "z_loss": 16.97305965423584, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1779540628194809, "tokens_per_second": 1425347.9510771828, "eta_seconds": 16118.012345433235, "data_wait_seconds": 1.65402889251709e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 28670, "total_steps": 72479, "tokens": 15031336960, "loss": 4.784818768501282, "l_mix": 2.964622586965561, "l_a": 3.053715318441391, "l_c": 3.007883071899414, "z_loss": 17.16608500480652, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.17185334861278534, "tokens_per_second": 1458972.8472380026, "eta_seconds": 15742.947537016867, "data_wait_seconds": 2.759695053100586e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 28680, "total_steps": 72479, "tokens": 15036579840, "loss": 3.1578588485717773, "l_mix": 1.9515477269887924, "l_a": 2.0270471572875977, "l_c": 1.9873855412006378, "z_loss": 19.812335729599, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.3331370949745178, "tokens_per_second": 1459530.5177050661, "eta_seconds": 15733.340162086488, "data_wait_seconds": 3.007054328918457e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08236}}
{"step": 28690, "total_steps": 72479, "tokens": 15041822720, "loss": 4.787220001220703, "l_mix": 2.964403599500656, "l_a": 3.060994952917099, "l_c": 3.009324222803116, "z_loss": 17.205736875534058, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18657268583774567, "tokens_per_second": 1368272.0757861647, "eta_seconds": 16778.861191630363, "data_wait_seconds": 1.9490718841552734e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 28700, "total_steps": 72479, "tokens": 15047065600, "loss": 4.74465399980545, "l_mix": 2.9370613992214203, "l_a": 3.0264260172843933, "l_c": 2.9931042790412903, "z_loss": 17.334905862808228, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.39978402853012085, "tokens_per_second": 1412036.1844424668, "eta_seconds": 16255.110601902006, "data_wait_seconds": 1.963973045349121e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.32906768, "gate_a2b/L1_absmax": 0.66731071, "gate_a2b/L1_rms": 0.3424502, "gate_a2b/L3_absmean": 0.34333104, "gate_a2b/L3_absmax": 0.54712582, "gate_a2b/L3_rms": 0.34767771, "gate_a2b/L5_absmean": 0.51599658, "gate_a2b/L5_absmax": 0.67626625, "gate_a2b/L5_rms": 0.5214147, "gate_c2b/L1_absmean": 0.01458061, "gate_c2b/L1_absmax": 0.42184213, "gate_c2b/L1_rms": 0.05366042, "gate_c2b/L3_absmean": 0.2103374, "gate_c2b/L3_absmax": 0.4265078, "gate_c2b/L3_rms": 0.21417925, "gate_c2b/L5_absmean": 0.52864009, "gate_c2b/L5_absmax": 0.71584523, "gate_c2b/L5_rms": 0.54028744, "gate_b2a/L7_absmean": 0.09233523, "gate_b2a/L7_absmax": 0.29346466, "gate_b2a/L7_rms": 0.09560271, "gate_b2a/L11_absmean": 0.13879469, "gate_b2a/L11_absmax": 0.51262021, "gate_b2a/L11_rms": 0.14206347, "gate_b2a/L14_absmean": 0.33053327, "gate_b2a/L14_absmax": 0.57333672, "gate_b2a/L14_rms": 0.33218253, "gate_b2c/L3_absmean": 0.11164358, "gate_b2c/L3_absmax": 0.64811492, "gate_b2c/L3_rms": 0.12625545, "gate_b2c/L5_absmean": 0.12998241, "gate_b2c/L5_absmax": 0.82923478, "gate_b2c/L5_rms": 0.13924403, "gate_b2c/L6_absmean": 0.18943232, "gate_b2c/L6_absmax": 0.5521009, "gate_b2c/L6_rms": 0.19491169}, "streams": {"cos_a2b/round0": 0.398835, "cos_c2b/round0": 0.306883, "cos_b2a/round0": 0.100146, "cos_b2c/round0": 0.153362, "rms_a/round0": 0.813949, "rms_b/round0": 1.501166, "rms_c/round0": 1.238609, "cos_a2b/round1": 0.416657, "cos_c2b/round1": 0.271138, "cos_b2a/round1": 0.218326, "cos_b2c/round1": 0.112254, "rms_a/round1": 1.43937, "rms_b/round1": 3.048881, "rms_c/round1": 2.031605, "cos_a2b/round2": 0.529904, "cos_c2b/round2": 0.506616, "cos_b2a/round2": 0.052814, "cos_b2c/round2": 0.12713, "rms_a/round2": 3.269239, "rms_b/round2": 6.928251, "rms_c/round2": 3.334663}, "alpha_mean": 0.4385169160542688, "alpha_hist": [0.01276, 0.07918, 0.16875, 0.1871, 0.18015, 0.15712, 0.12593, 0.06825, 0.01927, 0.0015], "alpha_by_type": {"special": {"share": 0.00101, "mean": 0.54908, "std": 0.125}, "whitespace": {"share": 0.02802, "mean": 0.60275, "std": 0.15823}, "digit": {"share": 0.0463, "mean": 0.55023, "std": 0.11933}, "word_start": {"share": 0.55638, "mean": 0.43646, "std": 0.16984}, "word_cont": {"share": 0.25996, "mean": 0.3365, "std": 0.14642}, "punct": {"share": 0.10834, "mean": 0.60262, "std": 0.16636}}, "alpha_p05": 0.14367, "alpha_p50": 0.44131, "alpha_p95": 0.75238}
{"step": 28710, "total_steps": 72479, "tokens": 15052308480, "loss": 4.685906171798706, "l_mix": 2.901531368494034, "l_a": 2.9974557161331177, "l_c": 2.9447479844093323, "z_loss": 17.13691234588623, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1551131308078766, "tokens_per_second": 1380566.7078835866, "eta_seconds": 16621.841842889786, "data_wait_seconds": 2.4884939193725586e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 28720, "total_steps": 72479, "tokens": 15057551360, "loss": 4.703466773033142, "l_mix": 2.9123689234256744, "l_a": 3.005211681127548, "l_c": 2.9594162106513977, "z_loss": 17.09456968307495, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16639557480812073, "tokens_per_second": 1390124.7037731847, "eta_seconds": 16503.78453798294, "data_wait_seconds": 1.5795230865478516e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 28730, "total_steps": 72479, "tokens": 15062794240, "loss": 4.618056952953339, "l_mix": 2.8591792285442352, "l_a": 2.954652190208435, "l_c": 2.9024712443351746, "z_loss": 17.406329870224, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19620917737483978, "tokens_per_second": 1229977.1489285775, "eta_seconds": 18648.375485658646, "data_wait_seconds": 2.9742717742919922e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 28740, "total_steps": 72479, "tokens": 15068037120, "loss": 3.1508657932281494, "l_mix": 1.9473376125097275, "l_a": 2.0209895819425583, "l_c": 1.9843472987413406, "z_loss": 19.26996350288391, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.46898016333580017, "tokens_per_second": 1420470.0262690512, "eta_seconds": 16143.83437025547, "data_wait_seconds": 2.7000904083251953e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08236}}
{"step": 28750, "total_steps": 72479, "tokens": 15073280000, "loss": 3.4339796602725983, "l_mix": 2.1114405393600464, "l_a": 2.237413078546524, "l_c": 2.165819436311722, "z_loss": 15.69391644001007, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19750218093395233, "tokens_per_second": 1410043.3624157847, "eta_seconds": 16259.492837667463, "data_wait_seconds": 1.9997358322143555e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20045, "finemath": 0.08233}}
{"step": 28760, "total_steps": 72479, "tokens": 15078522880, "loss": 3.4491124153137207, "l_mix": 2.121441215276718, "l_a": 2.244660586118698, "l_c": 2.1756626069545746, "z_loss": 15.742253422737122, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22200831770896912, "tokens_per_second": 1362411.3841321333, "eta_seconds": 16824.101250886917, "data_wait_seconds": 2.568960189819336e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20045, "finemath": 0.08234}}
{"step": 28770, "total_steps": 72479, "tokens": 15083765760, "loss": 4.858558475971222, "l_mix": 3.0103812217712402, "l_a": 3.0966751873493195, "l_c": 3.0582162141799927, "z_loss": 17.09690833091736, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15677516162395477, "tokens_per_second": 1417763.6202040333, "eta_seconds": 16163.557778906821, "data_wait_seconds": 1.5974044799804688e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 28780, "total_steps": 72479, "tokens": 15089008640, "loss": 4.939046621322632, "l_mix": 3.0614998638629913, "l_a": 3.1459800601005554, "l_c": 3.106720983982086, "z_loss": 17.36324191093445, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15600678324699402, "tokens_per_second": 1435348.0486731231, "eta_seconds": 15961.885574150087, "data_wait_seconds": 1.7404556274414062e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 28790, "total_steps": 72479, "tokens": 15094251520, "loss": 4.808168411254883, "l_mix": 2.9802724421024323, "l_a": 3.0626566112041473, "l_c": 3.024857372045517, "z_loss": 16.417709350585938, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.27520984411239624, "tokens_per_second": 1004351.025307323, "eta_seconds": 22806.387263846394, "data_wait_seconds": 3.394484519958496e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20768, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 28800, "total_steps": 72479, "tokens": 15099494400, "loss": 4.653780937194824, "l_mix": 2.8804291784763336, "l_a": 2.9813003540039062, "l_c": 2.9240967333316803, "z_loss": 17.32713508605957, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18342724442481995, "tokens_per_second": 1465462.1451953077, "eta_seconds": 15626.726099395752, "data_wait_seconds": 1.874566078186035e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.32879025, "gate_a2b/L1_absmax": 0.65836525, "gate_a2b/L1_rms": 0.3422181, "gate_a2b/L3_absmean": 0.34265682, "gate_a2b/L3_absmax": 0.55018836, "gate_a2b/L3_rms": 0.34701207, "gate_a2b/L5_absmean": 0.51646608, "gate_a2b/L5_absmax": 0.67990077, "gate_a2b/L5_rms": 0.5218789, "gate_c2b/L1_absmean": 0.01605493, "gate_c2b/L1_absmax": 0.42572924, "gate_c2b/L1_rms": 0.05420133, "gate_c2b/L3_absmean": 0.21037748, "gate_c2b/L3_absmax": 0.42990625, "gate_c2b/L3_rms": 0.21425633, "gate_c2b/L5_absmean": 0.52743483, "gate_c2b/L5_absmax": 0.72311395, "gate_c2b/L5_rms": 0.53902757, "gate_b2a/L7_absmean": 0.09234481, "gate_b2a/L7_absmax": 0.29602158, "gate_b2a/L7_rms": 0.09556652, "gate_b2a/L11_absmean": 0.13930252, "gate_b2a/L11_absmax": 0.51384622, "gate_b2a/L11_rms": 0.14268403, "gate_b2a/L14_absmean": 0.33022779, "gate_b2a/L14_absmax": 0.57657427, "gate_b2a/L14_rms": 0.33191702, "gate_b2c/L3_absmean": 0.11164075, "gate_b2c/L3_absmax": 0.64777464, "gate_b2c/L3_rms": 0.12642448, "gate_b2c/L5_absmean": 0.12942937, "gate_b2c/L5_absmax": 0.82022387, "gate_b2c/L5_rms": 0.13871729, "gate_b2c/L6_absmean": 0.18903296, "gate_b2c/L6_absmax": 0.55400109, "gate_b2c/L6_rms": 0.19458827}, "streams": {"cos_a2b/round0": 0.39051, "cos_c2b/round0": 0.299207, "cos_b2a/round0": 0.100073, "cos_b2c/round0": 0.150836, "rms_a/round0": 0.825285, "rms_b/round0": 1.531006, "rms_c/round0": 1.255259, "cos_a2b/round1": 0.414234, "cos_c2b/round1": 0.270547, "cos_b2a/round1": 0.220292, "cos_b2c/round1": 0.122661, "rms_a/round1": 1.458797, "rms_b/round1": 3.128181, "rms_c/round1": 2.067858, "cos_a2b/round2": 0.526726, "cos_c2b/round2": 0.507065, "cos_b2a/round2": 0.057272, "cos_b2c/round2": 0.124745, "rms_a/round2": 3.318249, "rms_b/round2": 7.014232, "rms_c/round2": 3.427026}, "alpha_mean": 0.4375679239649486, "alpha_hist": [0.01814, 0.09181, 0.16132, 0.17593, 0.17075, 0.15578, 0.12965, 0.08067, 0.01553, 0.00041], "alpha_by_type": {"special": {"share": 0.00124, "mean": 0.59523, "std": 0.09808}, "whitespace": {"share": 0.02753, "mean": 0.54487, "std": 0.13913}, "digit": {"share": 0.05128, "mean": 0.5992, "std": 0.12515}, "word_start": {"share": 0.54643, "mean": 0.44927, "std": 0.1766}, "word_cont": {"share": 0.27454, "mean": 0.32066, "std": 0.14521}, "punct": {"share": 0.09898, "mean": 0.58167, "std": 0.17674}}, "alpha_p05": 0.15078, "alpha_p50": 0.44179, "alpha_p95": 0.74798}
{"step": 28810, "total_steps": 72479, "tokens": 15104737280, "loss": 4.615989029407501, "l_mix": 2.8558792173862457, "l_a": 2.95870104432106, "l_c": 2.902551770210266, "z_loss": 17.33875584602356, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1639254242181778, "tokens_per_second": 1445539.43756818, "eta_seconds": 15838.469762206078, "data_wait_seconds": 1.621246337890625e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 28820, "total_steps": 72479, "tokens": 15109980160, "loss": 4.591300070285797, "l_mix": 2.8416687548160553, "l_a": 2.9406627416610718, "l_c": 2.8856192231178284, "z_loss": 17.466919660568237, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16296900808811188, "tokens_per_second": 1304957.9424348134, "eta_seconds": 17540.710736846922, "data_wait_seconds": 2.288818359375e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 28830, "total_steps": 72479, "tokens": 15115223040, "loss": 4.623741030693054, "l_mix": 2.861856997013092, "l_a": 2.957414835691452, "l_c": 2.90979728102684, "z_loss": 17.202650785446167, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.15376342833042145, "tokens_per_second": 1227611.062066611, "eta_seconds": 18641.610212826727, "data_wait_seconds": 2.3186206817626953e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 28840, "total_steps": 72479, "tokens": 15120465920, "loss": 4.720532119274139, "l_mix": 2.923043817281723, "l_a": 3.016450136899948, "l_c": 2.9694053530693054, "z_loss": 17.316399812698364, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16387991607189178, "tokens_per_second": 1184389.0319475194, "eta_seconds": 19317.473747944834, "data_wait_seconds": 1.9103288650512695e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 28850, "total_steps": 72479, "tokens": 15125708800, "loss": 3.205581396818161, "l_mix": 1.980451911687851, "l_a": 2.0595824122428894, "l_c": 2.0175536274909973, "z_loss": 19.886067628860474, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.26764005422592163, "tokens_per_second": 1403719.6425231781, "eta_seconds": 16295.391514849664, "data_wait_seconds": 3.108382225036621e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08236}}
{"step": 28860, "total_steps": 72479, "tokens": 15130951680, "loss": 3.4431037604808807, "l_mix": 2.1182348132133484, "l_a": 2.240388125181198, "l_c": 2.1704802811145782, "z_loss": 16.08438241481781, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.22094812989234924, "tokens_per_second": 1445723.6161695013, "eta_seconds": 15818.319640231133, "data_wait_seconds": 3.4183263778686523e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20045, "finemath": 0.08233}}
{"step": 28870, "total_steps": 72479, "tokens": 15136194560, "loss": 3.4087939858436584, "l_mix": 2.096101090312004, "l_a": 2.218989461660385, "l_c": 2.151204079389572, "z_loss": 16.348228693008423, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21837137639522552, "tokens_per_second": 1453199.351104119, "eta_seconds": 15733.337187790872, "data_wait_seconds": 2.193450927734375e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20045, "finemath": 0.08233}}
{"step": 28880, "total_steps": 72479, "tokens": 15141437440, "loss": 3.406570315361023, "l_mix": 2.095436781644821, "l_a": 2.2133834958076477, "l_c": 2.1516069769859314, "z_loss": 16.364031314849854, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.21946045756340027, "tokens_per_second": 1303785.8633233025, "eta_seconds": 17532.35186469555, "data_wait_seconds": 2.154707908630371e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20045, "finemath": 0.08234}}
{"step": 28890, "total_steps": 72479, "tokens": 15146680320, "loss": 4.585348546504974, "l_mix": 2.8392472863197327, "l_a": 2.933586210012436, "l_c": 2.8810615837574005, "z_loss": 17.06907343864441, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20033153891563416, "tokens_per_second": 1424530.2391093094, "eta_seconds": 16042.614613986016, "data_wait_seconds": 2.6404857635498047e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 28900, "total_steps": 72479, "tokens": 15151923200, "loss": 4.486296474933624, "l_mix": 2.775810122489929, "l_a": 2.874305844306946, "l_c": 2.8214848339557648, "z_loss": 17.489609718322754, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.19786451756954193, "tokens_per_second": 1255429.4354223977, "eta_seconds": 18199.30782833099, "data_wait_seconds": 4.190206527709961e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.32917902, "gate_a2b/L1_absmax": 0.65359116, "gate_a2b/L1_rms": 0.34227476, "gate_a2b/L3_absmean": 0.34196025, "gate_a2b/L3_absmax": 0.54453933, "gate_a2b/L3_rms": 0.3462593, "gate_a2b/L5_absmean": 0.51557219, "gate_a2b/L5_absmax": 0.67429584, "gate_a2b/L5_rms": 0.52099925, "gate_c2b/L1_absmean": 0.01459697, "gate_c2b/L1_absmax": 0.42597759, "gate_c2b/L1_rms": 0.05394828, "gate_c2b/L3_absmean": 0.21054621, "gate_c2b/L3_absmax": 0.42610252, "gate_c2b/L3_rms": 0.21436898, "gate_c2b/L5_absmean": 0.52835172, "gate_c2b/L5_absmax": 0.71775323, "gate_c2b/L5_rms": 0.54006588, "gate_b2a/L7_absmean": 0.09265908, "gate_b2a/L7_absmax": 0.29505265, "gate_b2a/L7_rms": 0.09589577, "gate_b2a/L11_absmean": 0.13876992, "gate_b2a/L11_absmax": 0.51524562, "gate_b2a/L11_rms": 0.14206815, "gate_b2a/L14_absmean": 0.33134365, "gate_b2a/L14_absmax": 0.57938743, "gate_b2a/L14_rms": 0.33302552, "gate_b2c/L3_absmean": 0.1118273, "gate_b2c/L3_absmax": 0.64680749, "gate_b2c/L3_rms": 0.12646112, "gate_b2c/L5_absmean": 0.12938976, "gate_b2c/L5_absmax": 0.81967539, "gate_b2c/L5_rms": 0.13851261, "gate_b2c/L6_absmean": 0.18933754, "gate_b2c/L6_absmax": 0.55585802, "gate_b2c/L6_rms": 0.19482526}, "streams": {"cos_a2b/round0": 0.395992, "cos_c2b/round0": 0.31565, "cos_b2a/round0": 0.107134, "cos_b2c/round0": 0.160939, "rms_a/round0": 0.814405, "rms_b/round0": 1.501651, "rms_c/round0": 1.232723, "cos_a2b/round1": 0.418736, "cos_c2b/round1": 0.27246, "cos_b2a/round1": 0.231096, "cos_b2c/round1": 0.11393, "rms_a/round1": 1.435856, "rms_b/round1": 3.048561, "rms_c/round1": 2.029247, "cos_a2b/round2": 0.54058, "cos_c2b/round2": 0.503832, "cos_b2a/round2": 0.059166, "cos_b2c/round2": 0.136722, "rms_a/round2": 3.322411, "rms_b/round2": 6.959403, "rms_c/round2": 3.295359}, "alpha_mean": 0.42933630361714137, "alpha_hist": [0.02002, 0.08983, 0.16901, 0.18047, 0.1785, 0.1552, 0.12675, 0.0703, 0.00977, 0.00017], "alpha_by_type": {"special": {"share": 0.00075, "mean": 0.57624, "std": 0.0863}, "whitespace": {"share": 0.02827, "mean": 0.54864, "std": 0.09932}, "digit": {"share": 0.05638, "mean": 0.46842, "std": 0.1156}, "word_start": {"share": 0.55194, "mean": 0.43986, "std": 0.17421}, "word_cont": {"share": 0.24631, "mean": 0.30192, "std": 0.13861}, "punct": {"share": 0.11635, "mean": 0.60027, "std": 0.15012}}, "alpha_p05": 0.15103, "alpha_p50": 0.42179, "alpha_p95": 0.74017}
{"step": 28910, "total_steps": 72479, "tokens": 15157166080, "loss": 4.487899720668793, "l_mix": 2.7769372165203094, "l_a": 2.874506652355194, "l_c": 2.82287135720253, "z_loss": 17.490903854370117, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.23556667566299438, "tokens_per_second": 1418271.0172784731, "eta_seconds": 16106.021764326095, "data_wait_seconds": 2.0742416381835938e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20042, "finemath": 0.08236}}
{"step": 28920, "total_steps": 72479, "tokens": 15162408960, "loss": 4.449168503284454, "l_mix": 2.7543153762817383, "l_a": 2.8490112125873566, "l_c": 2.79488742351532, "z_loss": 16.83518946170807, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.16476990282535553, "tokens_per_second": 1479940.660022326, "eta_seconds": 15431.335599398613, "data_wait_seconds": 1.990795135498047e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20768, "cosmopedia_v2": 0.20041, "finemath": 0.08233}}
{"step": 28930, "total_steps": 72479, "tokens": 15167651840, "loss": 4.409036934375763, "l_mix": 2.7273362278938293, "l_a": 2.8296500742435455, "l_c": 2.770256847143173, "z_loss": 17.285192489624023, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1721220165491104, "tokens_per_second": 1427621.618930465, "eta_seconds": 15993.186015987396, "data_wait_seconds": 1.6063451766967773e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08234}}
{"step": 28940, "total_steps": 72479, "tokens": 15172894720, "loss": 4.4205708503723145, "l_mix": 2.7335220277309418, "l_a": 2.8428837060928345, "l_c": 2.7748502790927887, "z_loss": 17.285775423049927, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.18600977957248688, "tokens_per_second": 1371983.5899969914, "eta_seconds": 16637.936049985885, "data_wait_seconds": 2.0802021026611328e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08234}}
{"step": 28950, "total_steps": 72479, "tokens": 15178137600, "loss": 4.382480323314667, "l_mix": 2.709440916776657, "l_a": 2.8174737989902496, "l_c": 2.753694087266922, "z_loss": 16.890013694763184, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.23007667064666748, "tokens_per_second": 1467901.1695437292, "eta_seconds": 15547.18589065075, "data_wait_seconds": 1.5914440155029297e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 28960, "total_steps": 72479, "tokens": 15183380480, "loss": 3.462856888771057, "l_mix": 2.1387979686260223, "l_a": 2.2321531921625137, "l_c": 2.175161123275757, "z_loss": 18.64660120010376, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.4076038599014282, "tokens_per_second": 1403708.9796413921, "eta_seconds": 16254.430086946488, "data_wait_seconds": 2.1576881408691406e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 28970, "total_steps": 72479, "tokens": 15188623360, "loss": 3.4015097320079803, "l_mix": 2.0913769602775574, "l_a": 2.213023215532303, "l_c": 2.14865705370903, "z_loss": 16.286717891693115, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.1974625438451767, "tokens_per_second": 1272730.5039033461, "eta_seconds": 17923.07681951523, "data_wait_seconds": 2.3126602172851562e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20045, "finemath": 0.08233}}
{"step": 28980, "total_steps": 72479, "tokens": 15193866240, "loss": 3.3901272118091583, "l_mix": 2.0846235901117325, "l_a": 2.2060039043426514, "l_c": 2.1404407918453217, "z_loss": 15.701443314552307, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20003432035446167, "tokens_per_second": 1451582.3238828117, "eta_seconds": 15711.133524274825, "data_wait_seconds": 1.5407800674438477e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20045, "finemath": 0.08233}}
{"step": 28990, "total_steps": 72479, "tokens": 15199109120, "loss": 3.4068877398967743, "l_mix": 2.094569504261017, "l_a": 2.2195310294628143, "l_c": 2.1497066020965576, "z_loss": 15.468741178512573, "lr": 0.0023, "wd": 0.1, "grad_norm": 0.20102892816066742, "tokens_per_second": 1292174.0710779892, "eta_seconds": 17645.270356631278, "data_wait_seconds": 2.631545066833496e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20045, "finemath": 0.08234}}
{"step": 29000, "total_steps": 72479, "tokens": 15204352000, "loss": 4.313377678394318, "l_mix": 2.6677712202072144, "l_a": 2.7702015936374664, "l_c": 2.7151532769203186, "z_loss": 16.89371371269226, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1910836100578308, "tokens_per_second": 1378188.273820677, "eta_seconds": 16540.205997258425, "data_wait_seconds": 1.7762184143066406e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20766, "cosmopedia_v2": 0.20041, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.3277128, "gate_a2b/L1_absmax": 0.64653605, "gate_a2b/L1_rms": 0.34085441, "gate_a2b/L3_absmean": 0.34269777, "gate_a2b/L3_absmax": 0.5397892, "gate_a2b/L3_rms": 0.34705994, "gate_a2b/L5_absmean": 0.51523095, "gate_a2b/L5_absmax": 0.66921324, "gate_a2b/L5_rms": 0.52064925, "gate_c2b/L1_absmean": 0.0166457, "gate_c2b/L1_absmax": 0.43085143, "gate_c2b/L1_rms": 0.0546279, "gate_c2b/L3_absmean": 0.21054314, "gate_c2b/L3_absmax": 0.41947606, "gate_c2b/L3_rms": 0.21444456, "gate_c2b/L5_absmean": 0.52902853, "gate_c2b/L5_absmax": 0.726928, "gate_c2b/L5_rms": 0.54082453, "gate_b2a/L7_absmean": 0.09213699, "gate_b2a/L7_absmax": 0.29541171, "gate_b2a/L7_rms": 0.09558617, "gate_b2a/L11_absmean": 0.13906109, "gate_b2a/L11_absmax": 0.5275349, "gate_b2a/L11_rms": 0.14256844, "gate_b2a/L14_absmean": 0.33192885, "gate_b2a/L14_absmax": 0.58225048, "gate_b2a/L14_rms": 0.33370104, "gate_b2c/L3_absmean": 0.11143947, "gate_b2c/L3_absmax": 0.65727937, "gate_b2c/L3_rms": 0.12613875, "gate_b2c/L5_absmean": 0.12939185, "gate_b2c/L5_absmax": 0.81156725, "gate_b2c/L5_rms": 0.13853255, "gate_b2c/L6_absmean": 0.18761492, "gate_b2c/L6_absmax": 0.56463856, "gate_b2c/L6_rms": 0.19331172}, "streams": {"cos_a2b/round0": 0.389972, "cos_c2b/round0": 0.300405, "cos_b2a/round0": 0.107535, "cos_b2c/round0": 0.145468, "rms_a/round0": 0.819045, "rms_b/round0": 1.526782, "rms_c/round0": 1.254842, "cos_a2b/round1": 0.425097, "cos_c2b/round1": 0.276957, "cos_b2a/round1": 0.233616, "cos_b2c/round1": 0.128904, "rms_a/round1": 1.452326, "rms_b/round1": 3.145323, "rms_c/round1": 2.070322, "cos_a2b/round2": 0.524936, "cos_c2b/round2": 0.517107, "cos_b2a/round2": 0.050566, "cos_b2c/round2": 0.128519, "rms_a/round2": 3.344394, "rms_b/round2": 7.108809, "rms_c/round2": 3.43991}, "alpha_mean": 0.41150191514836365, "alpha_hist": [0.03488, 0.11526, 0.18182, 0.1721, 0.15575, 0.14059, 0.11716, 0.06911, 0.01321, 0.00011], "alpha_by_type": {"special": {"share": 0.0009, "mean": 0.54564, "std": 0.11529}, "whitespace": {"share": 0.02248, "mean": 0.54498, "std": 0.13063}, "digit": {"share": 0.03041, "mean": 0.53662, "std": 0.13432}, "word_start": {"share": 0.59134, "mean": 0.42545, "std": 0.17872}, "word_cont": {"share": 0.25, "mean": 0.28239, "std": 0.15354}, "punct": {"share": 0.10487, "mean": 0.57458, "std": 0.17775}}, "alpha_p05": 0.13501, "alpha_p50": 0.38306, "alpha_p95": 0.73754}
{"step": 29010, "total_steps": 72479, "tokens": 15209594880, "loss": 4.226712942123413, "l_mix": 2.61350154876709, "l_a": 2.7186836898326874, "l_c": 2.658687323331833, "z_loss": 17.184091806411743, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2095809429883957, "tokens_per_second": 1405440.817092536, "eta_seconds": 16215.748678159714, "data_wait_seconds": 1.7344951629638672e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 29020, "total_steps": 72479, "tokens": 15214837760, "loss": 4.244471430778503, "l_mix": 2.625623196363449, "l_a": 2.729907661676407, "l_c": 2.6662526428699493, "z_loss": 17.258219003677368, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.29290100932121277, "tokens_per_second": 1433398.7072379612, "eta_seconds": 15895.80908434391, "data_wait_seconds": 1.7255544662475586e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20041, "finemath": 0.08236}}
{"step": 29030, "total_steps": 72479, "tokens": 15220080640, "loss": 3.4065505266189575, "l_mix": 2.0952245593070984, "l_a": 2.220796585083008, "l_c": 2.1502898931503296, "z_loss": 16.16801953315735, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20198816061019897, "tokens_per_second": 1437566.8229769256, "eta_seconds": 15846.073342752456, "data_wait_seconds": 1.862645149230957e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20045, "finemath": 0.08233}}
{"step": 29040, "total_steps": 72479, "tokens": 15225323520, "loss": 3.4232523441314697, "l_mix": 2.106493651866913, "l_a": 2.2295918464660645, "l_c": 2.15960356593132, "z_loss": 16.86534571647644, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2523875832557678, "tokens_per_second": 1319654.9487179348, "eta_seconds": 17257.95553915501, "data_wait_seconds": 3.2275915145874023e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20764, "cosmopedia_v2": 0.20045, "finemath": 0.08233}}
{"step": 29050, "total_steps": 72479, "tokens": 15230566400, "loss": 4.933559715747833, "l_mix": 3.058918535709381, "l_a": 3.145440101623535, "l_c": 3.103363484144211, "z_loss": 17.48815369606018, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1918918788433075, "tokens_per_second": 1350342.3843502929, "eta_seconds": 16861.87430379391, "data_wait_seconds": 3.2454729080200195e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20768, "cosmopedia_v2": 0.20041, "finemath": 0.08234}}
{"step": 29060, "total_steps": 72479, "tokens": 15235809280, "loss": 4.916486024856567, "l_mix": 3.047573447227478, "l_a": 3.138958066701889, "l_c": 3.0907502472400665, "z_loss": 18.256598234176636, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2123875916004181, "tokens_per_second": 1335675.157246602, "eta_seconds": 17043.111529397964, "data_wait_seconds": 4.3511390686035156e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 29070, "total_steps": 72479, "tokens": 15241052160, "loss": 4.953383147716522, "l_mix": 3.071053624153137, "l_a": 3.1591144502162933, "l_c": 3.115317076444626, "z_loss": 17.701533794403076, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17186184227466583, "tokens_per_second": 1334885.5977820372, "eta_seconds": 17049.26461849213, "data_wait_seconds": 2.8371810913085938e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 29080, "total_steps": 72479, "tokens": 15246295040, "loss": 3.4412419199943542, "l_mix": 2.11723792552948, "l_a": 2.241068124771118, "l_c": 2.172278255224228, "z_loss": 17.19343066215515, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.23445557057857513, "tokens_per_second": 1450943.3939140332, "eta_seconds": 15681.91771466732, "data_wait_seconds": 3.847479820251465e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20045, "finemath": 0.08232}}
{"step": 29090, "total_steps": 72479, "tokens": 15251537920, "loss": 3.4017005264759064, "l_mix": 2.0920176208019257, "l_a": 2.215757429599762, "l_c": 2.149852156639099, "z_loss": 17.266985177993774, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20433664321899414, "tokens_per_second": 1180382.7166916053, "eta_seconds": 19271.996878910064, "data_wait_seconds": 4.3958425521850586e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20045, "finemath": 0.08233}}
{"step": 29100, "total_steps": 72479, "tokens": 15256780800, "loss": 3.416692405939102, "l_mix": 2.10204616189003, "l_a": 2.22510102391243, "l_c": 2.1570528745651245, "z_loss": 17.50376319885254, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.21437489986419678, "tokens_per_second": 1410565.6055220205, "eta_seconds": 16123.3827501297, "data_wait_seconds": 2.47955322265625e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20766, "cosmopedia_v2": 0.20045, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.3289952, "gate_a2b/L1_absmax": 0.6422649, "gate_a2b/L1_rms": 0.34218368, "gate_a2b/L3_absmean": 0.34522438, "gate_a2b/L3_absmax": 0.5476262, "gate_a2b/L3_rms": 0.34977019, "gate_a2b/L5_absmean": 0.51838732, "gate_a2b/L5_absmax": 0.67169648, "gate_a2b/L5_rms": 0.52380216, "gate_c2b/L1_absmean": 0.01461554, "gate_c2b/L1_absmax": 0.45099267, "gate_c2b/L1_rms": 0.05566797, "gate_c2b/L3_absmean": 0.21389964, "gate_c2b/L3_absmax": 0.43411639, "gate_c2b/L3_rms": 0.21774052, "gate_c2b/L5_absmean": 0.53336042, "gate_c2b/L5_absmax": 0.73002106, "gate_c2b/L5_rms": 0.5452565, "gate_b2a/L7_absmean": 0.09256483, "gate_b2a/L7_absmax": 0.30179879, "gate_b2a/L7_rms": 0.09601829, "gate_b2a/L11_absmean": 0.13872534, "gate_b2a/L11_absmax": 0.53359133, "gate_b2a/L11_rms": 0.14222722, "gate_b2a/L14_absmean": 0.33219513, "gate_b2a/L14_absmax": 0.58976531, "gate_b2a/L14_rms": 0.33406147, "gate_b2c/L3_absmean": 0.11018336, "gate_b2c/L3_absmax": 0.67220086, "gate_b2c/L3_rms": 0.12451744, "gate_b2c/L5_absmean": 0.12748814, "gate_b2c/L5_absmax": 0.81656432, "gate_b2c/L5_rms": 0.13677853, "gate_b2c/L6_absmean": 0.18562686, "gate_b2c/L6_absmax": 0.55866647, "gate_b2c/L6_rms": 0.1914241}, "streams": {"cos_a2b/round0": 0.39, "cos_c2b/round0": 0.266529, "cos_b2a/round0": 0.094366, "cos_b2c/round0": 0.117269, "rms_a/round0": 0.921131, "rms_b/round0": 1.637568, "rms_c/round0": 1.320776, "cos_a2b/round1": 0.427497, "cos_c2b/round1": 0.279565, "cos_b2a/round1": 0.217899, "cos_b2c/round1": 0.12784, "rms_a/round1": 1.636359, "rms_b/round1": 3.466223, "rms_c/round1": 2.257865, "cos_a2b/round2": 0.52371, "cos_c2b/round2": 0.477377, "cos_b2a/round2": 0.027672, "cos_b2c/round2": 0.112725, "rms_a/round2": 3.654906, "rms_b/round2": 7.69254, "rms_c/round2": 3.710232}, "alpha_mean": 0.4388137096262028, "alpha_hist": [0.03368, 0.12816, 0.17224, 0.15417, 0.12187, 0.10966, 0.11351, 0.10352, 0.05727, 0.00592], "alpha_by_type": {"special": {"share": 0.00114, "mean": 0.65136, "std": 0.11371}, "whitespace": {"share": 0.03358, "mean": 0.66172, "std": 0.17484}, "digit": {"share": 0.00978, "mean": 0.40191, "std": 0.16485}, "word_start": {"share": 0.58289, "mean": 0.45397, "std": 0.20928}, "word_cont": {"share": 0.25812, "mean": 0.28293, "std": 0.15375}, "punct": {"share": 0.11449, "mean": 0.64877, "std": 0.18039}}, "alpha_p05": 0.11596, "alpha_p50": 0.41466, "alpha_p95": 0.81931}
{"step": 29110, "total_steps": 72479, "tokens": 15262023680, "loss": 4.273612022399902, "l_mix": 2.6432042717933655, "l_a": 2.74455726146698, "l_c": 2.6901349425315857, "z_loss": 18.54018807411194, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19142812490463257, "tokens_per_second": 1457733.3431212783, "eta_seconds": 15598.083407568933, "data_wait_seconds": 1.6242265701293945e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20041, "finemath": 0.08234}}
{"step": 29120, "total_steps": 72479, "tokens": 15267266560, "loss": 4.331684648990631, "l_mix": 2.6795877516269684, "l_a": 2.7831541895866394, "l_c": 2.7238349318504333, "z_loss": 18.85054636001587, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17122815549373627, "tokens_per_second": 1362497.9928627918, "eta_seconds": 16684.504132175447, "data_wait_seconds": 2.5600194931030273e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 29130, "total_steps": 72479, "tokens": 15272509440, "loss": 4.212668716907501, "l_mix": 2.6053633093833923, "l_a": 2.710323750972748, "l_c": 2.6473605632781982, "z_loss": 18.803277492523193, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15738533437252045, "tokens_per_second": 1356080.4986228745, "eta_seconds": 16759.59541862011, "data_wait_seconds": 2.282857894897461e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 29140, "total_steps": 72479, "tokens": 15277752320, "loss": 3.4181801080703735, "l_mix": 2.102973610162735, "l_a": 2.2234510481357574, "l_c": 2.1605705320835114, "z_loss": 18.349173545837402, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1816856563091278, "tokens_per_second": 1437550.3770127208, "eta_seconds": 15806.13660247326, "data_wait_seconds": 1.6033649444580078e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20045, "finemath": 0.08233}}
{"step": 29150, "total_steps": 72479, "tokens": 15282995200, "loss": 3.4408372044563293, "l_mix": 2.1179755330085754, "l_a": 2.23620942234993, "l_c": 2.1733293533325195, "z_loss": 18.209887742996216, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.21447306871414185, "tokens_per_second": 1386126.1245149162, "eta_seconds": 16388.750165104866, "data_wait_seconds": 3.221631050109863e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20045, "finemath": 0.08233}}
{"step": 29160, "total_steps": 72479, "tokens": 15288238080, "loss": 3.450708031654358, "l_mix": 2.123269736766815, "l_a": 2.243768334388733, "l_c": 2.181025803089142, "z_loss": 18.036542654037476, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.22662122547626495, "tokens_per_second": 1455105.0291854986, "eta_seconds": 15608.242303109168, "data_wait_seconds": 2.5987625122070312e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20045, "finemath": 0.08234}}
{"step": 29170, "total_steps": 72479, "tokens": 15293480960, "loss": 3.4229778051376343, "l_mix": 2.10557958483696, "l_a": 2.228067547082901, "l_c": 2.16325980424881, "z_loss": 17.7159640789032, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.22707366943359375, "tokens_per_second": 1267315.2567824998, "eta_seconds": 17916.922305226326, "data_wait_seconds": 2.092123031616211e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20764, "cosmopedia_v2": 0.20045, "finemath": 0.08234}}
{"step": 29180, "total_steps": 72479, "tokens": 15298723840, "loss": 3.4304173290729523, "l_mix": 2.1103466153144836, "l_a": 2.232223689556122, "l_c": 2.1680118441581726, "z_loss": 18.472851514816284, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.21182343363761902, "tokens_per_second": 1329276.6276118967, "eta_seconds": 17077.819349598885, "data_wait_seconds": 2.014636993408203e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20764, "cosmopedia_v2": 0.20045, "finemath": 0.08235}}
{"step": 29190, "total_steps": 72479, "tokens": 15303966720, "loss": 4.849050998687744, "l_mix": 3.006197065114975, "l_a": 3.089666962623596, "l_c": 3.0531791150569916, "z_loss": 19.27738070487976, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1823558658361435, "tokens_per_second": 1460441.7763433116, "eta_seconds": 15540.436873030661, "data_wait_seconds": 1.5914440155029297e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08236}}
{"step": 29200, "total_steps": 72479, "tokens": 15309209600, "loss": 3.4185856580734253, "l_mix": 2.1028424203395844, "l_a": 2.224118411540985, "l_c": 2.161692351102829, "z_loss": 17.79759407043457, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.21953986585140228, "tokens_per_second": 1423586.642685409, "eta_seconds": 15939.079274582862, "data_wait_seconds": 1.7315149307250977e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20045, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.3275682, "gate_a2b/L1_absmax": 0.64947212, "gate_a2b/L1_rms": 0.34080538, "gate_a2b/L3_absmean": 0.34380123, "gate_a2b/L3_absmax": 0.5571698, "gate_a2b/L3_rms": 0.34838495, "gate_a2b/L5_absmean": 0.51838934, "gate_a2b/L5_absmax": 0.67575508, "gate_a2b/L5_rms": 0.52384615, "gate_c2b/L1_absmean": 0.01515513, "gate_c2b/L1_absmax": 0.45624045, "gate_c2b/L1_rms": 0.05592078, "gate_c2b/L3_absmean": 0.2141201, "gate_c2b/L3_absmax": 0.43426785, "gate_c2b/L3_rms": 0.21801591, "gate_c2b/L5_absmean": 0.53761262, "gate_c2b/L5_absmax": 0.73284966, "gate_c2b/L5_rms": 0.54964447, "gate_b2a/L7_absmean": 0.09090834, "gate_b2a/L7_absmax": 0.3078281, "gate_b2a/L7_rms": 0.09445728, "gate_b2a/L11_absmean": 0.13702786, "gate_b2a/L11_absmax": 0.53723508, "gate_b2a/L11_rms": 0.14069463, "gate_b2a/L14_absmean": 0.33053917, "gate_b2a/L14_absmax": 0.59750283, "gate_b2a/L14_rms": 0.33250099, "gate_b2c/L3_absmean": 0.1067581, "gate_b2c/L3_absmax": 0.67473865, "gate_b2c/L3_rms": 0.12131799, "gate_b2c/L5_absmean": 0.12405951, "gate_b2c/L5_absmax": 0.82505155, "gate_b2c/L5_rms": 0.13363385, "gate_b2c/L6_absmean": 0.1811344, "gate_b2c/L6_absmax": 0.55586803, "gate_b2c/L6_rms": 0.18719159}, "streams": {"cos_a2b/round0": 0.394439, "cos_c2b/round0": 0.266347, "cos_b2a/round0": 0.096207, "cos_b2c/round0": 0.118748, "rms_a/round0": 0.975135, "rms_b/round0": 1.739892, "rms_c/round0": 1.362684, "cos_a2b/round1": 0.422786, "cos_c2b/round1": 0.286523, "cos_b2a/round1": 0.223491, "cos_b2c/round1": 0.126306, "rms_a/round1": 1.730749, "rms_b/round1": 3.674207, "rms_c/round1": 2.391662, "cos_a2b/round2": 0.519894, "cos_c2b/round2": 0.481601, "cos_b2a/round2": 0.028975, "cos_b2c/round2": 0.113873, "rms_a/round2": 3.879951, "rms_b/round2": 8.195271, "rms_c/round2": 3.896156}, "alpha_mean": 0.4174020354759307, "alpha_hist": [0.04869, 0.15488, 0.17659, 0.14465, 0.11592, 0.10516, 0.10425, 0.09026, 0.05251, 0.0071], "alpha_by_type": {"special": {"share": 0.0011, "mean": 0.54158, "std": 0.09012}, "whitespace": {"share": 0.03639, "mean": 0.60307, "std": 0.19498}, "digit": {"share": 0.01196, "mean": 0.37147, "std": 0.16666}, "word_start": {"share": 0.57462, "mean": 0.4327, "std": 0.21601}, "word_cont": {"share": 0.26118, "mean": 0.26635, "std": 0.1549}, "punct": {"share": 0.11475, "mean": 0.62935, "std": 0.19555}}, "alpha_p05": 0.10375, "alpha_p50": 0.37342, "alpha_p95": 0.81287}
{"step": 29210, "total_steps": 72479, "tokens": 15314452480, "loss": 3.3784747421741486, "l_mix": 2.0783799588680267, "l_a": 2.1983096301555634, "l_c": 2.135339617729187, "z_loss": 18.23433828353882, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2062462568283081, "tokens_per_second": 1469070.6810276438, "eta_seconds": 15442.019070267677, "data_wait_seconds": 2.9861927032470703e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20045, "finemath": 0.08233}}
{"step": 29220, "total_steps": 72479, "tokens": 15319695360, "loss": 4.242338448762894, "l_mix": 2.623681366443634, "l_a": 2.72732874751091, "l_c": 2.668194681406021, "z_loss": 20.014148235321045, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18933694064617157, "tokens_per_second": 1394749.2119984964, "eta_seconds": 16261.113035154343, "data_wait_seconds": 2.065300941467285e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08234}}
{"step": 29230, "total_steps": 72479, "tokens": 15324938240, "loss": 4.28684675693512, "l_mix": 2.652675062417984, "l_a": 2.7518226504325867, "l_c": 2.6954162418842316, "z_loss": 19.625146865844727, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1429501324892044, "tokens_per_second": 1388487.0169641138, "eta_seconds": 16330.676077604294, "data_wait_seconds": 1.990795135498047e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 29240, "total_steps": 72479, "tokens": 15330181120, "loss": 4.209862172603607, "l_mix": 2.604771673679352, "l_a": 2.7027901709079742, "l_c": 2.6475111544132233, "z_loss": 19.88069725036621, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18129825592041016, "tokens_per_second": 1317053.6614644895, "eta_seconds": 17212.426110863686, "data_wait_seconds": 1.627206802368164e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 29250, "total_steps": 72479, "tokens": 15335424000, "loss": 3.357670694589615, "l_mix": 2.0654730796813965, "l_a": 2.1829166412353516, "l_c": 2.1244086623191833, "z_loss": 18.782359838485718, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1544468253850937, "tokens_per_second": 1400320.1379138366, "eta_seconds": 16185.188899564744, "data_wait_seconds": 4.7087669372558594e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08232}}
{"step": 29260, "total_steps": 72479, "tokens": 15340666880, "loss": 3.3968514800071716, "l_mix": 2.090275138616562, "l_a": 2.207225650548935, "l_c": 2.1480287313461304, "z_loss": 19.409602880477905, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2183043509721756, "tokens_per_second": 1367493.5212971338, "eta_seconds": 16569.87965142727, "data_wait_seconds": 1.6689300537109375e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 29270, "total_steps": 72479, "tokens": 15345909760, "loss": 3.387827843427658, "l_mix": 2.084436923265457, "l_a": 2.205580323934555, "l_c": 2.139055758714676, "z_loss": 18.739194869995117, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19970515370368958, "tokens_per_second": 1380200.004212735, "eta_seconds": 16413.534359407426, "data_wait_seconds": 1.901388168334961e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 29280, "total_steps": 72479, "tokens": 15351152640, "loss": 3.4317972362041473, "l_mix": 2.11095729470253, "l_a": 2.233305960893631, "l_c": 2.1694937646389008, "z_loss": 18.838798999786377, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.23074577748775482, "tokens_per_second": 1285137.9115777195, "eta_seconds": 17623.569507956505, "data_wait_seconds": 2.5272369384765625e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 29290, "total_steps": 72479, "tokens": 15356395520, "loss": 3.400262713432312, "l_mix": 2.091529369354248, "l_a": 2.2138851284980774, "l_c": 2.148559033870697, "z_loss": 19.462575674057007, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1981222927570343, "tokens_per_second": 1449001.9029575668, "eta_seconds": 15626.945958995819, "data_wait_seconds": 2.5719404220581055e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 29300, "total_steps": 72479, "tokens": 15361638400, "loss": 3.395262748003006, "l_mix": 2.088508814573288, "l_a": 2.211616337299347, "l_c": 2.1442299485206604, "z_loss": 19.459214687347412, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1802447885274887, "tokens_per_second": 1401948.155340078, "eta_seconds": 16147.69523806572, "data_wait_seconds": 1.6063451766967773e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.32626075, "gate_a2b/L1_absmax": 0.65879524, "gate_a2b/L1_rms": 0.33983162, "gate_a2b/L3_absmean": 0.3430022, "gate_a2b/L3_absmax": 0.55887038, "gate_a2b/L3_rms": 0.34756994, "gate_a2b/L5_absmean": 0.51790965, "gate_a2b/L5_absmax": 0.67371589, "gate_a2b/L5_rms": 0.52338833, "gate_c2b/L1_absmean": 0.01532314, "gate_c2b/L1_absmax": 0.45929179, "gate_c2b/L1_rms": 0.05654193, "gate_c2b/L3_absmean": 0.21463044, "gate_c2b/L3_absmax": 0.43893844, "gate_c2b/L3_rms": 0.2185356, "gate_c2b/L5_absmean": 0.54070157, "gate_c2b/L5_absmax": 0.73826766, "gate_c2b/L5_rms": 0.55281478, "gate_b2a/L7_absmean": 0.08960012, "gate_b2a/L7_absmax": 0.3095302, "gate_b2a/L7_rms": 0.09329236, "gate_b2a/L11_absmean": 0.13532834, "gate_b2a/L11_absmax": 0.54073507, "gate_b2a/L11_rms": 0.13917395, "gate_b2a/L14_absmean": 0.32775417, "gate_b2a/L14_absmax": 0.59837532, "gate_b2a/L14_rms": 0.32988584, "gate_b2c/L3_absmean": 0.10348164, "gate_b2c/L3_absmax": 0.677643, "gate_b2c/L3_rms": 0.11813495, "gate_b2c/L5_absmean": 0.1211078, "gate_b2c/L5_absmax": 0.83203667, "gate_b2c/L5_rms": 0.13100143, "gate_b2c/L6_absmean": 0.17670509, "gate_b2c/L6_absmax": 0.55788815, "gate_b2c/L6_rms": 0.18300968}, "streams": {"cos_a2b/round0": 0.396763, "cos_c2b/round0": 0.27477, "cos_b2a/round0": 0.100402, "cos_b2c/round0": 0.126438, "rms_a/round0": 1.033794, "rms_b/round0": 1.818923, "rms_c/round0": 1.396953, "cos_a2b/round1": 0.42469, "cos_c2b/round1": 0.286062, "cos_b2a/round1": 0.224277, "cos_b2c/round1": 0.131188, "rms_a/round1": 1.826815, "rms_b/round1": 3.840994, "rms_c/round1": 2.466539, "cos_a2b/round2": 0.526216, "cos_c2b/round2": 0.486576, "cos_b2a/round2": 0.031429, "cos_b2c/round2": 0.122816, "rms_a/round2": 4.10993, "rms_b/round2": 8.662178, "rms_c/round2": 4.082284}, "alpha_mean": 0.4125477008501264, "alpha_hist": [0.04335, 0.15157, 0.18118, 0.14954, 0.12393, 0.11247, 0.10591, 0.09058, 0.03825, 0.00322], "alpha_by_type": {"special": {"share": 0.00117, "mean": 0.63447, "std": 0.08003}, "whitespace": {"share": 0.0331, "mean": 0.62719, "std": 0.1787}, "digit": {"share": 0.00977, "mean": 0.39353, "std": 0.16172}, "word_start": {"share": 0.58998, "mean": 0.42131, "std": 0.2058}, "word_cont": {"share": 0.25171, "mean": 0.26774, "std": 0.14757}, "punct": {"share": 0.11427, "mean": 0.62343, "std": 0.17709}}, "alpha_p05": 0.09303, "alpha_p50": 0.38053, "alpha_p95": 0.78793}
{"step": 29310, "total_steps": 72479, "tokens": 15366881280, "loss": 4.844965636730194, "l_mix": 3.0029695630073547, "l_a": 3.0869532227516174, "l_c": 3.053033471107483, "z_loss": 20.04865264892578, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.3284178376197815, "tokens_per_second": 1438446.0548218112, "eta_seconds": 15734.332612705231, "data_wait_seconds": 1.6957521438598633e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20768, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 29320, "total_steps": 72479, "tokens": 15372124160, "loss": 4.203572064638138, "l_mix": 2.60021048784256, "l_a": 2.6983525156974792, "l_c": 2.6461855471134186, "z_loss": 20.833928108215332, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19025856256484985, "tokens_per_second": 1438840.695030569, "eta_seconds": 15726.373232388496, "data_wait_seconds": 1.5795230865478516e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 29330, "total_steps": 72479, "tokens": 15377367040, "loss": 4.25734668970108, "l_mix": 2.6352169513702393, "l_a": 2.7341023087501526, "l_c": 2.6729966700077057, "z_loss": 20.731192588806152, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1821839064359665, "tokens_per_second": 1433084.371315989, "eta_seconds": 15785.883486557006, "data_wait_seconds": 1.9282102584838867e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08234}}
{"step": 29340, "total_steps": 72479, "tokens": 15382609920, "loss": 4.199886322021484, "l_mix": 2.598191887140274, "l_a": 2.6986023783683777, "l_c": 2.6403788030147552, "z_loss": 20.76010298728943, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15759457647800446, "tokens_per_second": 1485446.1293709246, "eta_seconds": 15225.903911828995, "data_wait_seconds": 2.6464462280273438e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08234}}
{"step": 29350, "total_steps": 72479, "tokens": 15387852800, "loss": 4.1825955510139465, "l_mix": 2.5873429477214813, "l_a": 2.686163008213043, "l_c": 2.6313455998897552, "z_loss": 21.00705623626709, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14447079598903656, "tokens_per_second": 1398344.3088859676, "eta_seconds": 16170.564723086358, "data_wait_seconds": 1.6242265701293945e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 29360, "total_steps": 72479, "tokens": 15393095680, "loss": 3.3723209500312805, "l_mix": 2.074148863554001, "l_a": 2.195243686437607, "l_c": 2.131996273994446, "z_loss": 19.502297401428223, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.22036240994930267, "tokens_per_second": 1415760.8085833727, "eta_seconds": 15967.933378958702, "data_wait_seconds": 2.7686357498168945e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08232}}
{"step": 29370, "total_steps": 72479, "tokens": 15398338560, "loss": 3.407341241836548, "l_mix": 2.096912682056427, "l_a": 2.2165169417858124, "l_c": 2.1515780985355377, "z_loss": 19.281498432159424, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.23088331520557404, "tokens_per_second": 1335572.0512560287, "eta_seconds": 16922.73462202549, "data_wait_seconds": 2.142786979675293e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 29380, "total_steps": 72479, "tokens": 15403581440, "loss": 3.38004133105278, "l_mix": 2.079597383737564, "l_a": 2.1980977058410645, "l_c": 2.136715292930603, "z_loss": 20.61785125732422, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.23502258956432343, "tokens_per_second": 1432110.7588556781, "eta_seconds": 15778.31070137024, "data_wait_seconds": 2.8401613235473633e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 29390, "total_steps": 72479, "tokens": 15408824320, "loss": 3.3956309258937836, "l_mix": 2.089113086462021, "l_a": 2.2063312232494354, "l_c": 2.1487280428409576, "z_loss": 19.698878049850464, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19603775441646576, "tokens_per_second": 1444254.914898557, "eta_seconds": 15642.007099270819, "data_wait_seconds": 1.609325408935547e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 29400, "total_steps": 72479, "tokens": 15414067200, "loss": 3.432611644268036, "l_mix": 2.1120573580265045, "l_a": 2.2346605956554413, "l_c": 2.167186915874481, "z_loss": 19.987558364868164, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19703803956508636, "tokens_per_second": 1171134.588703906, "eta_seconds": 19285.403206300736, "data_wait_seconds": 1.9103288650512695e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.32479134, "gate_a2b/L1_absmax": 0.66725343, "gate_a2b/L1_rms": 0.33850473, "gate_a2b/L3_absmean": 0.34181437, "gate_a2b/L3_absmax": 0.56664282, "gate_a2b/L3_rms": 0.34643102, "gate_a2b/L5_absmean": 0.51602286, "gate_a2b/L5_absmax": 0.68036968, "gate_a2b/L5_rms": 0.52150112, "gate_c2b/L1_absmean": 0.01588617, "gate_c2b/L1_absmax": 0.46337488, "gate_c2b/L1_rms": 0.05729739, "gate_c2b/L3_absmean": 0.2156513, "gate_c2b/L3_absmax": 0.44113573, "gate_c2b/L3_rms": 0.21956581, "gate_c2b/L5_absmean": 0.54337329, "gate_c2b/L5_absmax": 0.73598284, "gate_c2b/L5_rms": 0.55555964, "gate_b2a/L7_absmean": 0.08840461, "gate_b2a/L7_absmax": 0.31320113, "gate_b2a/L7_rms": 0.09227338, "gate_b2a/L11_absmean": 0.13360667, "gate_b2a/L11_absmax": 0.54453874, "gate_b2a/L11_rms": 0.13759501, "gate_b2a/L14_absmean": 0.32488716, "gate_b2a/L14_absmax": 0.60443127, "gate_b2a/L14_rms": 0.32715392, "gate_b2c/L3_absmean": 0.10075808, "gate_b2c/L3_absmax": 0.68273425, "gate_b2c/L3_rms": 0.11549379, "gate_b2c/L5_absmean": 0.11864849, "gate_b2c/L5_absmax": 0.83352339, "gate_b2c/L5_rms": 0.12866992, "gate_b2c/L6_absmean": 0.17250809, "gate_b2c/L6_absmax": 0.55363619, "gate_b2c/L6_rms": 0.17897867}, "streams": {"cos_a2b/round0": 0.392914, "cos_c2b/round0": 0.270447, "cos_b2a/round0": 0.102262, "cos_b2c/round0": 0.122086, "rms_a/round0": 1.08078, "rms_b/round0": 1.939425, "rms_c/round0": 1.448226, "cos_a2b/round1": 0.425386, "cos_c2b/round1": 0.289187, "cos_b2a/round1": 0.223872, "cos_b2c/round1": 0.128675, "rms_a/round1": 1.92523, "rms_b/round1": 4.079554, "rms_c/round1": 2.557633, "cos_a2b/round2": 0.524306, "cos_c2b/round2": 0.486597, "cos_b2a/round2": 0.039141, "cos_b2c/round2": 0.124551, "rms_a/round2": 4.327296, "rms_b/round2": 9.146354, "rms_c/round2": 4.229437}, "alpha_mean": 0.4278968594799366, "alpha_hist": [0.03293, 0.12985, 0.18367, 0.15881, 0.12433, 0.11061, 0.11351, 0.10182, 0.04132, 0.00314], "alpha_by_type": {"special": {"share": 0.00116, "mean": 0.60392, "std": 0.11106}, "whitespace": {"share": 0.02974, "mean": 0.63838, "std": 0.17246}, "digit": {"share": 0.00835, "mean": 0.37667, "std": 0.16405}, "word_start": {"share": 0.59268, "mean": 0.44399, "std": 0.204}, "word_cont": {"share": 0.25682, "mean": 0.28161, "std": 0.15186}, "punct": {"share": 0.11125, "mean": 0.62562, "std": 0.18076}}, "alpha_p05": 0.11839, "alpha_p50": 0.39001, "alpha_p95": 0.79597}
{"step": 29410, "total_steps": 72479, "tokens": 15419310080, "loss": 3.351387679576874, "l_mix": 2.060978874564171, "l_a": 2.1813561022281647, "l_c": 2.12000635266304, "z_loss": 20.000744581222534, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18489348888397217, "tokens_per_second": 1452316.2882596098, "eta_seconds": 15547.962970972061, "data_wait_seconds": 2.5957822799682617e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 29420, "total_steps": 72479, "tokens": 15424552960, "loss": 4.157230317592621, "l_mix": 2.5717852413654327, "l_a": 2.6721253395080566, "l_c": 2.612691342830658, "z_loss": 21.130680799484253, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17021161317825317, "tokens_per_second": 1469422.8994821792, "eta_seconds": 15363.389940333367, "data_wait_seconds": 1.806020736694336e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08232}}
{"step": 29430, "total_steps": 72479, "tokens": 15429795840, "loss": 4.17106568813324, "l_mix": 2.580854594707489, "l_a": 2.677507370710373, "l_c": 2.6231957972049713, "z_loss": 21.3072988986969, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17858488857746124, "tokens_per_second": 1290400.7379864024, "eta_seconds": 17490.747988271713, "data_wait_seconds": 2.0891427993774414e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 29440, "total_steps": 72479, "tokens": 15435038720, "loss": 4.26746541261673, "l_mix": 2.6398364901542664, "l_a": 2.744829446077347, "l_c": 2.680599957704544, "z_loss": 20.83538007736206, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19679175317287445, "tokens_per_second": 1385971.9288831856, "eta_seconds": 16280.871756315231, "data_wait_seconds": 1.6629695892333984e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 29450, "total_steps": 72479, "tokens": 15440281600, "loss": 4.261638283729553, "l_mix": 2.6369378864765167, "l_a": 2.7376511991024017, "l_c": 2.678016781806946, "z_loss": 20.752237796783447, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1672157496213913, "tokens_per_second": 1376971.873346978, "eta_seconds": 16383.477969789505, "data_wait_seconds": 1.65402889251709e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 29460, "total_steps": 72479, "tokens": 15445524480, "loss": 4.241717875003815, "l_mix": 2.624082088470459, "l_a": 2.72382053732872, "l_c": 2.668298602104187, "z_loss": 22.104800701141357, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18328827619552612, "tokens_per_second": 1320527.4484863125, "eta_seconds": 17079.800573515895, "data_wait_seconds": 2.288818359375e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 29470, "total_steps": 72479, "tokens": 15450767360, "loss": 4.302737534046173, "l_mix": 2.662617564201355, "l_a": 2.7588108479976654, "l_c": 2.708255559206009, "z_loss": 21.41661047935486, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15330344438552856, "tokens_per_second": 1407086.9823000056, "eta_seconds": 16025.37929470539, "data_wait_seconds": 3.266334533691406e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 29480, "total_steps": 72479, "tokens": 15456010240, "loss": 3.4032112658023834, "l_mix": 2.092575281858444, "l_a": 2.217195600271225, "l_c": 2.1515907645225525, "z_loss": 20.34554672241211, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.22886967658996582, "tokens_per_second": 1407891.7262375487, "eta_seconds": 16012.495344543457, "data_wait_seconds": 3.933906555175781e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 29490, "total_steps": 72479, "tokens": 15461253120, "loss": 3.3612635135650635, "l_mix": 2.0668435990810394, "l_a": 2.1898462176322937, "l_c": 2.1248867511749268, "z_loss": 20.371640920639038, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19026295840740204, "tokens_per_second": 1417555.791931827, "eta_seconds": 15899.632988190651, "data_wait_seconds": 2.3990869522094727e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 29500, "total_steps": 72479, "tokens": 15466496000, "loss": 3.391082674264908, "l_mix": 2.0858378410339355, "l_a": 2.2070597410202026, "l_c": 2.1437561213970184, "z_loss": 20.35246229171753, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17597389221191406, "tokens_per_second": 1425687.1853355588, "eta_seconds": 15805.272140884399, "data_wait_seconds": 1.582503318786621e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.32231084, "gate_a2b/L1_absmax": 0.65757912, "gate_a2b/L1_rms": 0.33625048, "gate_a2b/L3_absmean": 0.34020743, "gate_a2b/L3_absmax": 0.57884449, "gate_a2b/L3_rms": 0.3449229, "gate_a2b/L5_absmean": 0.51467115, "gate_a2b/L5_absmax": 0.67671782, "gate_a2b/L5_rms": 0.52015549, "gate_c2b/L1_absmean": 0.01595004, "gate_c2b/L1_absmax": 0.46920374, "gate_c2b/L1_rms": 0.05780395, "gate_c2b/L3_absmean": 0.21601148, "gate_c2b/L3_absmax": 0.45028198, "gate_c2b/L3_rms": 0.22003742, "gate_c2b/L5_absmean": 0.54616749, "gate_c2b/L5_absmax": 0.74417478, "gate_c2b/L5_rms": 0.55837828, "gate_b2a/L7_absmean": 0.08734142, "gate_b2a/L7_absmax": 0.31319103, "gate_b2a/L7_rms": 0.09123286, "gate_b2a/L11_absmean": 0.13287321, "gate_b2a/L11_absmax": 0.54530388, "gate_b2a/L11_rms": 0.13691622, "gate_b2a/L14_absmean": 0.32281035, "gate_b2a/L14_absmax": 0.60465747, "gate_b2a/L14_rms": 0.32523489, "gate_b2c/L3_absmean": 0.09841224, "gate_b2c/L3_absmax": 0.68214166, "gate_b2c/L3_rms": 0.11308781, "gate_b2c/L5_absmean": 0.11500289, "gate_b2c/L5_absmax": 0.83768004, "gate_b2c/L5_rms": 0.12538506, "gate_b2c/L6_absmean": 0.16852498, "gate_b2c/L6_absmax": 0.56117874, "gate_b2c/L6_rms": 0.17525883}, "streams": {"cos_a2b/round0": 0.395444, "cos_c2b/round0": 0.269595, "cos_b2a/round0": 0.102906, "cos_b2c/round0": 0.119065, "rms_a/round0": 1.131998, "rms_b/round0": 2.00983, "rms_c/round0": 1.478664, "cos_a2b/round1": 0.425625, "cos_c2b/round1": 0.287236, "cos_b2a/round1": 0.222357, "cos_b2c/round1": 0.130011, "rms_a/round1": 2.018233, "rms_b/round1": 4.320546, "rms_c/round1": 2.663317, "cos_a2b/round2": 0.524848, "cos_c2b/round2": 0.486519, "cos_b2a/round2": 0.037641, "cos_b2c/round2": 0.122833, "rms_a/round2": 4.568595, "rms_b/round2": 9.712794, "rms_c/round2": 4.447888}, "alpha_mean": 0.44091389361188504, "alpha_hist": [0.04016, 0.12206, 0.16072, 0.14778, 0.1324, 0.11963, 0.11346, 0.10287, 0.05449, 0.00642], "alpha_by_type": {"special": {"share": 0.0011, "mean": 0.59512, "std": 0.10508}, "whitespace": {"share": 0.03519, "mean": 0.62131, "std": 0.19127}, "digit": {"share": 0.01245, "mean": 0.38332, "std": 0.14921}, "word_start": {"share": 0.57767, "mean": 0.46139, "std": 0.20676}, "word_cont": {"share": 0.25504, "mean": 0.28153, "std": 0.16294}, "punct": {"share": 0.11856, "mean": 0.63509, "std": 0.18797}}, "alpha_p05": 0.12169, "alpha_p50": 0.42799, "alpha_p95": 0.79912}
{"step": 29510, "total_steps": 72479, "tokens": 15471738880, "loss": 3.4172011613845825, "l_mix": 2.102724015712738, "l_a": 2.221675455570221, "l_c": 2.159914880990982, "z_loss": 20.361891984939575, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1568143665790558, "tokens_per_second": 1380120.6582685425, "eta_seconds": 16323.30545668602, "data_wait_seconds": 1.9162893295288086e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 29520, "total_steps": 72479, "tokens": 15476981760, "loss": 3.3465373516082764, "l_mix": 2.0574784576892853, "l_a": 2.1803221106529236, "l_c": 2.116540879011154, "z_loss": 20.502378225326538, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20190311968326569, "tokens_per_second": 1394873.4253301513, "eta_seconds": 16146.904645967485, "data_wait_seconds": 1.7136335372924805e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 29530, "total_steps": 72479, "tokens": 15482224640, "loss": 4.11624276638031, "l_mix": 2.5456346571445465, "l_a": 2.644382417201996, "l_c": 2.5909773111343384, "z_loss": 21.73649501800537, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16222555935382843, "tokens_per_second": 1430953.3312111113, "eta_seconds": 15736.11439371109, "data_wait_seconds": 1.576542854309082e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08232}}
{"step": 29540, "total_steps": 72479, "tokens": 15487467520, "loss": 4.154500842094421, "l_mix": 2.570259839296341, "l_a": 2.6680696308612823, "l_c": 2.61273330450058, "z_loss": 21.335954427719116, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17690123617649078, "tokens_per_second": 1438754.27544113, "eta_seconds": 15647.148937296866, "data_wait_seconds": 2.54213809967041e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 29550, "total_steps": 72479, "tokens": 15492710400, "loss": 4.188668638467789, "l_mix": 2.5921328961849213, "l_a": 2.68568953871727, "l_c": 2.6360963881015778, "z_loss": 21.0519540309906, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2063073217868805, "tokens_per_second": 1382956.9530441083, "eta_seconds": 16274.663866043089, "data_wait_seconds": 1.531839370727539e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 29560, "total_steps": 72479, "tokens": 15497953280, "loss": 4.151462256908417, "l_mix": 2.5676995515823364, "l_a": 2.669764459133148, "l_c": 2.6094445288181305, "z_loss": 21.487226963043213, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15342529118061066, "tokens_per_second": 1410668.1277963282, "eta_seconds": 15951.247659611701, "data_wait_seconds": 1.7702579498291016e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 29570, "total_steps": 72479, "tokens": 15503196160, "loss": 4.196947157382965, "l_mix": 2.5972445607185364, "l_a": 2.692674458026886, "l_c": 2.6396671533584595, "z_loss": 22.10481882095337, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2134278267621994, "tokens_per_second": 1452264.9748313057, "eta_seconds": 15490.750091671942, "data_wait_seconds": 1.6689300537109375e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 29580, "total_steps": 72479, "tokens": 15508439040, "loss": 3.361195683479309, "l_mix": 2.066683679819107, "l_a": 2.190528482198715, "l_c": 2.1245115101337433, "z_loss": 21.48345685005188, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2023647576570511, "tokens_per_second": 1337020.3182086865, "eta_seconds": 16822.056183958055, "data_wait_seconds": 2.9027462005615234e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 29590, "total_steps": 72479, "tokens": 15513681920, "loss": 4.843996942043304, "l_mix": 3.0041943192481995, "l_a": 3.0860359966754913, "l_c": 3.046639233827591, "z_loss": 21.07788395881653, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15177199244499207, "tokens_per_second": 1426555.550615751, "eta_seconds": 15762.574420809748, "data_wait_seconds": 2.9861927032470703e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 29600, "total_steps": 72479, "tokens": 15518924800, "loss": 4.982141852378845, "l_mix": 3.090049147605896, "l_a": 3.1721199452877045, "l_c": 3.1348555088043213, "z_loss": 20.993337154388428, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17549200356006622, "tokens_per_second": 1464841.091770025, "eta_seconds": 15347.01974043846, "data_wait_seconds": 1.71661376953125e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.32029322, "gate_a2b/L1_absmax": 0.65919727, "gate_a2b/L1_rms": 0.33411708, "gate_a2b/L3_absmean": 0.33889461, "gate_a2b/L3_absmax": 0.58133745, "gate_a2b/L3_rms": 0.34364688, "gate_a2b/L5_absmean": 0.51311779, "gate_a2b/L5_absmax": 0.67854941, "gate_a2b/L5_rms": 0.51863664, "gate_c2b/L1_absmean": 0.01463614, "gate_c2b/L1_absmax": 0.47082826, "gate_c2b/L1_rms": 0.05780905, "gate_c2b/L3_absmean": 0.2163115, "gate_c2b/L3_absmax": 0.44947085, "gate_c2b/L3_rms": 0.22029053, "gate_c2b/L5_absmean": 0.54907936, "gate_c2b/L5_absmax": 0.74795902, "gate_c2b/L5_rms": 0.56137127, "gate_b2a/L7_absmean": 0.08614987, "gate_b2a/L7_absmax": 0.31352079, "gate_b2a/L7_rms": 0.09011928, "gate_b2a/L11_absmean": 0.13078606, "gate_b2a/L11_absmax": 0.54635918, "gate_b2a/L11_rms": 0.13492332, "gate_b2a/L14_absmean": 0.32081938, "gate_b2a/L14_absmax": 0.60191399, "gate_b2a/L14_rms": 0.32332489, "gate_b2c/L3_absmean": 0.09626019, "gate_b2c/L3_absmax": 0.68482023, "gate_b2c/L3_rms": 0.11086563, "gate_b2c/L5_absmean": 0.11260211, "gate_b2c/L5_absmax": 0.83655286, "gate_b2c/L5_rms": 0.12307096, "gate_b2c/L6_absmean": 0.1648982, "gate_b2c/L6_absmax": 0.55957377, "gate_b2c/L6_rms": 0.17177044}, "streams": {"cos_a2b/round0": 0.3951, "cos_c2b/round0": 0.307755, "cos_b2a/round0": 0.11717, "cos_b2c/round0": 0.159651, "rms_a/round0": 1.1287, "rms_b/round0": 2.093674, "rms_c/round0": 1.471081, "cos_a2b/round1": 0.426643, "cos_c2b/round1": 0.290494, "cos_b2a/round1": 0.233328, "cos_b2c/round1": 0.127028, "rms_a/round1": 2.007518, "rms_b/round1": 4.246238, "rms_c/round1": 2.614205, "cos_a2b/round2": 0.571574, "cos_c2b/round2": 0.503723, "cos_b2a/round2": 0.084282, "cos_b2c/round2": 0.155812, "rms_a/round2": 4.702581, "rms_b/round2": 9.71521, "rms_c/round2": 4.403597}, "alpha_mean": 0.4889634654909969, "alpha_hist": [0.00337, 0.02664, 0.09705, 0.16771, 0.21478, 0.23396, 0.16908, 0.07619, 0.01094, 0.00027], "alpha_by_type": {"special": {"share": 0.00085, "mean": 0.59808, "std": 0.09843}, "whitespace": {"share": 0.05545, "mean": 0.56286, "std": 0.11418}, "digit": {"share": 0.03336, "mean": 0.52053, "std": 0.11634}, "word_start": {"share": 0.54189, "mean": 0.50347, "std": 0.14734}, "word_cont": {"share": 0.25497, "mean": 0.38836, "std": 0.12978}, "punct": {"share": 0.11348, "mean": 0.59954, "std": 0.14115}}, "alpha_p05": 0.20308, "alpha_p50": 0.49121, "alpha_p95": 0.73754}
{"step": 29610, "total_steps": 72479, "tokens": 15524167680, "loss": 3.3641603589057922, "l_mix": 2.0695458203554153, "l_a": 2.192591428756714, "l_c": 2.122790038585663, "z_loss": 22.333160638809204, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.28701475262641907, "tokens_per_second": 1356761.053553612, "eta_seconds": 16565.704191708566, "data_wait_seconds": 1.8149614334106445e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 29620, "total_steps": 72479, "tokens": 15529410560, "loss": 3.4389835000038147, "l_mix": 2.1175091564655304, "l_a": 2.227260500192642, "l_c": 2.1776537597179413, "z_loss": 20.552133560180664, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.28555554151535034, "tokens_per_second": 1415276.06411963, "eta_seconds": 15877.085723185539, "data_wait_seconds": 3.325939178466797e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 29630, "total_steps": 72479, "tokens": 15534653440, "loss": 3.392973482608795, "l_mix": 2.0877095758914948, "l_a": 2.209047704935074, "l_c": 2.1418318450450897, "z_loss": 21.87550640106201, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2620086073875427, "tokens_per_second": 1374950.8584079451, "eta_seconds": 16338.923223781585, "data_wait_seconds": 3.820657730102539e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 29640, "total_steps": 72479, "tokens": 15539896320, "loss": 4.223507106304169, "l_mix": 2.6149014830589294, "l_a": 2.7089067101478577, "l_c": 2.653111457824707, "z_loss": 21.66222381591797, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.21772468090057373, "tokens_per_second": 1423105.734959682, "eta_seconds": 15782.36464111805, "data_wait_seconds": 1.519918441772461e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08232}}
{"step": 29650, "total_steps": 72479, "tokens": 15545139200, "loss": 4.2673229575157166, "l_mix": 2.64035502076149, "l_a": 2.7413824796676636, "l_c": 2.6818434596061707, "z_loss": 22.169103145599365, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1555185467004776, "tokens_per_second": 1431703.4887829595, "eta_seconds": 15683.925427246093, "data_wait_seconds": 2.5838613510131836e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 29660, "total_steps": 72479, "tokens": 15550382080, "loss": 4.23871922492981, "l_mix": 2.622577428817749, "l_a": 2.7197841703891754, "l_c": 2.6673548817634583, "z_loss": 21.831634759902954, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15375301241874695, "tokens_per_second": 1358481.7977731882, "eta_seconds": 16525.424123311044, "data_wait_seconds": 2.154707908630371e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 29670, "total_steps": 72479, "tokens": 15555624960, "loss": 4.151354968547821, "l_mix": 2.567543387413025, "l_a": 2.667422592639923, "l_c": 2.611949145793915, "z_loss": 22.481473207473755, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1406991183757782, "tokens_per_second": 1425504.5646468236, "eta_seconds": 15744.772446632383, "data_wait_seconds": 2.2411346435546875e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 29680, "total_steps": 72479, "tokens": 15560867840, "loss": 4.096403658390045, "l_mix": 2.533200591802597, "l_a": 2.635241985321045, "l_c": 2.575434625148773, "z_loss": 22.3301203250885, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1900736540555954, "tokens_per_second": 1399193.3856660484, "eta_seconds": 16037.098475360872, "data_wait_seconds": 1.7076730728149414e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 29690, "total_steps": 72479, "tokens": 15566110720, "loss": 3.375545471906662, "l_mix": 2.0765801072120667, "l_a": 2.196390688419342, "l_c": 2.133493721485138, "z_loss": 21.400557041168213, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15242627263069153, "tokens_per_second": 1406214.4528873262, "eta_seconds": 15953.298720502851, "data_wait_seconds": 2.676248550415039e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 29700, "total_steps": 72479, "tokens": 15571353600, "loss": 4.859919488430023, "l_mix": 3.0139462649822235, "l_a": 3.091835379600525, "l_c": 3.06140860915184, "z_loss": 21.758887767791748, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15682458877563477, "tokens_per_second": 1431584.6521113566, "eta_seconds": 15666.915902543069, "data_wait_seconds": 2.4497509002685547e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.20044, "finemath": 0.08232}, "gates": {"gate_a2b/L1_absmean": 0.31878546, "gate_a2b/L1_absmax": 0.65500861, "gate_a2b/L1_rms": 0.33270785, "gate_a2b/L3_absmean": 0.33647391, "gate_a2b/L3_absmax": 0.58239174, "gate_a2b/L3_rms": 0.34133649, "gate_a2b/L5_absmean": 0.5112257, "gate_a2b/L5_absmax": 0.67495233, "gate_a2b/L5_rms": 0.51670527, "gate_c2b/L1_absmean": 0.01539394, "gate_c2b/L1_absmax": 0.4774285, "gate_c2b/L1_rms": 0.05834915, "gate_c2b/L3_absmean": 0.21594433, "gate_c2b/L3_absmax": 0.45255432, "gate_c2b/L3_rms": 0.22002739, "gate_c2b/L5_absmean": 0.55147296, "gate_c2b/L5_absmax": 0.75529641, "gate_c2b/L5_rms": 0.56379133, "gate_b2a/L7_absmean": 0.08516864, "gate_b2a/L7_absmax": 0.31470293, "gate_b2a/L7_rms": 0.08919907, "gate_b2a/L11_absmean": 0.12962982, "gate_b2a/L11_absmax": 0.54942876, "gate_b2a/L11_rms": 0.13390802, "gate_b2a/L14_absmean": 0.31850082, "gate_b2a/L14_absmax": 0.60388672, "gate_b2a/L14_rms": 0.3211264, "gate_b2c/L3_absmean": 0.09416699, "gate_b2c/L3_absmax": 0.68554384, "gate_b2c/L3_rms": 0.10864038, "gate_b2c/L5_absmean": 0.11022614, "gate_b2c/L5_absmax": 0.83693528, "gate_b2c/L5_rms": 0.12074391, "gate_b2c/L6_absmean": 0.16097106, "gate_b2c/L6_absmax": 0.55734926, "gate_b2c/L6_rms": 0.16801131}, "streams": {"cos_a2b/round0": 0.405358, "cos_c2b/round0": 0.322591, "cos_b2a/round0": 0.123368, "cos_b2c/round0": 0.164059, "rms_a/round0": 1.169905, "rms_b/round0": 2.159918, "rms_c/round0": 1.502679, "cos_a2b/round1": 0.429411, "cos_c2b/round1": 0.291898, "cos_b2a/round1": 0.244242, "cos_b2c/round1": 0.130663, "rms_a/round1": 2.056168, "rms_b/round1": 4.388418, "rms_c/round1": 2.675434, "cos_a2b/round2": 0.562876, "cos_c2b/round2": 0.519386, "cos_b2a/round2": 0.090622, "cos_b2c/round2": 0.168544, "rms_a/round2": 4.801777, "rms_b/round2": 10.024536, "rms_c/round2": 4.464042}, "alpha_mean": 0.44632849557292786, "alpha_hist": [0.00433, 0.05022, 0.142, 0.19296, 0.22136, 0.21765, 0.12582, 0.04242, 0.00316, 8e-05], "alpha_by_type": {"special": {"share": 0.00053, "mean": 0.51353, "std": 0.13294}, "whitespace": {"share": 0.08273, "mean": 0.52671, "std": 0.10472}, "digit": {"share": 0.02997, "mean": 0.47473, "std": 0.1048}, "word_start": {"share": 0.54517, "mean": 0.45406, "std": 0.14885}, "word_cont": {"share": 0.23495, "mean": 0.34634, "std": 0.12672}, "punct": {"share": 0.10664, "mean": 0.5564, "std": 0.14257}}, "alpha_p05": 0.19163, "alpha_p50": 0.43915, "alpha_p95": 0.69118}
{"step": 29710, "total_steps": 72479, "tokens": 15576596480, "loss": 4.873224079608917, "l_mix": 3.0207323729991913, "l_a": 3.107550323009491, "l_c": 3.0674217343330383, "z_loss": 22.07613730430603, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1671075075864792, "tokens_per_second": 1412588.2155236085, "eta_seconds": 15873.892494344713, "data_wait_seconds": 3.063678741455078e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 29720, "total_steps": 72479, "tokens": 15581839360, "loss": 4.860714256763458, "l_mix": 3.013252466917038, "l_a": 3.0987323820590973, "l_c": 3.059473007917404, "z_loss": 22.276222944259644, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.24428284168243408, "tokens_per_second": 1361796.154556222, "eta_seconds": 16462.104491186143, "data_wait_seconds": 1.8656253814697266e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 29730, "total_steps": 72479, "tokens": 15587082240, "loss": 4.742510616779327, "l_mix": 2.9410938918590546, "l_a": 3.0191080272197723, "l_c": 2.985613763332367, "z_loss": 21.16532039642334, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19701337814331055, "tokens_per_second": 1473265.7390563777, "eta_seconds": 15212.997301054, "data_wait_seconds": 1.7017126083374023e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 29740, "total_steps": 72479, "tokens": 15592325120, "loss": 3.3899711668491364, "l_mix": 2.087277054786682, "l_a": 2.2022849023342133, "l_c": 2.140028566122055, "z_loss": 21.423064947128296, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2217046618461609, "tokens_per_second": 1402130.60102818, "eta_seconds": 15981.068251109124, "data_wait_seconds": 1.6570091247558594e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 29750, "total_steps": 72479, "tokens": 15597568000, "loss": 3.347856968641281, "l_mix": 2.0587739646434784, "l_a": 2.1803524494171143, "l_c": 2.116590678691864, "z_loss": 21.931003093719482, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19382387399673462, "tokens_per_second": 1350490.7440787533, "eta_seconds": 16588.267672491074, "data_wait_seconds": 4.482269287109375e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 29760, "total_steps": 72479, "tokens": 15602810880, "loss": 4.191406846046448, "l_mix": 2.593072086572647, "l_a": 2.6944019198417664, "l_c": 2.633380562067032, "z_loss": 22.036137342453003, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17682458460330963, "tokens_per_second": 1363890.3071356695, "eta_seconds": 16421.451897430423, "data_wait_seconds": 1.7851591110229492e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 29770, "total_steps": 72479, "tokens": 15608053760, "loss": 4.270901679992676, "l_mix": 2.6432237327098846, "l_a": 2.737952023744583, "l_c": 2.6876406967639923, "z_loss": 22.416720151901245, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17097187042236328, "tokens_per_second": 1438432.8819218045, "eta_seconds": 15566.8133518219, "data_wait_seconds": 2.3484230041503906e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 29780, "total_steps": 72479, "tokens": 15613296640, "loss": 4.124848902225494, "l_mix": 2.5515244603157043, "l_a": 2.6526966392993927, "l_c": 2.591718167066574, "z_loss": 22.207777738571167, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1742546707391739, "tokens_per_second": 1429581.0114331467, "eta_seconds": 15659.53459997177, "data_wait_seconds": 2.053380012512207e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 29790, "total_steps": 72479, "tokens": 15618539520, "loss": 4.269111692905426, "l_mix": 2.6412510573863983, "l_a": 2.7379060685634613, "l_c": 2.6882958710193634, "z_loss": 23.268383026123047, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.23804138600826263, "tokens_per_second": 1421791.8989055227, "eta_seconds": 15741.635923814774, "data_wait_seconds": 1.683831214904785e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 29800, "total_steps": 72479, "tokens": 15623782400, "loss": 3.3942929804325104, "l_mix": 2.088679790496826, "l_a": 2.2063309848308563, "l_c": 2.1457128822803497, "z_loss": 21.934101343154907, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2014295756816864, "tokens_per_second": 1393417.4335727422, "eta_seconds": 16058.423709130288, "data_wait_seconds": 3.4749507904052734e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.31593439, "gate_a2b/L1_absmax": 0.64657974, "gate_a2b/L1_rms": 0.32984009, "gate_a2b/L3_absmean": 0.33499488, "gate_a2b/L3_absmax": 0.59003931, "gate_a2b/L3_rms": 0.3399522, "gate_a2b/L5_absmean": 0.50970089, "gate_a2b/L5_absmax": 0.66998059, "gate_a2b/L5_rms": 0.51521885, "gate_c2b/L1_absmean": 0.0167513, "gate_c2b/L1_absmax": 0.48069325, "gate_c2b/L1_rms": 0.05924911, "gate_c2b/L3_absmean": 0.21613789, "gate_c2b/L3_absmax": 0.4518216, "gate_c2b/L3_rms": 0.22016302, "gate_c2b/L5_absmean": 0.5519008, "gate_c2b/L5_absmax": 0.7487151, "gate_c2b/L5_rms": 0.56441069, "gate_b2a/L7_absmean": 0.08413666, "gate_b2a/L7_absmax": 0.31314024, "gate_b2a/L7_rms": 0.08832175, "gate_b2a/L11_absmean": 0.1283232, "gate_b2a/L11_absmax": 0.54835039, "gate_b2a/L11_rms": 0.13276675, "gate_b2a/L14_absmean": 0.31553969, "gate_b2a/L14_absmax": 0.61501098, "gate_b2a/L14_rms": 0.31836465, "gate_b2c/L3_absmean": 0.0921859, "gate_b2c/L3_absmax": 0.68241328, "gate_b2c/L3_rms": 0.10655588, "gate_b2c/L5_absmean": 0.1082465, "gate_b2c/L5_absmax": 0.82549655, "gate_b2c/L5_rms": 0.11880784, "gate_b2c/L6_absmean": 0.15745848, "gate_b2c/L6_absmax": 0.56782031, "gate_b2c/L6_rms": 0.16478881}, "streams": {"cos_a2b/round0": 0.398232, "cos_c2b/round0": 0.263257, "cos_b2a/round0": 0.104102, "cos_b2c/round0": 0.113051, "rms_a/round0": 1.284978, "rms_b/round0": 2.283891, "rms_c/round0": 1.604849, "cos_a2b/round1": 0.426328, "cos_c2b/round1": 0.2891, "cos_b2a/round1": 0.23087, "cos_b2c/round1": 0.127068, "rms_a/round1": 2.261666, "rms_b/round1": 4.889303, "rms_c/round1": 2.887154, "cos_a2b/round2": 0.525103, "cos_c2b/round2": 0.493156, "cos_b2a/round2": 0.047504, "cos_b2c/round2": 0.135998, "rms_a/round2": 5.139515, "rms_b/round2": 11.113435, "rms_c/round2": 4.76569}, "alpha_mean": 0.4055054747657536, "alpha_hist": [0.03893, 0.16409, 0.19231, 0.14876, 0.11569, 0.10683, 0.10893, 0.0873, 0.03464, 0.00253], "alpha_by_type": {"special": {"share": 0.00113, "mean": 0.58473, "std": 0.11795}, "whitespace": {"share": 0.03229, "mean": 0.64228, "std": 0.1744}, "digit": {"share": 0.00891, "mean": 0.36272, "std": 0.15441}, "word_start": {"share": 0.56522, "mean": 0.41286, "std": 0.20558}, "word_cont": {"share": 0.26997, "mean": 0.26983, "std": 0.14624}, "punct": {"share": 0.12248, "mean": 0.60966, "std": 0.17787}}, "alpha_p05": 0.10744, "alpha_p50": 0.36207, "alpha_p95": 0.78497}
{"step": 29810, "total_steps": 72479, "tokens": 15629025280, "loss": 3.205063134431839, "l_mix": 1.9829526990652084, "l_a": 2.0579796731472015, "l_c": 2.0157217532396317, "z_loss": 25.287723302841187, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.22740329802036285, "tokens_per_second": 1383741.116637697, "eta_seconds": 16166.929205918312, "data_wait_seconds": 2.3484230041503906e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 29820, "total_steps": 72479, "tokens": 15634268160, "loss": 4.894629955291748, "l_mix": 3.035833716392517, "l_a": 3.1171058118343353, "l_c": 3.0788814425468445, "z_loss": 21.65591025352478, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15390215814113617, "tokens_per_second": 1374215.782312024, "eta_seconds": 16275.174597668647, "data_wait_seconds": 3.3795833587646484e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 29830, "total_steps": 72479, "tokens": 15639511040, "loss": 4.878188252449036, "l_mix": 3.024395912885666, "l_a": 3.1103131473064423, "l_c": 3.068994164466858, "z_loss": 22.257911920547485, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.25877705216407776, "tokens_per_second": 1416435.08222805, "eta_seconds": 15786.363379836082, "data_wait_seconds": 2.3037195205688477e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20768, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 29840, "total_steps": 72479, "tokens": 15644753920, "loss": 4.795619368553162, "l_mix": 2.9730489552021027, "l_a": 3.05845108628273, "l_c": 3.016783207654953, "z_loss": 22.505630493164062, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.21253322064876556, "tokens_per_second": 1400109.9928225225, "eta_seconds": 15966.685579419136, "data_wait_seconds": 2.2172927856445312e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 29850, "total_steps": 72479, "tokens": 15649996800, "loss": 4.157718181610107, "l_mix": 2.571522831916809, "l_a": 2.668627977371216, "l_c": 2.6186894178390503, "z_loss": 22.468635082244873, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.22921547293663025, "tokens_per_second": 1413699.2992150034, "eta_seconds": 15809.495813155174, "data_wait_seconds": 2.1159648895263672e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 29860, "total_steps": 72479, "tokens": 15655239680, "loss": 4.096276968717575, "l_mix": 2.5332846343517303, "l_a": 2.637966275215149, "l_c": 2.572007954120636, "z_loss": 23.18761110305786, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17510466277599335, "tokens_per_second": 1276966.5631972721, "eta_seconds": 17498.21092891693, "data_wait_seconds": 0.06316736340522766, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 29870, "total_steps": 72479, "tokens": 15660482560, "loss": 4.204095810651779, "l_mix": 2.600845068693161, "l_a": 2.6972625255584717, "l_c": 2.6469064950942993, "z_loss": 22.319635152816772, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1885775625705719, "tokens_per_second": 1433124.2511789913, "eta_seconds": 15587.892936444283, "data_wait_seconds": 2.1398067474365234e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08232}}
{"step": 29880, "total_steps": 72479, "tokens": 15665725440, "loss": 4.154458582401276, "l_mix": 2.569851577281952, "l_a": 2.672868996858597, "l_c": 2.609154134988785, "z_loss": 23.00753617286682, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19046135246753693, "tokens_per_second": 1323344.2593144788, "eta_seconds": 16877.04794485569, "data_wait_seconds": 3.018975257873535e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 29890, "total_steps": 72479, "tokens": 15670968320, "loss": 4.089851975440979, "l_mix": 2.5289952754974365, "l_a": 2.6300554275512695, "l_c": 2.5728000700473785, "z_loss": 22.97038459777832, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16901063919067383, "tokens_per_second": 1160522.9905944273, "eta_seconds": 19240.378530168535, "data_wait_seconds": 2.6792287826538086e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 29900, "total_steps": 72479, "tokens": 15676211200, "loss": 4.185865104198456, "l_mix": 2.5886715054512024, "l_a": 2.691382735967636, "l_c": 2.6325956881046295, "z_loss": 23.339078903198242, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17552492022514343, "tokens_per_second": 1443411.1083796644, "eta_seconds": 15465.904774045945, "data_wait_seconds": 1.919269561767578e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.31421682, "gate_a2b/L1_absmax": 0.6512273, "gate_a2b/L1_rms": 0.32817277, "gate_a2b/L3_absmean": 0.33283016, "gate_a2b/L3_absmax": 0.58944881, "gate_a2b/L3_rms": 0.33773455, "gate_a2b/L5_absmean": 0.50770205, "gate_a2b/L5_absmax": 0.67075795, "gate_a2b/L5_rms": 0.51326066, "gate_c2b/L1_absmean": 0.01561167, "gate_c2b/L1_absmax": 0.48232368, "gate_c2b/L1_rms": 0.05925032, "gate_c2b/L3_absmean": 0.21610485, "gate_c2b/L3_absmax": 0.45314655, "gate_c2b/L3_rms": 0.22017433, "gate_c2b/L5_absmean": 0.55494541, "gate_c2b/L5_absmax": 0.75348264, "gate_c2b/L5_rms": 0.56757623, "gate_b2a/L7_absmean": 0.08367343, "gate_b2a/L7_absmax": 0.31555438, "gate_b2a/L7_rms": 0.08783206, "gate_b2a/L11_absmean": 0.12688032, "gate_b2a/L11_absmax": 0.55098605, "gate_b2a/L11_rms": 0.13135323, "gate_b2a/L14_absmean": 0.31357032, "gate_b2a/L14_absmax": 0.62176913, "gate_b2a/L14_rms": 0.31646901, "gate_b2c/L3_absmean": 0.09035202, "gate_b2c/L3_absmax": 0.68293035, "gate_b2c/L3_rms": 0.10456936, "gate_b2c/L5_absmean": 0.10600986, "gate_b2c/L5_absmax": 0.83415115, "gate_b2c/L5_rms": 0.11678239, "gate_b2c/L6_absmean": 0.1548253, "gate_b2c/L6_absmax": 0.55916882, "gate_b2c/L6_rms": 0.16211687}, "streams": {"cos_a2b/round0": 0.405571, "cos_c2b/round0": 0.312171, "cos_b2a/round0": 0.118938, "cos_b2c/round0": 0.151074, "rms_a/round0": 1.270056, "rms_b/round0": 2.341876, "rms_c/round0": 1.60495, "cos_a2b/round1": 0.432226, "cos_c2b/round1": 0.290147, "cos_b2a/round1": 0.235879, "cos_b2c/round1": 0.131574, "rms_a/round1": 2.267313, "rms_b/round1": 4.904665, "rms_c/round1": 2.852692, "cos_a2b/round2": 0.560633, "cos_c2b/round2": 0.504142, "cos_b2a/round2": 0.075006, "cos_b2c/round2": 0.154726, "rms_a/round2": 5.338386, "rms_b/round2": 11.377634, "rms_c/round2": 4.770756}, "alpha_mean": 0.45344822114257965, "alpha_hist": [0.00864, 0.06223, 0.15366, 0.19266, 0.17963, 0.16541, 0.1384, 0.08345, 0.01578, 0.00017], "alpha_by_type": {"special": {"share": 0.00092, "mean": 0.57092, "std": 0.08435}, "whitespace": {"share": 0.02426, "mean": 0.56257, "std": 0.1052}, "digit": {"share": 0.02283, "mean": 0.45672, "std": 0.12806}, "word_start": {"share": 0.61894, "mean": 0.4662, "std": 0.17069}, "word_cont": {"share": 0.23557, "mean": 0.34087, "std": 0.13566}, "punct": {"share": 0.09749, "mean": 0.61549, "std": 0.15695}}, "alpha_p05": 0.18654, "alpha_p50": 0.43975, "alpha_p95": 0.76278}
{"step": 29910, "total_steps": 72479, "tokens": 15681454080, "loss": 3.3087157607078552, "l_mix": 2.0344058722257614, "l_a": 2.158478856086731, "l_c": 2.0892206132411957, "z_loss": 22.634901762008667, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17129208147525787, "tokens_per_second": 1425329.2891000668, "eta_seconds": 15658.427875351905, "data_wait_seconds": 2.54213809967041e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 29920, "total_steps": 72479, "tokens": 15686696960, "loss": 3.7765032947063446, "l_mix": 2.3381745368242264, "l_a": 2.430110216140747, "l_c": 2.3643186688423157, "z_loss": 26.4562029838562, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.8682329058647156, "tokens_per_second": 1350221.0836554733, "eta_seconds": 16525.5699693203, "data_wait_seconds": 2.855062484741211e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 29930, "total_steps": 72479, "tokens": 15691939840, "loss": 3.7376159727573395, "l_mix": 2.315041244029999, "l_a": 2.379667893052101, "l_c": 2.3622477054595947, "z_loss": 27.24577569961548, "lr": 0.0023, "wd": 0.01, "grad_norm": 1.789169430732727, "tokens_per_second": 1415255.2967747387, "eta_seconds": 15762.477740120888, "data_wait_seconds": 1.8596649169921875e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08236}}
{"step": 29940, "total_steps": 72479, "tokens": 15697182720, "loss": 4.26030820608139, "l_mix": 2.634481370449066, "l_a": 2.7402679920196533, "l_c": 2.6791545748710632, "z_loss": 23.939191818237305, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19592799246311188, "tokens_per_second": 1373601.9457043817, "eta_seconds": 16236.645049715044, "data_wait_seconds": 2.9146671295166016e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 29950, "total_steps": 72479, "tokens": 15702425600, "loss": 4.416781187057495, "l_mix": 2.7348697185516357, "l_a": 2.826678544282913, "l_c": 2.7796925008296967, "z_loss": 22.329230546951294, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.24008141458034515, "tokens_per_second": 1436068.2158180687, "eta_seconds": 15526.730629086493, "data_wait_seconds": 1.6480684280395508e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 29960, "total_steps": 72479, "tokens": 15707668480, "loss": 4.24744987487793, "l_mix": 2.628270983695984, "l_a": 2.728782504796982, "l_c": 2.66848024725914, "z_loss": 22.950843572616577, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17038299143314362, "tokens_per_second": 698481.153247209, "eta_seconds": 31915.251210951803, "data_wait_seconds": 1.7315149307250977e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 29970, "total_steps": 72479, "tokens": 15712911360, "loss": 4.462736129760742, "l_mix": 2.763623893260956, "l_a": 2.8590800762176514, "l_c": 2.804626911878586, "z_loss": 22.61253023147583, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1557171493768692, "tokens_per_second": 1467616.9673360905, "eta_seconds": 15185.814206314086, "data_wait_seconds": 2.6702880859375e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 29980, "total_steps": 72479, "tokens": 15718154240, "loss": 4.373855948448181, "l_mix": 2.7080070674419403, "l_a": 2.8046644628047943, "l_c": 2.7481649219989777, "z_loss": 22.6136155128479, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19196341931819916, "tokens_per_second": 1358423.6420490865, "eta_seconds": 16402.626560878754, "data_wait_seconds": 2.9653310775756836e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20764, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 29990, "total_steps": 72479, "tokens": 15723397120, "loss": 4.447041153907776, "l_mix": 2.752894163131714, "l_a": 2.8525944352149963, "l_c": 2.7945623993873596, "z_loss": 23.177717208862305, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1760879009962082, "tokens_per_second": 1369587.1629323752, "eta_seconds": 16265.10048787594, "data_wait_seconds": 2.5510787963867188e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 30000, "total_steps": 72479, "tokens": 15728640000, "loss": 4.4049859046936035, "l_mix": 2.7271120250225067, "l_a": 2.824602335691452, "l_c": 2.7683102786540985, "z_loss": 23.195966243743896, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15217256546020508, "tokens_per_second": 1441613.6931477396, "eta_seconds": 15448.81965110302, "data_wait_seconds": 2.1696090698242188e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.31021154, "gate_a2b/L1_absmax": 0.63717002, "gate_a2b/L1_rms": 0.32422903, "gate_a2b/L3_absmean": 0.32915449, "gate_a2b/L3_absmax": 0.59839439, "gate_a2b/L3_rms": 0.33421186, "gate_a2b/L5_absmean": 0.50436604, "gate_a2b/L5_absmax": 0.66684341, "gate_a2b/L5_rms": 0.50993073, "gate_c2b/L1_absmean": 0.01509819, "gate_c2b/L1_absmax": 0.48687619, "gate_c2b/L1_rms": 0.06009381, "gate_c2b/L3_absmean": 0.21711989, "gate_c2b/L3_absmax": 0.45396933, "gate_c2b/L3_rms": 0.22120462, "gate_c2b/L5_absmean": 0.55783415, "gate_c2b/L5_absmax": 0.75982577, "gate_c2b/L5_rms": 0.57053548, "gate_b2a/L7_absmean": 0.08247182, "gate_b2a/L7_absmax": 0.32237288, "gate_b2a/L7_rms": 0.08677799, "gate_b2a/L11_absmean": 0.12560597, "gate_b2a/L11_absmax": 0.55569363, "gate_b2a/L11_rms": 0.1302506, "gate_b2a/L14_absmean": 0.31113863, "gate_b2a/L14_absmax": 0.63762343, "gate_b2a/L14_rms": 0.31432205, "gate_b2c/L3_absmean": 0.08754075, "gate_b2c/L3_absmax": 0.69101113, "gate_b2c/L3_rms": 0.10204112, "gate_b2c/L5_absmean": 0.10307841, "gate_b2c/L5_absmax": 0.82968092, "gate_b2c/L5_rms": 0.11389287, "gate_b2c/L6_absmean": 0.150313, "gate_b2c/L6_absmax": 0.5592044, "gate_b2c/L6_rms": 0.15788789}, "streams": {"cos_a2b/round0": 0.407411, "cos_c2b/round0": 0.304021, "cos_b2a/round0": 0.116477, "cos_b2c/round0": 0.149231, "rms_a/round0": 1.343113, "rms_b/round0": 2.476777, "rms_c/round0": 1.644134, "cos_a2b/round1": 0.426578, "cos_c2b/round1": 0.284203, "cos_b2a/round1": 0.240011, "cos_b2c/round1": 0.124423, "rms_a/round1": 2.352774, "rms_b/round1": 5.116048, "rms_c/round1": 2.928182, "cos_a2b/round2": 0.554381, "cos_c2b/round2": 0.513882, "cos_b2a/round2": 0.084732, "cos_b2c/round2": 0.161606, "rms_a/round2": 5.521172, "rms_b/round2": 11.885221, "rms_c/round2": 4.922329}, "alpha_mean": 0.4373816834015827, "alpha_hist": [0.01042, 0.06456, 0.1651, 0.20656, 0.19437, 0.16415, 0.11531, 0.06874, 0.0107, 9e-05], "alpha_by_type": {"special": {"share": 0.00121, "mean": 0.53833, "std": 0.09539}, "whitespace": {"share": 0.02855, "mean": 0.53031, "std": 0.10598}, "digit": {"share": 0.04723, "mean": 0.49209, "std": 0.10756}, "word_start": {"share": 0.56351, "mean": 0.45488, "std": 0.16083}, "word_cont": {"share": 0.25415, "mean": 0.31482, "std": 0.13352}, "punct": {"share": 0.10536, "mean": 0.58857, "std": 0.15421}}, "alpha_p05": 0.1833, "alpha_p50": 0.41822, "alpha_p95": 0.7345}
{"step": 30010, "total_steps": 72479, "tokens": 15733882880, "loss": 4.781853497028351, "l_mix": 2.9639312028884888, "l_a": 3.046213537454605, "l_c": 3.0135273039340973, "z_loss": 22.687708854675293, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19558054208755493, "tokens_per_second": 1442296.8360788936, "eta_seconds": 15437.86723718643, "data_wait_seconds": 1.8358230590820312e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 30020, "total_steps": 72479, "tokens": 15739125760, "loss": 3.3683811128139496, "l_mix": 2.0723521262407303, "l_a": 2.1852201521396637, "l_c": 2.1348764896392822, "z_loss": 23.264519691467285, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2775704860687256, "tokens_per_second": 1403610.5124477362, "eta_seconds": 15859.630570292473, "data_wait_seconds": 2.03549861907959e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 30030, "total_steps": 72479, "tokens": 15744368640, "loss": 3.4018013775348663, "l_mix": 2.094474494457245, "l_a": 2.210097372531891, "l_c": 2.1476586759090424, "z_loss": 22.927870273590088, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.24094747006893158, "tokens_per_second": 1461842.6251996136, "eta_seconds": 15224.279911088943, "data_wait_seconds": 1.9431114196777344e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 30040, "total_steps": 72479, "tokens": 15749611520, "loss": 3.109047055244446, "l_mix": 1.9231188148260117, "l_a": 2.0015333592891693, "l_c": 1.9515606611967087, "z_loss": 27.021024465560913, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.22594349086284637, "tokens_per_second": 1439670.587704767, "eta_seconds": 15455.103842520715, "data_wait_seconds": 2.2232532501220703e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08236}}
{"step": 30050, "total_steps": 72479, "tokens": 15754854400, "loss": 4.516140401363373, "l_mix": 2.7967394292354584, "l_a": 2.8869518041610718, "l_c": 2.8443843126296997, "z_loss": 22.69053292274475, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2246140092611313, "tokens_per_second": 1427939.0333961472, "eta_seconds": 15578.407083034515, "data_wait_seconds": 1.7464160919189453e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 30060, "total_steps": 72479, "tokens": 15760097280, "loss": 4.609237253665924, "l_mix": 2.855802059173584, "l_a": 2.946846902370453, "l_c": 2.8979365527629852, "z_loss": 22.669992685317993, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14384272694587708, "tokens_per_second": 1478391.3145139718, "eta_seconds": 15043.224654841422, "data_wait_seconds": 1.9311904907226562e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 30070, "total_steps": 72479, "tokens": 15765340160, "loss": 4.506217002868652, "l_mix": 2.789806216955185, "l_a": 2.8864453732967377, "l_c": 2.8349234461784363, "z_loss": 22.77043342590332, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1825922727584839, "tokens_per_second": 1421909.9427452905, "eta_seconds": 15637.087218809127, "data_wait_seconds": 1.773238182067871e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 30080, "total_steps": 72479, "tokens": 15770583040, "loss": 4.503094375133514, "l_mix": 2.7889568209648132, "l_a": 2.8836434185504913, "l_c": 2.830148369073868, "z_loss": 22.85679531097412, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1900872141122818, "tokens_per_second": 1428431.6574471332, "eta_seconds": 15562.023423457145, "data_wait_seconds": 4.5746564865112305e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 30090, "total_steps": 72479, "tokens": 15775825920, "loss": 4.5023751854896545, "l_mix": 2.7882924377918243, "l_a": 2.8822435438632965, "l_c": 2.8313651382923126, "z_loss": 23.727625846862793, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19875295460224152, "tokens_per_second": 1402929.3259987934, "eta_seconds": 15841.171483230592, "data_wait_seconds": 1.9341707229614258e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20764, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 30100, "total_steps": 72479, "tokens": 15781068800, "loss": 4.912175357341766, "l_mix": 3.046626001596451, "l_a": 3.128353387117386, "l_c": 3.0901440382003784, "z_loss": 22.564939737319946, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16860328614711761, "tokens_per_second": 1418580.0773546954, "eta_seconds": 15662.704916477203, "data_wait_seconds": 2.625584602355957e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.30970547, "gate_a2b/L1_absmax": 0.64324826, "gate_a2b/L1_rms": 0.3237339, "gate_a2b/L3_absmean": 0.32712555, "gate_a2b/L3_absmax": 0.59908748, "gate_a2b/L3_rms": 0.33221146, "gate_a2b/L5_absmean": 0.50159281, "gate_a2b/L5_absmax": 0.66559947, "gate_a2b/L5_rms": 0.50715137, "gate_c2b/L1_absmean": 0.0158035, "gate_c2b/L1_absmax": 0.4918808, "gate_c2b/L1_rms": 0.06029367, "gate_c2b/L3_absmean": 0.21737619, "gate_c2b/L3_absmax": 0.45461798, "gate_c2b/L3_rms": 0.22152811, "gate_c2b/L5_absmean": 0.56059182, "gate_c2b/L5_absmax": 0.76370364, "gate_c2b/L5_rms": 0.57342356, "gate_b2a/L7_absmean": 0.08140131, "gate_b2a/L7_absmax": 0.32340622, "gate_b2a/L7_rms": 0.08575745, "gate_b2a/L11_absmean": 0.12437896, "gate_b2a/L11_absmax": 0.55762559, "gate_b2a/L11_rms": 0.12913299, "gate_b2a/L14_absmean": 0.30873385, "gate_b2a/L14_absmax": 0.62698567, "gate_b2a/L14_rms": 0.31192166, "gate_b2c/L3_absmean": 0.08570969, "gate_b2c/L3_absmax": 0.69046229, "gate_b2c/L3_rms": 0.10024031, "gate_b2c/L5_absmean": 0.1005798, "gate_b2c/L5_absmax": 0.8315984, "gate_b2c/L5_rms": 0.11169134, "gate_b2c/L6_absmean": 0.1476344, "gate_b2c/L6_absmax": 0.55569166, "gate_b2c/L6_rms": 0.15520474}, "streams": {"cos_a2b/round0": 0.404002, "cos_c2b/round0": 0.313187, "cos_b2a/round0": 0.126161, "cos_b2c/round0": 0.156215, "rms_a/round0": 1.363952, "rms_b/round0": 2.517491, "rms_c/round0": 1.640285, "cos_a2b/round1": 0.43212, "cos_c2b/round1": 0.291088, "cos_b2a/round1": 0.245944, "cos_b2c/round1": 0.132185, "rms_a/round1": 2.394343, "rms_b/round1": 5.105982, "rms_c/round1": 2.964941, "cos_a2b/round2": 0.575237, "cos_c2b/round2": 0.510231, "cos_b2a/round2": 0.098259, "cos_b2c/round2": 0.169691, "rms_a/round2": 5.63778, "rms_b/round2": 11.85744, "rms_c/round2": 4.980551}, "alpha_mean": 0.46963886793240306, "alpha_hist": [0.00703, 0.04062, 0.12485, 0.17671, 0.20816, 0.20982, 0.14638, 0.07521, 0.01112, 9e-05], "alpha_by_type": {"special": {"share": 0.00075, "mean": 0.52861, "std": 0.15201}, "whitespace": {"share": 0.07571, "mean": 0.50856, "std": 0.10207}, "digit": {"share": 0.02667, "mean": 0.44939, "std": 0.13082}, "word_start": {"share": 0.55675, "mean": 0.49607, "std": 0.15536}, "word_cont": {"share": 0.23738, "mean": 0.34251, "std": 0.12703}, "punct": {"share": 0.10274, "mean": 0.59632, "std": 0.151}}, "alpha_p05": 0.20947, "alpha_p50": 0.4677, "alpha_p95": 0.73087}
{"step": 30110, "total_steps": 72479, "tokens": 15786311680, "loss": 4.830665230751038, "l_mix": 2.995579481124878, "l_a": 3.0791294872760773, "l_c": 3.037822663784027, "z_loss": 22.713146448135376, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16429395973682404, "tokens_per_second": 1374138.9261333465, "eta_seconds": 16165.438478994369, "data_wait_seconds": 2.828240394592285e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 30120, "total_steps": 72479, "tokens": 15791554560, "loss": 4.903165876865387, "l_mix": 3.0405379831790924, "l_a": 3.124373495578766, "l_c": 3.0843857526779175, "z_loss": 22.86977767944336, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14816175401210785, "tokens_per_second": 1380563.0676176192, "eta_seconds": 16086.418587398528, "data_wait_seconds": 1.6033649444580078e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 30130, "total_steps": 72479, "tokens": 15796797440, "loss": 4.820996046066284, "l_mix": 2.9880156815052032, "l_a": 3.0756334960460663, "l_c": 3.0343008935451508, "z_loss": 22.99250555038452, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1892750859260559, "tokens_per_second": 1417974.343867845, "eta_seconds": 15658.303415727616, "data_wait_seconds": 3.463029861450195e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 30140, "total_steps": 72479, "tokens": 15802040320, "loss": 4.879319787025452, "l_mix": 3.025330066680908, "l_a": 3.1098028123378754, "l_c": 3.0701625049114227, "z_loss": 23.41035556793213, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15545813739299774, "tokens_per_second": 1417352.226835318, "eta_seconds": 15661.477233195306, "data_wait_seconds": 2.1189451217651367e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 30150, "total_steps": 72479, "tokens": 15807283200, "loss": 3.1855381429195404, "l_mix": 1.971118912100792, "l_a": 2.0478761047124863, "l_c": 2.000187948346138, "z_loss": 26.867836475372314, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.23201380670070648, "tokens_per_second": 1415119.6864246347, "eta_seconds": 15682.480404233933, "data_wait_seconds": 1.749396324157715e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 30160, "total_steps": 72479, "tokens": 15812526080, "loss": 4.4547483921051025, "l_mix": 2.7576677203178406, "l_a": 2.852607846260071, "l_c": 2.804327517747879, "z_loss": 23.355154752731323, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2101757526397705, "tokens_per_second": 1407689.9358224366, "eta_seconds": 15761.527668404578, "data_wait_seconds": 2.4646520614624023e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 30170, "total_steps": 72479, "tokens": 15817768960, "loss": 4.668116450309753, "l_mix": 2.892004519701004, "l_a": 2.986393094062805, "l_c": 2.9339798986911774, "z_loss": 22.718907594680786, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1943642646074295, "tokens_per_second": 1349290.0413704272, "eta_seconds": 16439.831549835206, "data_wait_seconds": 1.77919864654541e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 30180, "total_steps": 72479, "tokens": 15823011840, "loss": 4.472572088241577, "l_mix": 2.7698074877262115, "l_a": 2.869588851928711, "l_c": 2.8062931299209595, "z_loss": 23.422236680984497, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.23279759287834167, "tokens_per_second": 1348212.2248460685, "eta_seconds": 16449.08546540737, "data_wait_seconds": 2.4437904357910156e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 30190, "total_steps": 72479, "tokens": 15828254720, "loss": 4.436785638332367, "l_mix": 2.746450513601303, "l_a": 2.841136783361435, "l_c": 2.7933134138584137, "z_loss": 23.240678310394287, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.22946517169475555, "tokens_per_second": 1424973.6962953731, "eta_seconds": 15559.315438342095, "data_wait_seconds": 1.5676021575927734e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 30200, "total_steps": 72479, "tokens": 15833497600, "loss": 4.577184200286865, "l_mix": 2.8355481326580048, "l_a": 2.9332617819309235, "l_c": 2.872191607952118, "z_loss": 23.498958587646484, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.22124478220939636, "tokens_per_second": 1432229.4963288354, "eta_seconds": 15476.82994158268, "data_wait_seconds": 1.6450881958007812e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.30674773, "gate_a2b/L1_absmax": 0.64020985, "gate_a2b/L1_rms": 0.3208997, "gate_a2b/L3_absmean": 0.3245565, "gate_a2b/L3_absmax": 0.60056204, "gate_a2b/L3_rms": 0.32973838, "gate_a2b/L5_absmean": 0.49921927, "gate_a2b/L5_absmax": 0.6662851, "gate_a2b/L5_rms": 0.5047996, "gate_c2b/L1_absmean": 0.01630362, "gate_c2b/L1_absmax": 0.50001746, "gate_c2b/L1_rms": 0.06130677, "gate_c2b/L3_absmean": 0.21787834, "gate_c2b/L3_absmax": 0.45712066, "gate_c2b/L3_rms": 0.22201899, "gate_c2b/L5_absmean": 0.56251353, "gate_c2b/L5_absmax": 0.77009475, "gate_c2b/L5_rms": 0.57554525, "gate_b2a/L7_absmean": 0.08083402, "gate_b2a/L7_absmax": 0.31535918, "gate_b2a/L7_rms": 0.08509484, "gate_b2a/L11_absmean": 0.12349325, "gate_b2a/L11_absmax": 0.55539483, "gate_b2a/L11_rms": 0.12816696, "gate_b2a/L14_absmean": 0.30675828, "gate_b2a/L14_absmax": 0.63129288, "gate_b2a/L14_rms": 0.31015414, "gate_b2c/L3_absmean": 0.08429994, "gate_b2c/L3_absmax": 0.6938172, "gate_b2c/L3_rms": 0.09889258, "gate_b2c/L5_absmean": 0.09886795, "gate_b2c/L5_absmax": 0.8208065, "gate_b2c/L5_rms": 0.10972242, "gate_b2c/L6_absmean": 0.14442906, "gate_b2c/L6_absmax": 0.56780356, "gate_b2c/L6_rms": 0.15218762}, "streams": {"cos_a2b/round0": 0.402463, "cos_c2b/round0": 0.29243, "cos_b2a/round0": 0.105792, "cos_b2c/round0": 0.142266, "rms_a/round0": 1.431039, "rms_b/round0": 2.646872, "rms_c/round0": 1.718387, "cos_a2b/round1": 0.421914, "cos_c2b/round1": 0.271154, "cos_b2a/round1": 0.241294, "cos_b2c/round1": 0.108486, "rms_a/round1": 2.509027, "rms_b/round1": 5.526782, "rms_c/round1": 3.077793, "cos_a2b/round2": 0.539692, "cos_c2b/round2": 0.51523, "cos_b2a/round2": 0.078809, "cos_b2c/round2": 0.149078, "rms_a/round2": 5.948142, "rms_b/round2": 13.039686, "rms_c/round2": 5.190444}, "alpha_mean": 0.44074852264968456, "alpha_hist": [0.00676, 0.05042, 0.16673, 0.21951, 0.20116, 0.15683, 0.12296, 0.06668, 0.00888, 8e-05], "alpha_by_type": {"special": {"share": 0.00076, "mean": 0.55808, "std": 0.11351}, "whitespace": {"share": 0.02769, "mean": 0.52234, "std": 0.11391}, "digit": {"share": 0.06618, "mean": 0.49891, "std": 0.11542}, "word_start": {"share": 0.49008, "mean": 0.46086, "std": 0.1603}, "word_cont": {"share": 0.31021, "mean": 0.3363, "std": 0.12312}, "punct": {"share": 0.10507, "mean": 0.59631, "std": 0.15211}}, "alpha_p05": 0.20057, "alpha_p50": 0.43063, "alpha_p95": 0.72623}
{"step": 30210, "total_steps": 72479, "tokens": 15838740480, "loss": 3.2471741139888763, "l_mix": 2.007591590285301, "l_a": 2.088973641395569, "l_c": 2.0429679304361343, "z_loss": 28.108362674713135, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2631170451641083, "tokens_per_second": 1449120.4976434084, "eta_seconds": 15292.813474130631, "data_wait_seconds": 2.4706125259399414e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08236}}
{"step": 30220, "total_steps": 72479, "tokens": 15843983360, "loss": 4.8042988777160645, "l_mix": 2.9794729351997375, "l_a": 3.060557395219803, "l_c": 3.02219557762146, "z_loss": 22.59121608734131, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18489989638328552, "tokens_per_second": 1461164.243590713, "eta_seconds": 15163.173263502122, "data_wait_seconds": 3.382563591003418e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 30230, "total_steps": 72479, "tokens": 15849226240, "loss": 4.823284327983856, "l_mix": 2.9913939237594604, "l_a": 3.070765107870102, "l_c": 3.0355360209941864, "z_loss": 23.742753267288208, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16956928372383118, "tokens_per_second": 1279707.570738594, "eta_seconds": 17309.14485347271, "data_wait_seconds": 2.7239322662353516e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 30240, "total_steps": 72479, "tokens": 15854469120, "loss": 4.778807103633881, "l_mix": 2.962997794151306, "l_a": 3.044135570526123, "l_c": 3.008561819791794, "z_loss": 23.254578828811646, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17825978994369507, "tokens_per_second": 1372431.0755751543, "eta_seconds": 16135.892888259887, "data_wait_seconds": 2.524256706237793e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 30250, "total_steps": 72479, "tokens": 15859712000, "loss": 4.896024107933044, "l_mix": 3.035647362470627, "l_a": 3.122521996498108, "l_c": 3.078733444213867, "z_loss": 23.9862322807312, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1856723576784134, "tokens_per_second": 1438909.6122084726, "eta_seconds": 15386.760755610467, "data_wait_seconds": 2.7000904083251953e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 30260, "total_steps": 72479, "tokens": 15864954880, "loss": 3.3644249737262726, "l_mix": 2.082775428891182, "l_a": 2.135259747505188, "l_c": 2.136905238032341, "z_loss": 25.670615434646606, "lr": 0.0023, "wd": 0.01, "grad_norm": 13.545036315917969, "tokens_per_second": 1442536.0181179033, "eta_seconds": 15344.445333766938, "data_wait_seconds": 2.0951032638549805e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 30270, "total_steps": 72479, "tokens": 15870197760, "loss": 4.530229926109314, "l_mix": 2.805913418531418, "l_a": 2.9017149806022644, "l_c": 2.846006602048874, "z_loss": 23.882290363311768, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14473214745521545, "tokens_per_second": 1467346.778521927, "eta_seconds": 15081.419413542746, "data_wait_seconds": 1.96993350982666e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 30280, "total_steps": 72479, "tokens": 15875440640, "loss": 4.558854162693024, "l_mix": 2.8233742713928223, "l_a": 2.919896125793457, "l_c": 2.865036725997925, "z_loss": 23.912142992019653, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14734745025634766, "tokens_per_second": 1405535.5884379654, "eta_seconds": 15740.924309563638, "data_wait_seconds": 1.9848346710205078e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 30290, "total_steps": 72479, "tokens": 15880683520, "loss": 4.549152970314026, "l_mix": 2.8166705667972565, "l_a": 2.9128951728343964, "l_c": 2.8620463013648987, "z_loss": 24.12988257408142, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17899788916110992, "tokens_per_second": 1300695.53976283, "eta_seconds": 17005.66024546623, "data_wait_seconds": 2.1517276763916016e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 30300, "total_steps": 72479, "tokens": 15885926400, "loss": 4.5857062339782715, "l_mix": 2.8410635590553284, "l_a": 2.9332945346832275, "l_c": 2.882180869579315, "z_loss": 23.89826464653015, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18239404261112213, "tokens_per_second": 1440973.4033144922, "eta_seconds": 15346.531380200386, "data_wait_seconds": 1.5437602996826172e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.30490759, "gate_a2b/L1_absmax": 0.64042634, "gate_a2b/L1_rms": 0.31901085, "gate_a2b/L3_absmean": 0.32230541, "gate_a2b/L3_absmax": 0.60205472, "gate_a2b/L3_rms": 0.32752886, "gate_a2b/L5_absmean": 0.49643308, "gate_a2b/L5_absmax": 0.66340536, "gate_a2b/L5_rms": 0.50199169, "gate_c2b/L1_absmean": 0.01724484, "gate_c2b/L1_absmax": 0.50451565, "gate_c2b/L1_rms": 0.06169667, "gate_c2b/L3_absmean": 0.21822679, "gate_c2b/L3_absmax": 0.46803617, "gate_c2b/L3_rms": 0.22250417, "gate_c2b/L5_absmean": 0.56301099, "gate_c2b/L5_absmax": 0.77193868, "gate_c2b/L5_rms": 0.57605308, "gate_b2a/L7_absmean": 0.07939821, "gate_b2a/L7_absmax": 0.31811854, "gate_b2a/L7_rms": 0.08378454, "gate_b2a/L11_absmean": 0.12276016, "gate_b2a/L11_absmax": 0.55673623, "gate_b2a/L11_rms": 0.12754169, "gate_b2a/L14_absmean": 0.30480814, "gate_b2a/L14_absmax": 0.64834559, "gate_b2a/L14_rms": 0.30837783, "gate_b2c/L3_absmean": 0.08190347, "gate_b2c/L3_absmax": 0.69310635, "gate_b2c/L3_rms": 0.09664107, "gate_b2c/L5_absmean": 0.09651568, "gate_b2c/L5_absmax": 0.81868225, "gate_b2c/L5_rms": 0.10762959, "gate_b2c/L6_absmean": 0.14134282, "gate_b2c/L6_absmax": 0.55811632, "gate_b2c/L6_rms": 0.149224}, "streams": {"cos_a2b/round0": 0.399553, "cos_c2b/round0": 0.305794, "cos_b2a/round0": 0.124577, "cos_b2c/round0": 0.149655, "rms_a/round0": 1.474097, "rms_b/round0": 2.716375, "rms_c/round0": 1.739068, "cos_a2b/round1": 0.434842, "cos_c2b/round1": 0.285746, "cos_b2a/round1": 0.242897, "cos_b2c/round1": 0.128429, "rms_a/round1": 2.628672, "rms_b/round1": 5.676281, "rms_c/round1": 3.093101, "cos_a2b/round2": 0.566239, "cos_c2b/round2": 0.505228, "cos_b2a/round2": 0.086506, "cos_b2c/round2": 0.162106, "rms_a/round2": 6.202507, "rms_b/round2": 13.239551, "rms_c/round2": 5.19927}, "alpha_mean": 0.4510956633787089, "alpha_hist": [0.01056, 0.06071, 0.15396, 0.19093, 0.18782, 0.17311, 0.12621, 0.0791, 0.01752, 8e-05], "alpha_by_type": {"special": {"share": 0.00121, "mean": 0.57777, "std": 0.08285}, "whitespace": {"share": 0.02943, "mean": 0.54377, "std": 0.12422}, "digit": {"share": 0.03152, "mean": 0.46583, "std": 0.11998}, "word_start": {"share": 0.59383, "mean": 0.46945, "std": 0.16623}, "word_cont": {"share": 0.23903, "mean": 0.32569, "std": 0.13158}, "punct": {"share": 0.10498, "mean": 0.60098, "std": 0.16906}}, "alpha_p05": 0.19193, "alpha_p50": 0.44252, "alpha_p95": 0.74503}
{"step": 30310, "total_steps": 72479, "tokens": 15891169280, "loss": 4.574184417724609, "l_mix": 2.8335598409175873, "l_a": 2.9239553213119507, "l_c": 2.8781261146068573, "z_loss": 22.90342926979065, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.22344104945659637, "tokens_per_second": 1452464.9738057086, "eta_seconds": 15221.503492832184, "data_wait_seconds": 1.8805265426635742e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 30320, "total_steps": 72479, "tokens": 15896412160, "loss": 3.1538757979869843, "l_mix": 1.951201930642128, "l_a": 2.0261669903993607, "l_c": 1.9827457666397095, "z_loss": 26.541317462921143, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2465152144432068, "tokens_per_second": 1393891.3819263198, "eta_seconds": 15857.374597907066, "data_wait_seconds": 3.6329030990600586e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 30330, "total_steps": 72479, "tokens": 15901655040, "loss": 2.9825279712677, "l_mix": 1.8447229862213135, "l_a": 1.9199447184801102, "l_c": 1.8727383613586426, "z_loss": 27.53667974472046, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.26274004578590393, "tokens_per_second": 1422058.6286468643, "eta_seconds": 15539.594828820229, "data_wait_seconds": 2.1129846572875977e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08236}}
{"step": 30340, "total_steps": 72479, "tokens": 15906897920, "loss": 4.881303250789642, "l_mix": 3.0260133743286133, "l_a": 3.1154274344444275, "l_c": 3.0688722133636475, "z_loss": 24.195474863052368, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19790680706501007, "tokens_per_second": 1442615.983918971, "eta_seconds": 15314.520481038093, "data_wait_seconds": 2.422928810119629e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 30350, "total_steps": 72479, "tokens": 15912140800, "loss": 4.8480560183525085, "l_mix": 3.0061112344264984, "l_a": 3.086130052804947, "l_c": 3.0536856651306152, "z_loss": 23.76668906211853, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17951688170433044, "tokens_per_second": 1315324.7140212848, "eta_seconds": 16792.605595064164, "data_wait_seconds": 2.2232532501220703e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 30360, "total_steps": 72479, "tokens": 15917383680, "loss": 4.727867543697357, "l_mix": 2.930403172969818, "l_a": 3.0160226225852966, "l_c": 2.9755249321460724, "z_loss": 24.18088436126709, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19126436114311218, "tokens_per_second": 1374542.7103232816, "eta_seconds": 16065.332933020592, "data_wait_seconds": 2.5510787963867188e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 30370, "total_steps": 72479, "tokens": 15922626560, "loss": 4.557506084442139, "l_mix": 2.8223769664764404, "l_a": 2.918799579143524, "l_c": 2.8649639189243317, "z_loss": 24.254196405410767, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16279855370521545, "tokens_per_second": 1421468.941427308, "eta_seconds": 15531.287915325163, "data_wait_seconds": 1.65402889251709e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 30380, "total_steps": 72479, "tokens": 15927869440, "loss": 4.545850992202759, "l_mix": 2.8153184354305267, "l_a": 2.9104440212249756, "l_c": 2.8579977452754974, "z_loss": 24.295283555984497, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13228128850460052, "tokens_per_second": 1410694.1000585314, "eta_seconds": 15646.198925113677, "data_wait_seconds": 2.0712614059448242e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08232}}
{"step": 30390, "total_steps": 72479, "tokens": 15933112320, "loss": 4.504987418651581, "l_mix": 2.789846509695053, "l_a": 2.8850899934768677, "l_c": 2.8320461213588715, "z_loss": 24.2567195892334, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13632169365882874, "tokens_per_second": 1479271.949971942, "eta_seconds": 14917.3095808506, "data_wait_seconds": 1.519918441772461e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 30400, "total_steps": 72479, "tokens": 15938355200, "loss": 4.484853982925415, "l_mix": 2.776686906814575, "l_a": 2.8729555308818817, "l_c": 2.820934444665909, "z_loss": 24.776190996170044, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14282245934009552, "tokens_per_second": 1437907.2919581358, "eta_seconds": 15342.79356908798, "data_wait_seconds": 1.6301870346069336e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.30506516, "gate_a2b/L1_absmax": 0.64623672, "gate_a2b/L1_rms": 0.31905499, "gate_a2b/L3_absmean": 0.32076764, "gate_a2b/L3_absmax": 0.60132599, "gate_a2b/L3_rms": 0.32593352, "gate_a2b/L5_absmean": 0.49404851, "gate_a2b/L5_absmax": 0.66023487, "gate_a2b/L5_rms": 0.49959356, "gate_c2b/L1_absmean": 0.01571664, "gate_c2b/L1_absmax": 0.50911456, "gate_c2b/L1_rms": 0.06172799, "gate_c2b/L3_absmean": 0.21765962, "gate_c2b/L3_absmax": 0.46882629, "gate_c2b/L3_rms": 0.22191891, "gate_c2b/L5_absmean": 0.56496686, "gate_c2b/L5_absmax": 0.77362955, "gate_c2b/L5_rms": 0.57788396, "gate_b2a/L7_absmean": 0.07896769, "gate_b2a/L7_absmax": 0.31968328, "gate_b2a/L7_rms": 0.08341233, "gate_b2a/L11_absmean": 0.12100282, "gate_b2a/L11_absmax": 0.55510646, "gate_b2a/L11_rms": 0.12587744, "gate_b2a/L14_absmean": 0.30269182, "gate_b2a/L14_absmax": 0.64645386, "gate_b2a/L14_rms": 0.30633333, "gate_b2c/L3_absmean": 0.08070534, "gate_b2c/L3_absmax": 0.69118708, "gate_b2c/L3_rms": 0.09513185, "gate_b2c/L5_absmean": 0.09451964, "gate_b2c/L5_absmax": 0.81452423, "gate_b2c/L5_rms": 0.10559848, "gate_b2c/L6_absmean": 0.13888104, "gate_b2c/L6_absmax": 0.56010324, "gate_b2c/L6_rms": 0.14692029}, "streams": {"cos_a2b/round0": 0.407479, "cos_c2b/round0": 0.30255, "cos_b2a/round0": 0.114732, "cos_b2c/round0": 0.148112, "rms_a/round0": 1.512778, "rms_b/round0": 2.804125, "rms_c/round0": 1.765953, "cos_a2b/round1": 0.43261, "cos_c2b/round1": 0.283135, "cos_b2a/round1": 0.236918, "cos_b2c/round1": 0.121916, "rms_a/round1": 2.696975, "rms_b/round1": 5.850741, "rms_c/round1": 3.178769, "cos_a2b/round2": 0.558247, "cos_c2b/round2": 0.510582, "cos_b2a/round2": 0.087664, "cos_b2c/round2": 0.151094, "rms_a/round2": 6.342427, "rms_b/round2": 13.749123, "rms_c/round2": 5.403176}, "alpha_mean": 0.4407681321529964, "alpha_hist": [0.01311, 0.06792, 0.16373, 0.19002, 0.18591, 0.17212, 0.12344, 0.07188, 0.01181, 6e-05], "alpha_by_type": {"special": {"share": 0.00084, "mean": 0.52402, "std": 0.11878}, "whitespace": {"share": 0.02617, "mean": 0.52573, "std": 0.10736}, "digit": {"share": 0.03868, "mean": 0.4397, "std": 0.10892}, "word_start": {"share": 0.56801, "mean": 0.46666, "std": 0.16484}, "word_cont": {"share": 0.25833, "mean": 0.31321, "std": 0.13295}, "punct": {"share": 0.10797, "mean": 0.58888, "std": 0.16428}}, "alpha_p05": 0.15304, "alpha_p50": 0.43842, "alpha_p95": 0.74909}
{"step": 30410, "total_steps": 72479, "tokens": 15943598080, "loss": 4.547188937664032, "l_mix": 2.8171278834342957, "l_a": 2.909215271472931, "l_c": 2.857654720544815, "z_loss": 24.500598669052124, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20374813675880432, "tokens_per_second": 1430097.3688645798, "eta_seconds": 15422.916195917129, "data_wait_seconds": 3.337860107421875e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 30420, "total_steps": 72479, "tokens": 15948840960, "loss": 4.587217688560486, "l_mix": 2.841656118631363, "l_a": 2.9307200014591217, "l_c": 2.8878182768821716, "z_loss": 23.69618797302246, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19287866353988647, "tokens_per_second": 1481464.0128993124, "eta_seconds": 14884.620078516005, "data_wait_seconds": 2.5600194931030273e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 30430, "total_steps": 72479, "tokens": 15954083840, "loss": 3.2791928350925446, "l_mix": 2.0290375500917435, "l_a": 2.108692154288292, "l_c": 2.0584922581911087, "z_loss": 28.700606107711792, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2493542730808258, "tokens_per_second": 1427221.4366470634, "eta_seconds": 15446.647272753715, "data_wait_seconds": 1.773238182067871e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 30440, "total_steps": 72479, "tokens": 15959326720, "loss": 3.130047529935837, "l_mix": 1.9354179054498672, "l_a": 2.0162866711616516, "l_c": 1.9658121317625046, "z_loss": 28.284689903259277, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2955740988254547, "tokens_per_second": 1393962.0690083164, "eta_seconds": 15811.436854720116, "data_wait_seconds": 1.8477439880371094e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08236}}
{"step": 30450, "total_steps": 72479, "tokens": 15964569600, "loss": 4.550006985664368, "l_mix": 2.818045526742935, "l_a": 2.9126713573932648, "l_c": 2.86053329706192, "z_loss": 23.913349390029907, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16958162188529968, "tokens_per_second": 1429092.888204526, "eta_seconds": 15419.081946229935, "data_wait_seconds": 1.609325408935547e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 30460, "total_steps": 72479, "tokens": 15969812480, "loss": 4.56425267457962, "l_mix": 2.827239841222763, "l_a": 2.92117041349411, "l_c": 2.8688719868659973, "z_loss": 24.439181566238403, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19461417198181152, "tokens_per_second": 1449810.5841857337, "eta_seconds": 15195.128047966957, "data_wait_seconds": 2.0235776901245117e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 30470, "total_steps": 72479, "tokens": 15975055360, "loss": 4.560060501098633, "l_mix": 2.823838323354721, "l_a": 2.918616682291031, "l_c": 2.868790626525879, "z_loss": 24.657301902770996, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1584147810935974, "tokens_per_second": 1350301.5890664922, "eta_seconds": 16311.033601927757, "data_wait_seconds": 1.6391277313232422e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 30480, "total_steps": 72479, "tokens": 15980298240, "loss": 4.452357113361359, "l_mix": 2.756771057844162, "l_a": 2.8528428971767426, "l_c": 2.799110382795334, "z_loss": 24.428224086761475, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15905778110027313, "tokens_per_second": 1265735.5515293682, "eta_seconds": 17396.660531020167, "data_wait_seconds": 2.0116567611694336e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 30490, "total_steps": 72479, "tokens": 15985541120, "loss": 4.4845805168151855, "l_mix": 2.7772717773914337, "l_a": 2.873487502336502, "l_c": 2.817541629076004, "z_loss": 24.915368795394897, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15645179152488708, "tokens_per_second": 1367512.145202757, "eta_seconds": 16098.08652100563, "data_wait_seconds": 1.4990568161010742e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 30500, "total_steps": 72479, "tokens": 15990784000, "loss": 4.48475056886673, "l_mix": 2.7778170704841614, "l_a": 2.870879650115967, "l_c": 2.818898469209671, "z_loss": 24.283323526382446, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12790793180465698, "tokens_per_second": 1428038.06889214, "eta_seconds": 15412.114306640624, "data_wait_seconds": 1.683831214904785e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.30358982, "gate_a2b/L1_absmax": 0.64999938, "gate_a2b/L1_rms": 0.31774065, "gate_a2b/L3_absmean": 0.31857547, "gate_a2b/L3_absmax": 0.60683239, "gate_a2b/L3_rms": 0.32383108, "gate_a2b/L5_absmean": 0.49134895, "gate_a2b/L5_absmax": 0.65814251, "gate_a2b/L5_rms": 0.49690232, "gate_c2b/L1_absmean": 0.01640015, "gate_c2b/L1_absmax": 0.51206756, "gate_c2b/L1_rms": 0.06204632, "gate_c2b/L3_absmean": 0.21789804, "gate_c2b/L3_absmax": 0.46380341, "gate_c2b/L3_rms": 0.22216323, "gate_c2b/L5_absmean": 0.56703299, "gate_c2b/L5_absmax": 0.77905262, "gate_c2b/L5_rms": 0.58015537, "gate_b2a/L7_absmean": 0.07800542, "gate_b2a/L7_absmax": 0.32226723, "gate_b2a/L7_rms": 0.08254685, "gate_b2a/L11_absmean": 0.12003894, "gate_b2a/L11_absmax": 0.56101412, "gate_b2a/L11_rms": 0.12505895, "gate_b2a/L14_absmean": 0.3002578, "gate_b2a/L14_absmax": 0.64972812, "gate_b2a/L14_rms": 0.30403858, "gate_b2c/L3_absmean": 0.07940476, "gate_b2c/L3_absmax": 0.68883026, "gate_b2c/L3_rms": 0.09374257, "gate_b2c/L5_absmean": 0.09267345, "gate_b2c/L5_absmax": 0.81255889, "gate_b2c/L5_rms": 0.10391267, "gate_b2c/L6_absmean": 0.13594788, "gate_b2c/L6_absmax": 0.56063306, "gate_b2c/L6_rms": 0.14403792}, "streams": {"cos_a2b/round0": 0.407157, "cos_c2b/round0": 0.309811, "cos_b2a/round0": 0.119889, "cos_b2c/round0": 0.148396, "rms_a/round0": 1.553903, "rms_b/round0": 2.865492, "rms_c/round0": 1.79364, "cos_a2b/round1": 0.427679, "cos_c2b/round1": 0.278773, "cos_b2a/round1": 0.235159, "cos_b2c/round1": 0.123505, "rms_a/round1": 2.775506, "rms_b/round1": 6.0914, "rms_c/round1": 3.253593, "cos_a2b/round2": 0.543911, "cos_c2b/round2": 0.519323, "cos_b2a/round2": 0.081509, "cos_b2c/round2": 0.153873, "rms_a/round2": 6.554888, "rms_b/round2": 14.432005, "rms_c/round2": 5.560673}, "alpha_mean": 0.42187290083616347, "alpha_hist": [0.00891, 0.0714, 0.17961, 0.21849, 0.20116, 0.15729, 0.10722, 0.05072, 0.00516, 5e-05], "alpha_by_type": {"special": {"share": 0.00093, "mean": 0.54956, "std": 0.11882}, "whitespace": {"share": 0.03873, "mean": 0.52606, "std": 0.11895}, "digit": {"share": 0.05937, "mean": 0.43617, "std": 0.1078}, "word_start": {"share": 0.52617, "mean": 0.43534, "std": 0.15741}, "word_cont": {"share": 0.2599, "mean": 0.31232, "std": 0.12164}, "punct": {"share": 0.1149, "mean": 0.56449, "std": 0.15196}}, "alpha_p05": 0.16668, "alpha_p50": 0.39817, "alpha_p95": 0.69637}
{"step": 30510, "total_steps": 72479, "tokens": 15996026880, "loss": 4.420552670955658, "l_mix": 2.73759725689888, "l_a": 2.8320012986660004, "l_c": 2.777849704027176, "z_loss": 24.794503450393677, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14372217655181885, "tokens_per_second": 1429268.533287663, "eta_seconds": 15395.177714705467, "data_wait_seconds": 1.5527009963989258e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 30520, "total_steps": 72479, "tokens": 16001269760, "loss": 4.352243542671204, "l_mix": 2.6947284936904907, "l_a": 2.7924218475818634, "l_c": 2.7326281368732452, "z_loss": 24.476153135299683, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1562664806842804, "tokens_per_second": 1410180.2623228186, "eta_seconds": 15599.849735355378, "data_wait_seconds": 2.130866050720215e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 30530, "total_steps": 72479, "tokens": 16006512640, "loss": 4.705793678760529, "l_mix": 2.9186882376670837, "l_a": 2.9956164062023163, "l_c": 2.961401343345642, "z_loss": 23.765095949172974, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1800689399242401, "tokens_per_second": 1378236.9693597783, "eta_seconds": 15957.602212786674, "data_wait_seconds": 2.6226043701171875e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 30540, "total_steps": 72479, "tokens": 16011755520, "loss": 4.836941301822662, "l_mix": 2.999634712934494, "l_a": 3.080203652381897, "l_c": 3.0441512763500214, "z_loss": 24.502371549606323, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16434374451637268, "tokens_per_second": 1322456.9002094439, "eta_seconds": 16626.715342116357, "data_wait_seconds": 2.0325183868408203e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 30550, "total_steps": 72479, "tokens": 16016998400, "loss": 2.9696087539196014, "l_mix": 1.8365080654621124, "l_a": 1.9047063291072845, "l_c": 1.8722959458827972, "z_loss": 28.268871307373047, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.183406800031662, "tokens_per_second": 1474416.4336570243, "eta_seconds": 14909.540513920785, "data_wait_seconds": 4.082918167114258e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08236}}
{"step": 30560, "total_steps": 72479, "tokens": 16022241280, "loss": 4.458764135837555, "l_mix": 2.7605609595775604, "l_a": 2.8589151799678802, "l_c": 2.801762044429779, "z_loss": 24.931047677993774, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15292611718177795, "tokens_per_second": 1300838.3459852673, "eta_seconds": 16894.972953271867, "data_wait_seconds": 1.531839370727539e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 30570, "total_steps": 72479, "tokens": 16027484160, "loss": 4.382860481739044, "l_mix": 2.714248478412628, "l_a": 2.8061254918575287, "l_c": 2.755914121866226, "z_loss": 24.84708571434021, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1353798508644104, "tokens_per_second": 1359222.3127377378, "eta_seconds": 16165.409871578217, "data_wait_seconds": 1.990795135498047e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 30580, "total_steps": 72479, "tokens": 16032727040, "loss": 4.513908743858337, "l_mix": 2.7955477833747864, "l_a": 2.887375980615616, "l_c": 2.8404937386512756, "z_loss": 24.731666803359985, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15678109228610992, "tokens_per_second": 1290481.7651115635, "eta_seconds": 17022.435733604434, "data_wait_seconds": 2.0295381546020508e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 30590, "total_steps": 72479, "tokens": 16037969920, "loss": 4.48406445980072, "l_mix": 2.7769987881183624, "l_a": 2.8717963695526123, "l_c": 2.8184222877025604, "z_loss": 25.110849857330322, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1720566749572754, "tokens_per_second": 1392626.6773735452, "eta_seconds": 15770.127334785462, "data_wait_seconds": 1.9729137420654297e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 30600, "total_steps": 72479, "tokens": 16043212800, "loss": 4.469507873058319, "l_mix": 2.768558591604233, "l_a": 2.859732747077942, "l_c": 2.8100981414318085, "z_loss": 24.406700372695923, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1635037213563919, "tokens_per_second": 1430916.2722754348, "eta_seconds": 15344.473731565475, "data_wait_seconds": 2.390146255493164e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.3027589, "gate_a2b/L1_absmax": 0.65042096, "gate_a2b/L1_rms": 0.31696671, "gate_a2b/L3_absmean": 0.31712404, "gate_a2b/L3_absmax": 0.60725844, "gate_a2b/L3_rms": 0.32235593, "gate_a2b/L5_absmean": 0.48947677, "gate_a2b/L5_absmax": 0.66124451, "gate_a2b/L5_rms": 0.49508944, "gate_c2b/L1_absmean": 0.01609148, "gate_c2b/L1_absmax": 0.51216203, "gate_c2b/L1_rms": 0.06218712, "gate_c2b/L3_absmean": 0.21763746, "gate_c2b/L3_absmax": 0.46932748, "gate_c2b/L3_rms": 0.22188388, "gate_c2b/L5_absmean": 0.56873727, "gate_c2b/L5_absmax": 0.77963793, "gate_c2b/L5_rms": 0.58182275, "gate_b2a/L7_absmean": 0.07710093, "gate_b2a/L7_absmax": 0.32073218, "gate_b2a/L7_rms": 0.08171724, "gate_b2a/L11_absmean": 0.11870823, "gate_b2a/L11_absmax": 0.56183445, "gate_b2a/L11_rms": 0.12385312, "gate_b2a/L14_absmean": 0.297719, "gate_b2a/L14_absmax": 0.65245628, "gate_b2a/L14_rms": 0.30162928, "gate_b2c/L3_absmean": 0.07810187, "gate_b2c/L3_absmax": 0.68862969, "gate_b2c/L3_rms": 0.09245277, "gate_b2c/L5_absmean": 0.0909797, "gate_b2c/L5_absmax": 0.81248599, "gate_b2c/L5_rms": 0.1022744, "gate_b2c/L6_absmean": 0.13359022, "gate_b2c/L6_absmax": 0.55874991, "gate_b2c/L6_rms": 0.1417339}, "streams": {"cos_a2b/round0": 0.404007, "cos_c2b/round0": 0.301753, "cos_b2a/round0": 0.118624, "cos_b2c/round0": 0.141052, "rms_a/round0": 1.593387, "rms_b/round0": 2.956991, "rms_c/round0": 1.832258, "cos_a2b/round1": 0.42938, "cos_c2b/round1": 0.280936, "cos_b2a/round1": 0.240088, "cos_b2c/round1": 0.114251, "rms_a/round1": 2.835925, "rms_b/round1": 6.189176, "rms_c/round1": 3.307784, "cos_a2b/round2": 0.54919, "cos_c2b/round2": 0.516626, "cos_b2a/round2": 0.084547, "cos_b2c/round2": 0.153622, "rms_a/round2": 6.677393, "rms_b/round2": 14.577474, "rms_c/round2": 5.632099}, "alpha_mean": 0.4289530757063247, "alpha_hist": [0.01512, 0.06364, 0.16626, 0.21352, 0.20471, 0.15866, 0.11287, 0.05812, 0.00703, 6e-05], "alpha_by_type": {"special": {"share": 0.00102, "mean": 0.51881, "std": 0.11966}, "whitespace": {"share": 0.02661, "mean": 0.53227, "std": 0.1134}, "digit": {"share": 0.04044, "mean": 0.48519, "std": 0.11555}, "word_start": {"share": 0.52243, "mean": 0.44701, "std": 0.16036}, "word_cont": {"share": 0.26428, "mean": 0.30835, "std": 0.12532}, "punct": {"share": 0.14522, "mean": 0.54827, "std": 0.14696}}, "alpha_p05": 0.16858, "alpha_p50": 0.40498, "alpha_p95": 0.71918}
{"step": 30610, "total_steps": 72479, "tokens": 16048455680, "loss": 4.606125235557556, "l_mix": 2.8571204990148544, "l_a": 2.931663006544113, "l_c": 2.898352563381195, "z_loss": 24.19596242904663, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1638791710138321, "tokens_per_second": 1440704.1565624624, "eta_seconds": 15236.587034201622, "data_wait_seconds": 3.993511199951172e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20768, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 30620, "total_steps": 72479, "tokens": 16053698560, "loss": 4.859876334667206, "l_mix": 3.013763904571533, "l_a": 3.0957741141319275, "l_c": 3.0579337775707245, "z_loss": 24.766822814941406, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1546005755662918, "tokens_per_second": 1376489.7139009845, "eta_seconds": 15943.57819776535, "data_wait_seconds": 1.8775463104248047e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 30630, "total_steps": 72479, "tokens": 16058941440, "loss": 4.930370569229126, "l_mix": 3.058454751968384, "l_a": 3.136093705892563, "l_c": 3.1036257445812225, "z_loss": 24.177980422973633, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17071062326431274, "tokens_per_second": 1470932.7337681379, "eta_seconds": 14916.337102508545, "data_wait_seconds": 2.086162567138672e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 30640, "total_steps": 72479, "tokens": 16064184320, "loss": 4.7541824877262115, "l_mix": 2.947492867708206, "l_a": 3.030478060245514, "l_c": 2.991820275783539, "z_loss": 24.697959661483765, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17621035873889923, "tokens_per_second": 1442044.3996460175, "eta_seconds": 15211.518894553186, "data_wait_seconds": 3.159046173095703e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 30650, "total_steps": 72479, "tokens": 16069427200, "loss": 4.783069610595703, "l_mix": 2.9648796021938324, "l_a": 3.050657629966736, "l_c": 3.0099756717681885, "z_loss": 24.839395761489868, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15991751849651337, "tokens_per_second": 1396002.7990556797, "eta_seconds": 15709.454713726045, "data_wait_seconds": 1.6808509826660156e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 30660, "total_steps": 72479, "tokens": 16074670080, "loss": 2.9165371656417847, "l_mix": 1.8047088235616684, "l_a": 1.8718100488185883, "l_c": 1.8342841416597366, "z_loss": 29.444483280181885, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.24160504341125488, "tokens_per_second": 1410141.1065970114, "eta_seconds": 15548.231144690515, "data_wait_seconds": 1.9758939743041992e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 30670, "total_steps": 72479, "tokens": 16079912960, "loss": 4.466916441917419, "l_mix": 2.7668755054473877, "l_a": 2.8584631979465485, "l_c": 2.8083399534225464, "z_loss": 24.993037462234497, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1263541728258133, "tokens_per_second": 1405465.6091200523, "eta_seconds": 15596.224375581742, "data_wait_seconds": 2.1129846572875977e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 30680, "total_steps": 72479, "tokens": 16085155840, "loss": 4.566112577915192, "l_mix": 2.8291221261024475, "l_a": 2.919847309589386, "l_c": 2.870120346546173, "z_loss": 24.923766613006592, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14922469854354858, "tokens_per_second": 1393699.3259877684, "eta_seconds": 15724.1333933115, "data_wait_seconds": 2.1845102310180664e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 30690, "total_steps": 72479, "tokens": 16090398720, "loss": 4.5022419691085815, "l_mix": 2.788520097732544, "l_a": 2.8823733031749725, "l_c": 2.830032765865326, "z_loss": 24.521984100341797, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16379740834236145, "tokens_per_second": 1441628.4365262433, "eta_seconds": 15197.724099278448, "data_wait_seconds": 1.7464160919189453e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 30700, "total_steps": 72479, "tokens": 16095641600, "loss": 4.534998655319214, "l_mix": 2.808952808380127, "l_a": 2.903407573699951, "l_c": 2.8500784635543823, "z_loss": 25.348859071731567, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1393359899520874, "tokens_per_second": 1452838.7406036386, "eta_seconds": 15076.847649931908, "data_wait_seconds": 1.7404556274414062e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.30006677, "gate_a2b/L1_absmax": 0.65253848, "gate_a2b/L1_rms": 0.31430563, "gate_a2b/L3_absmean": 0.31538853, "gate_a2b/L3_absmax": 0.61070114, "gate_a2b/L3_rms": 0.32067865, "gate_a2b/L5_absmean": 0.48722717, "gate_a2b/L5_absmax": 0.6546911, "gate_a2b/L5_rms": 0.49283174, "gate_c2b/L1_absmean": 0.0161093, "gate_c2b/L1_absmax": 0.51459026, "gate_c2b/L1_rms": 0.06257869, "gate_c2b/L3_absmean": 0.21836019, "gate_c2b/L3_absmax": 0.46886864, "gate_c2b/L3_rms": 0.22252747, "gate_c2b/L5_absmean": 0.57042879, "gate_c2b/L5_absmax": 0.7837562, "gate_c2b/L5_rms": 0.58353084, "gate_b2a/L7_absmean": 0.07638155, "gate_b2a/L7_absmax": 0.32191274, "gate_b2a/L7_rms": 0.08096727, "gate_b2a/L11_absmean": 0.11763176, "gate_b2a/L11_absmax": 0.56325483, "gate_b2a/L11_rms": 0.12285406, "gate_b2a/L14_absmean": 0.29582149, "gate_b2a/L14_absmax": 0.65547991, "gate_b2a/L14_rms": 0.29980567, "gate_b2c/L3_absmean": 0.07657745, "gate_b2c/L3_absmax": 0.6861071, "gate_b2c/L3_rms": 0.09082819, "gate_b2c/L5_absmean": 0.08940842, "gate_b2c/L5_absmax": 0.81390452, "gate_b2c/L5_rms": 0.100858, "gate_b2c/L6_absmean": 0.13143663, "gate_b2c/L6_absmax": 0.55713409, "gate_b2c/L6_rms": 0.13961564}, "streams": {"cos_a2b/round0": 0.402447, "cos_c2b/round0": 0.304961, "cos_b2a/round0": 0.124439, "cos_b2c/round0": 0.145331, "rms_a/round0": 1.626881, "rms_b/round0": 3.0465, "rms_c/round0": 1.855363, "cos_a2b/round1": 0.434986, "cos_c2b/round1": 0.290615, "cos_b2a/round1": 0.235119, "cos_b2c/round1": 0.123171, "rms_a/round1": 2.963336, "rms_b/round1": 6.390678, "rms_c/round1": 3.38977, "cos_a2b/round2": 0.557365, "cos_c2b/round2": 0.51467, "cos_b2a/round2": 0.081833, "cos_b2c/round2": 0.155811, "rms_a/round2": 6.956575, "rms_b/round2": 14.98271, "rms_c/round2": 5.830085}, "alpha_mean": 0.4392310683339531, "alpha_hist": [0.00786, 0.06215, 0.15927, 0.20082, 0.20111, 0.17941, 0.12595, 0.05704, 0.00638, 2e-05], "alpha_by_type": {"special": {"share": 0.00131, "mean": 0.56824, "std": 0.0905}, "whitespace": {"share": 0.02802, "mean": 0.53553, "std": 0.11401}, "digit": {"share": 0.02931, "mean": 0.45943, "std": 0.1317}, "word_start": {"share": 0.58124, "mean": 0.44881, "std": 0.16063}, "word_cont": {"share": 0.2527, "mean": 0.34467, "std": 0.13098}, "punct": {"share": 0.10742, "mean": 0.57767, "std": 0.1496}}, "alpha_p05": 0.18832, "alpha_p50": 0.44698, "alpha_p95": 0.72037}
{"step": 30710, "total_steps": 72479, "tokens": 16100884480, "loss": 4.5015756487846375, "l_mix": 2.786964565515518, "l_a": 2.8838694095611572, "l_c": 2.8315005600452423, "z_loss": 25.528170824050903, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16041064262390137, "tokens_per_second": 1251413.5192500248, "eta_seconds": 17499.39978682995, "data_wait_seconds": 1.5795230865478516e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 30720, "total_steps": 72479, "tokens": 16106127360, "loss": 2.930383086204529, "l_mix": 1.8132574558258057, "l_a": 1.8829632550477982, "l_c": 1.8407888561487198, "z_loss": 29.470332384109497, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16982360184192657, "tokens_per_second": 1332230.5171538163, "eta_seconds": 16433.89962179661, "data_wait_seconds": 2.0265579223632812e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08236}}
{"step": 30730, "total_steps": 72479, "tokens": 16111370240, "loss": 4.84640371799469, "l_mix": 3.005097836256027, "l_a": 3.086453467607498, "l_c": 3.0512325763702393, "z_loss": 24.756901025772095, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1385968029499054, "tokens_per_second": 1420187.6575385754, "eta_seconds": 15412.399619030954, "data_wait_seconds": 1.7642974853515625e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20768, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 30740, "total_steps": 72479, "tokens": 16116613120, "loss": 4.915266811847687, "l_mix": 3.0486723482608795, "l_a": 3.1306890547275543, "l_c": 3.0912922024726868, "z_loss": 24.381601095199585, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15795254707336426, "tokens_per_second": 1286433.5784933513, "eta_seconds": 17010.79418156147, "data_wait_seconds": 1.6897916793823242e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 30750, "total_steps": 72479, "tokens": 16121856000, "loss": 4.803565084934235, "l_mix": 2.9799415469169617, "l_a": 3.0543258488178253, "l_c": 3.0244188606739044, "z_loss": 24.55642342567444, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20825724303722382, "tokens_per_second": 1446637.600162148, "eta_seconds": 15123.35497815609, "data_wait_seconds": 2.205371856689453e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 30760, "total_steps": 72479, "tokens": 16127098880, "loss": 4.858730792999268, "l_mix": 3.0131359100341797, "l_a": 3.095402419567108, "l_c": 3.056580126285553, "z_loss": 24.593132972717285, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16261760890483856, "tokens_per_second": 1451878.562141759, "eta_seconds": 15065.151895165443, "data_wait_seconds": 2.1249055862426758e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 30770, "total_steps": 72479, "tokens": 16132341760, "loss": 4.834167063236237, "l_mix": 2.9986073970794678, "l_a": 3.075908809900284, "l_c": 3.0426231920719147, "z_loss": 24.898428916931152, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17906656861305237, "tokens_per_second": 1289642.5297393173, "eta_seconds": 16956.270972561837, "data_wait_seconds": 2.104043960571289e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 30780, "total_steps": 72479, "tokens": 16137584640, "loss": 4.46062296628952, "l_mix": 2.761399418115616, "l_a": 2.8611136078834534, "l_c": 2.802964895963669, "z_loss": 24.80758833885193, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17353659868240356, "tokens_per_second": 1437323.3671809928, "eta_seconds": 15210.415283846856, "data_wait_seconds": 2.4944543838500977e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 30790, "total_steps": 72479, "tokens": 16142827520, "loss": 4.503930628299713, "l_mix": 2.7893628776073456, "l_a": 2.886050045490265, "l_c": 2.8291757106781006, "z_loss": 24.83459782600403, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13420751690864563, "tokens_per_second": 1441598.572047142, "eta_seconds": 15161.67042324543, "data_wait_seconds": 1.537799835205078e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 30800, "total_steps": 72479, "tokens": 16148070400, "loss": 4.430929273366928, "l_mix": 2.744388997554779, "l_a": 2.835163027048111, "l_c": 2.7866379022598267, "z_loss": 24.894920825958252, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11963506042957306, "tokens_per_second": 1408571.6059897288, "eta_seconds": 15513.44600379467, "data_wait_seconds": 1.9311904907226562e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.29809928, "gate_a2b/L1_absmax": 0.65515268, "gate_a2b/L1_rms": 0.31253761, "gate_a2b/L3_absmean": 0.3129921, "gate_a2b/L3_absmax": 0.60692537, "gate_a2b/L3_rms": 0.31838819, "gate_a2b/L5_absmean": 0.48442528, "gate_a2b/L5_absmax": 0.65765607, "gate_a2b/L5_rms": 0.49013793, "gate_c2b/L1_absmean": 0.01725904, "gate_c2b/L1_absmax": 0.51765496, "gate_c2b/L1_rms": 0.06316139, "gate_c2b/L3_absmean": 0.21886043, "gate_c2b/L3_absmax": 0.4678424, "gate_c2b/L3_rms": 0.22315432, "gate_c2b/L5_absmean": 0.57159215, "gate_c2b/L5_absmax": 0.78661788, "gate_c2b/L5_rms": 0.58459598, "gate_b2a/L7_absmean": 0.07580242, "gate_b2a/L7_absmax": 0.32253787, "gate_b2a/L7_rms": 0.08043461, "gate_b2a/L11_absmean": 0.11676444, "gate_b2a/L11_absmax": 0.56393528, "gate_b2a/L11_rms": 0.12208878, "gate_b2a/L14_absmean": 0.29323208, "gate_b2a/L14_absmax": 0.66408694, "gate_b2a/L14_rms": 0.29738879, "gate_b2c/L3_absmean": 0.07475393, "gate_b2c/L3_absmax": 0.68722636, "gate_b2c/L3_rms": 0.08918952, "gate_b2c/L5_absmean": 0.08731571, "gate_b2c/L5_absmax": 0.80910116, "gate_b2c/L5_rms": 0.0989187, "gate_b2c/L6_absmean": 0.12921724, "gate_b2c/L6_absmax": 0.55988741, "gate_b2c/L6_rms": 0.13754089}, "streams": {"cos_a2b/round0": 0.401101, "cos_c2b/round0": 0.303773, "cos_b2a/round0": 0.12417, "cos_b2c/round0": 0.141763, "rms_a/round0": 1.667859, "rms_b/round0": 3.120141, "rms_c/round0": 1.880976, "cos_a2b/round1": 0.427811, "cos_c2b/round1": 0.282755, "cos_b2a/round1": 0.238604, "cos_b2c/round1": 0.117962, "rms_a/round1": 3.001156, "rms_b/round1": 6.575343, "rms_c/round1": 3.439918, "cos_a2b/round2": 0.543308, "cos_c2b/round2": 0.521358, "cos_b2a/round2": 0.088774, "cos_b2c/round2": 0.156305, "rms_a/round2": 7.119616, "rms_b/round2": 15.639421, "rms_c/round2": 5.941754}, "alpha_mean": 0.4265989240102357, "alpha_hist": [0.01396, 0.0925, 0.17677, 0.19182, 0.17155, 0.15126, 0.12471, 0.06677, 0.01059, 6e-05], "alpha_by_type": {"special": {"share": 0.00076, "mean": 0.54716, "std": 0.09927}, "whitespace": {"share": 0.02692, "mean": 0.52526, "std": 0.12416}, "digit": {"share": 0.03491, "mean": 0.52102, "std": 0.12272}, "word_start": {"share": 0.56104, "mean": 0.43816, "std": 0.16795}, "word_cont": {"share": 0.25238, "mean": 0.30201, "std": 0.13821}, "punct": {"share": 0.12399, "mean": 0.57913, "std": 0.16137}}, "alpha_p05": 0.1538, "alpha_p50": 0.41064, "alpha_p95": 0.71879}
{"step": 30810, "total_steps": 72479, "tokens": 16153313280, "loss": 4.597318053245544, "l_mix": 2.847607284784317, "l_a": 2.9437520503997803, "l_c": 2.8886170387268066, "z_loss": 25.172032833099365, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13193295896053314, "tokens_per_second": 1447662.9020271741, "eta_seconds": 15090.914218640326, "data_wait_seconds": 1.7821788787841797e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 30820, "total_steps": 72479, "tokens": 16158556160, "loss": 4.571117460727692, "l_mix": 2.8312028348445892, "l_a": 2.9251136481761932, "l_c": 2.874601364135742, "z_loss": 24.49406147003174, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1552240252494812, "tokens_per_second": 1465794.364191589, "eta_seconds": 14900.667055058479, "data_wait_seconds": 1.570582389831543e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 30830, "total_steps": 72479, "tokens": 16163799040, "loss": 3.1986339688301086, "l_mix": 1.9789720326662064, "l_a": 2.0539013147354126, "l_c": 2.0116383880376816, "z_loss": 30.075580835342407, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16795361042022705, "tokens_per_second": 1377143.8223647294, "eta_seconds": 15856.056976318361, "data_wait_seconds": 2.181529998779297e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 30840, "total_steps": 72479, "tokens": 16169041920, "loss": 3.1139679551124573, "l_mix": 1.9274447113275528, "l_a": 1.9944660663604736, "l_c": 1.960611268877983, "z_loss": 28.5166175365448, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20942077040672302, "tokens_per_second": 1403694.1056399182, "eta_seconds": 15552.411272716521, "data_wait_seconds": 2.574920654296875e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08236}}
{"step": 30850, "total_steps": 72479, "tokens": 16174284800, "loss": 4.902978539466858, "l_mix": 3.041109949350357, "l_a": 3.1208130419254303, "l_c": 3.0854155123233795, "z_loss": 24.591087818145752, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14518342912197113, "tokens_per_second": 1465766.6165298466, "eta_seconds": 14890.218473982812, "data_wait_seconds": 1.8268823623657227e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 30860, "total_steps": 72479, "tokens": 16179527680, "loss": 4.900485455989838, "l_mix": 3.039802074432373, "l_a": 3.119624584913254, "l_c": 3.082653135061264, "z_loss": 24.999480724334717, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15072396397590637, "tokens_per_second": 1316753.0324681278, "eta_seconds": 16571.324868035317, "data_wait_seconds": 3.6597251892089844e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 30870, "total_steps": 72479, "tokens": 16184770560, "loss": 4.934268295764923, "l_mix": 3.060334324836731, "l_a": 3.140464425086975, "l_c": 3.1059819161891937, "z_loss": 24.877503156661987, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18409770727157593, "tokens_per_second": 1440246.234016645, "eta_seconds": 15146.784540557863, "data_wait_seconds": 2.7239322662353516e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 30880, "total_steps": 72479, "tokens": 16190013440, "loss": 4.877869129180908, "l_mix": 3.0252146124839783, "l_a": 3.1058026254177094, "l_c": 3.0697124004364014, "z_loss": 25.541910648345947, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20202451944351196, "tokens_per_second": 1477068.0097333656, "eta_seconds": 14765.641370797157, "data_wait_seconds": 1.722574234008789e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 30890, "total_steps": 72479, "tokens": 16195256320, "loss": 4.54069584608078, "l_mix": 2.8126258552074432, "l_a": 2.9052586555480957, "l_c": 2.8549744486808777, "z_loss": 25.283539295196533, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11461728811264038, "tokens_per_second": 1226812.0911134053, "eta_seconds": 17773.393162608147, "data_wait_seconds": 1.722574234008789e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08232}}
{"step": 30900, "total_steps": 72479, "tokens": 16200499200, "loss": 4.472388029098511, "l_mix": 2.7699474692344666, "l_a": 2.8640478253364563, "l_c": 2.810753971338272, "z_loss": 25.207911252975464, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13762405514717102, "tokens_per_second": 1365839.8043695854, "eta_seconds": 15960.415476441382, "data_wait_seconds": 2.0265579223632812e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.29689416, "gate_a2b/L1_absmax": 0.65801609, "gate_a2b/L1_rms": 0.31129843, "gate_a2b/L3_absmean": 0.31143817, "gate_a2b/L3_absmax": 0.61211175, "gate_a2b/L3_rms": 0.31682506, "gate_a2b/L5_absmean": 0.48212227, "gate_a2b/L5_absmax": 0.64960635, "gate_a2b/L5_rms": 0.48782033, "gate_c2b/L1_absmean": 0.01650928, "gate_c2b/L1_absmax": 0.52091616, "gate_c2b/L1_rms": 0.06290466, "gate_c2b/L3_absmean": 0.21915014, "gate_c2b/L3_absmax": 0.46830159, "gate_c2b/L3_rms": 0.22340357, "gate_c2b/L5_absmean": 0.57333672, "gate_c2b/L5_absmax": 0.79619694, "gate_c2b/L5_rms": 0.58651096, "gate_b2a/L7_absmean": 0.0747413, "gate_b2a/L7_absmax": 0.32269627, "gate_b2a/L7_rms": 0.07939211, "gate_b2a/L11_absmean": 0.11558695, "gate_b2a/L11_absmax": 0.5657866, "gate_b2a/L11_rms": 0.12103292, "gate_b2a/L14_absmean": 0.29182613, "gate_b2a/L14_absmax": 0.6681276, "gate_b2a/L14_rms": 0.29603201, "gate_b2c/L3_absmean": 0.07377423, "gate_b2c/L3_absmax": 0.6857388, "gate_b2c/L3_rms": 0.08805957, "gate_b2c/L5_absmean": 0.08612436, "gate_b2c/L5_absmax": 0.80676907, "gate_b2c/L5_rms": 0.09761506, "gate_b2c/L6_absmean": 0.12654999, "gate_b2c/L6_absmax": 0.55882055, "gate_b2c/L6_rms": 0.13495669}, "streams": {"cos_a2b/round0": 0.40345, "cos_c2b/round0": 0.308427, "cos_b2a/round0": 0.127285, "cos_b2c/round0": 0.143511, "rms_a/round0": 1.716441, "rms_b/round0": 3.208746, "rms_c/round0": 1.903869, "cos_a2b/round1": 0.436825, "cos_c2b/round1": 0.293993, "cos_b2a/round1": 0.241188, "cos_b2c/round1": 0.122431, "rms_a/round1": 3.114691, "rms_b/round1": 6.735116, "rms_c/round1": 3.513287, "cos_a2b/round2": 0.564104, "cos_c2b/round2": 0.511709, "cos_b2a/round2": 0.097277, "cos_b2c/round2": 0.164637, "rms_a/round2": 7.402885, "rms_b/round2": 15.865441, "rms_c/round2": 6.050137}, "alpha_mean": 0.44423898317756993, "alpha_hist": [0.00827, 0.05667, 0.14793, 0.19812, 0.22697, 0.16704, 0.12302, 0.06261, 0.00935, 2e-05], "alpha_by_type": {"special": {"share": 0.00107, "mean": 0.55487, "std": 0.1067}, "whitespace": {"share": 0.03416, "mean": 0.5189, "std": 0.10806}, "digit": {"share": 0.0266, "mean": 0.4261, "std": 0.11594}, "word_start": {"share": 0.584, "mean": 0.46274, "std": 0.16083}, "word_cont": {"share": 0.24945, "mean": 0.34007, "std": 0.12642}, "punct": {"share": 0.10472, "mean": 0.56832, "std": 0.15637}}, "alpha_p05": 0.19437, "alpha_p50": 0.44228, "alpha_p95": 0.74017}
{"step": 30910, "total_steps": 72479, "tokens": 16205742080, "loss": 4.484714686870575, "l_mix": 2.778658479452133, "l_a": 2.86905699968338, "l_c": 2.8177970349788666, "z_loss": 25.02037525177002, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13591015338897705, "tokens_per_second": 1399996.4319580598, "eta_seconds": 15567.273869061471, "data_wait_seconds": 1.77919864654541e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 30920, "total_steps": 72479, "tokens": 16210984960, "loss": 4.438600182533264, "l_mix": 2.748628854751587, "l_a": 2.8434669971466064, "l_c": 2.789770156145096, "z_loss": 25.398092031478882, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1496431976556778, "tokens_per_second": 1442046.7637586463, "eta_seconds": 15109.693762779236, "data_wait_seconds": 1.7076730728149414e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 30930, "total_steps": 72479, "tokens": 16216227840, "loss": 4.34012508392334, "l_mix": 2.6870571076869965, "l_a": 2.784065067768097, "l_c": 2.7261615097522736, "z_loss": 26.124633073806763, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13923314213752747, "tokens_per_second": 1455800.925037505, "eta_seconds": 14963.3385563612, "data_wait_seconds": 1.582503318786621e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 30940, "total_steps": 72479, "tokens": 16221470720, "loss": 3.1744989156723022, "l_mix": 1.9647955745458603, "l_a": 2.0360791385173798, "l_c": 1.99626524746418, "z_loss": 29.422120809555054, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.22399966418743134, "tokens_per_second": 1424541.1283814113, "eta_seconds": 15288.010151553153, "data_wait_seconds": 1.6748905181884766e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 30950, "total_steps": 72479, "tokens": 16226713600, "loss": 3.1925703287124634, "l_mix": 1.9735557585954666, "l_a": 2.0332555025815964, "l_c": 2.0301260352134705, "z_loss": 29.148458242416382, "lr": 0.0023, "wd": 0.01, "grad_norm": 1.132035732269287, "tokens_per_second": 1407486.0411708676, "eta_seconds": 15469.536261892317, "data_wait_seconds": 1.665949821472168e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08236}}
{"step": 30960, "total_steps": 72479, "tokens": 16231956480, "loss": 3.266065329313278, "l_mix": 2.0082249343395233, "l_a": 2.135004371404648, "l_c": 2.0577967315912247, "z_loss": 26.208197116851807, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2113879919052124, "tokens_per_second": 1323788.3058849324, "eta_seconds": 16443.651432204246, "data_wait_seconds": 1.8805265426635742e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20045, "finemath": 0.08233}}
{"step": 30970, "total_steps": 72479, "tokens": 16237199360, "loss": 4.52312308549881, "l_mix": 2.8019879758358, "l_a": 2.895004391670227, "l_c": 2.8421121537685394, "z_loss": 24.57966423034668, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1385863721370697, "tokens_per_second": 1412791.9574067325, "eta_seconds": 15404.016478085518, "data_wait_seconds": 2.1457672119140625e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 30980, "total_steps": 72479, "tokens": 16242442240, "loss": 4.458946645259857, "l_mix": 2.7611189484596252, "l_a": 2.8566320538520813, "l_c": 2.802793502807617, "z_loss": 25.226986408233643, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16518892347812653, "tokens_per_second": 1461248.9097577038, "eta_seconds": 14889.61091208458, "data_wait_seconds": 1.728534698486328e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 30990, "total_steps": 72479, "tokens": 16247685120, "loss": 4.475305497646332, "l_mix": 2.7706433534622192, "l_a": 2.872037261724472, "l_c": 2.810169816017151, "z_loss": 25.627062797546387, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14880289137363434, "tokens_per_second": 1362727.5677069826, "eta_seconds": 15962.240250706673, "data_wait_seconds": 2.0176172256469727e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 31000, "total_steps": 72479, "tokens": 16252928000, "loss": 4.422894060611725, "l_mix": 2.739476054906845, "l_a": 2.832145571708679, "l_c": 2.7792471945285797, "z_loss": 24.950236558914185, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1685585230588913, "tokens_per_second": 1450891.6988112645, "eta_seconds": 14988.673496317864, "data_wait_seconds": 2.1219253540039062e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.29601154, "gate_a2b/L1_absmax": 0.65943456, "gate_a2b/L1_rms": 0.31042209, "gate_a2b/L3_absmean": 0.30976281, "gate_a2b/L3_absmax": 0.60967553, "gate_a2b/L3_rms": 0.3152419, "gate_a2b/L5_absmean": 0.4802064, "gate_a2b/L5_absmax": 0.65304166, "gate_a2b/L5_rms": 0.48588833, "gate_c2b/L1_absmean": 0.01609633, "gate_c2b/L1_absmax": 0.52419305, "gate_c2b/L1_rms": 0.06289625, "gate_c2b/L3_absmean": 0.21899672, "gate_c2b/L3_absmax": 0.47018227, "gate_c2b/L3_rms": 0.22318409, "gate_c2b/L5_absmean": 0.57507104, "gate_c2b/L5_absmax": 0.80096495, "gate_c2b/L5_rms": 0.58818507, "gate_b2a/L7_absmean": 0.07485333, "gate_b2a/L7_absmax": 0.32237083, "gate_b2a/L7_rms": 0.07953354, "gate_b2a/L11_absmean": 0.1152753, "gate_b2a/L11_absmax": 0.56530488, "gate_b2a/L11_rms": 0.12070697, "gate_b2a/L14_absmean": 0.2903533, "gate_b2a/L14_absmax": 0.67278808, "gate_b2a/L14_rms": 0.29466757, "gate_b2c/L3_absmean": 0.07335999, "gate_b2c/L3_absmax": 0.68432736, "gate_b2c/L3_rms": 0.08719843, "gate_b2c/L5_absmean": 0.08509676, "gate_b2c/L5_absmax": 0.7997331, "gate_b2c/L5_rms": 0.09650198, "gate_b2c/L6_absmean": 0.12466274, "gate_b2c/L6_absmax": 0.55945963, "gate_b2c/L6_rms": 0.13309638}, "streams": {"cos_a2b/round0": 0.406453, "cos_c2b/round0": 0.299807, "cos_b2a/round0": 0.127646, "cos_b2c/round0": 0.142387, "rms_a/round0": 1.760778, "rms_b/round0": 3.283998, "rms_c/round0": 1.931844, "cos_a2b/round1": 0.434831, "cos_c2b/round1": 0.291963, "cos_b2a/round1": 0.243912, "cos_b2c/round1": 0.127766, "rms_a/round1": 3.154948, "rms_b/round1": 6.901234, "rms_c/round1": 3.550737, "cos_a2b/round2": 0.563435, "cos_c2b/round2": 0.511465, "cos_b2a/round2": 0.091027, "cos_b2c/round2": 0.165894, "rms_a/round2": 7.564355, "rms_b/round2": 16.329634, "rms_c/round2": 6.158852}, "alpha_mean": 0.4246973771894318, "alpha_hist": [0.01056, 0.07365, 0.17903, 0.1993, 0.19357, 0.17519, 0.11723, 0.04839, 0.00308, 0.0], "alpha_by_type": {"special": {"share": 0.00119, "mean": 0.55063, "std": 0.0931}, "whitespace": {"share": 0.02824, "mean": 0.52748, "std": 0.10604}, "digit": {"share": 0.03574, "mean": 0.45965, "std": 0.10461}, "word_start": {"share": 0.57835, "mean": 0.44341, "std": 0.15671}, "word_cont": {"share": 0.25299, "mean": 0.30938, "std": 0.13054}, "punct": {"share": 0.10349, "mean": 0.56047, "std": 0.14844}}, "alpha_p05": 0.17024, "alpha_p50": 0.42728, "alpha_p95": 0.70781}
{"step": 31010, "total_steps": 72479, "tokens": 16258170880, "loss": 4.4501612186431885, "l_mix": 2.7562440037727356, "l_a": 2.8504919707775116, "l_c": 2.7958986163139343, "z_loss": 25.471630811691284, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15069405734539032, "tokens_per_second": 1436500.2126651076, "eta_seconds": 15135.18681049347, "data_wait_seconds": 1.7017126083374023e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 31020, "total_steps": 72479, "tokens": 16263413760, "loss": 4.484433889389038, "l_mix": 2.776290863752365, "l_a": 2.874946177005768, "l_c": 2.8188635408878326, "z_loss": 25.536754846572876, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14395424723625183, "tokens_per_second": 1340352.7836754438, "eta_seconds": 16216.96650071144, "data_wait_seconds": 1.6301870346069336e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 31030, "total_steps": 72479, "tokens": 16268656640, "loss": 4.437572419643402, "l_mix": 2.7479933500289917, "l_a": 2.8430125415325165, "l_c": 2.7889179587364197, "z_loss": 24.9904887676239, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15143826603889465, "tokens_per_second": 1435618.5773163827, "eta_seconds": 15137.177559113503, "data_wait_seconds": 1.531839370727539e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 31040, "total_steps": 72479, "tokens": 16273899520, "loss": 4.511038720607758, "l_mix": 2.7939513325691223, "l_a": 2.883917659521103, "l_c": 2.8397064805030823, "z_loss": 25.28334379196167, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13246841728687286, "tokens_per_second": 1448984.4305020978, "eta_seconds": 14993.92952377796, "data_wait_seconds": 1.71661376953125e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 31050, "total_steps": 72479, "tokens": 16279142400, "loss": 4.9288522601127625, "l_mix": 3.056959331035614, "l_a": 3.1397712528705597, "l_c": 3.099871575832367, "z_loss": 25.19415783882141, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13511046767234802, "tokens_per_second": 1359925.0320973545, "eta_seconds": 15972.003632068634, "data_wait_seconds": 3.018975257873535e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 31060, "total_steps": 72479, "tokens": 16284385280, "loss": 3.150995522737503, "l_mix": 1.9496030062437057, "l_a": 2.0232934206724167, "l_c": 1.9813483655452728, "z_loss": 29.37253713607788, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.21585917472839355, "tokens_per_second": 1400445.9698514366, "eta_seconds": 15506.121006798743, "data_wait_seconds": 1.9371509552001953e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08236}}
{"step": 31070, "total_steps": 72479, "tokens": 16289628160, "loss": 3.3309242129325867, "l_mix": 2.0503706634044647, "l_a": 2.163923680782318, "l_c": 2.104588031768799, "z_loss": 25.86892056465149, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1439685821533203, "tokens_per_second": 1415435.847321208, "eta_seconds": 15338.202598929405, "data_wait_seconds": 2.345442771911621e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20045, "finemath": 0.08233}}
{"step": 31080, "total_steps": 72479, "tokens": 16294871040, "loss": 4.4862000942230225, "l_mix": 2.7782831490039825, "l_a": 2.873068183660507, "l_c": 2.819987952709198, "z_loss": 25.77642822265625, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13240690529346466, "tokens_per_second": 1433488.87656468, "eta_seconds": 15141.379376459121, "data_wait_seconds": 2.014636993408203e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 31090, "total_steps": 72479, "tokens": 16300113920, "loss": 4.559824347496033, "l_mix": 2.824418604373932, "l_a": 2.920519709587097, "l_c": 2.8641660511493683, "z_loss": 25.463517427444458, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16163906455039978, "tokens_per_second": 1283120.6796091467, "eta_seconds": 16911.70314440727, "data_wait_seconds": 2.753734588623047e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 31100, "total_steps": 72479, "tokens": 16305356800, "loss": 4.499058187007904, "l_mix": 2.787719249725342, "l_a": 2.877702921628952, "l_c": 2.826759785413742, "z_loss": 25.87730646133423, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1912703961133957, "tokens_per_second": 1373846.0068935228, "eta_seconds": 15791.080691099167, "data_wait_seconds": 2.22623348236084e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.29412016, "gate_a2b/L1_absmax": 0.66166627, "gate_a2b/L1_rms": 0.30858904, "gate_a2b/L3_absmean": 0.30877367, "gate_a2b/L3_absmax": 0.60763907, "gate_a2b/L3_rms": 0.31422025, "gate_a2b/L5_absmean": 0.47813126, "gate_a2b/L5_absmax": 0.65455139, "gate_a2b/L5_rms": 0.48382771, "gate_c2b/L1_absmean": 0.01639876, "gate_c2b/L1_absmax": 0.52715254, "gate_c2b/L1_rms": 0.06336063, "gate_c2b/L3_absmean": 0.21907194, "gate_c2b/L3_absmax": 0.4779003, "gate_c2b/L3_rms": 0.22328416, "gate_c2b/L5_absmean": 0.57640845, "gate_c2b/L5_absmax": 0.79948086, "gate_c2b/L5_rms": 0.58953172, "gate_b2a/L7_absmean": 0.07375687, "gate_b2a/L7_absmax": 0.32233685, "gate_b2a/L7_rms": 0.07858028, "gate_b2a/L11_absmean": 0.11464405, "gate_b2a/L11_absmax": 0.56491828, "gate_b2a/L11_rms": 0.12011807, "gate_b2a/L14_absmean": 0.28839475, "gate_b2a/L14_absmax": 0.67661816, "gate_b2a/L14_rms": 0.29280746, "gate_b2c/L3_absmean": 0.07231249, "gate_b2c/L3_absmax": 0.68422872, "gate_b2c/L3_rms": 0.08627218, "gate_b2c/L5_absmean": 0.08426026, "gate_b2c/L5_absmax": 0.80215424, "gate_b2c/L5_rms": 0.09570274, "gate_b2c/L6_absmean": 0.1230927, "gate_b2c/L6_absmax": 0.55654186, "gate_b2c/L6_rms": 0.13152698}, "streams": {"cos_a2b/round0": 0.405263, "cos_c2b/round0": 0.303736, "cos_b2a/round0": 0.127073, "cos_b2c/round0": 0.142991, "rms_a/round0": 1.802995, "rms_b/round0": 3.388119, "rms_c/round0": 1.967207, "cos_a2b/round1": 0.433207, "cos_c2b/round1": 0.293735, "cos_b2a/round1": 0.246002, "cos_b2c/round1": 0.123704, "rms_a/round1": 3.258707, "rms_b/round1": 7.110023, "rms_c/round1": 3.629735, "cos_a2b/round2": 0.560388, "cos_c2b/round2": 0.518715, "cos_b2a/round2": 0.090259, "cos_b2c/round2": 0.165063, "rms_a/round2": 7.771553, "rms_b/round2": 16.780661, "rms_c/round2": 6.315997}, "alpha_mean": 0.44756898035329584, "alpha_hist": [0.00789, 0.05817, 0.15852, 0.19518, 0.18805, 0.18015, 0.13216, 0.07048, 0.0094, 2e-05], "alpha_by_type": {"special": {"share": 0.00133, "mean": 0.56143, "std": 0.12846}, "whitespace": {"share": 0.02565, "mean": 0.54729, "std": 0.1181}, "digit": {"share": 0.02596, "mean": 0.49017, "std": 0.11618}, "word_start": {"share": 0.59288, "mean": 0.45958, "std": 0.16235}, "word_cont": {"share": 0.24799, "mean": 0.34155, "std": 0.13559}, "punct": {"share": 0.1062, "mean": 0.59215, "std": 0.15288}}, "alpha_p05": 0.18359, "alpha_p50": 0.43326, "alpha_p95": 0.74354}
{"step": 31110, "total_steps": 72479, "tokens": 16310599680, "loss": 4.517990410327911, "l_mix": 2.798428535461426, "l_a": 2.8926249146461487, "l_c": 2.83924800157547, "z_loss": 25.102936506271362, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15169961750507355, "tokens_per_second": 1321952.4722245634, "eta_seconds": 16406.99701972008, "data_wait_seconds": 2.4557113647460938e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 31120, "total_steps": 72479, "tokens": 16315842560, "loss": 4.584644913673401, "l_mix": 2.84045872092247, "l_a": 2.935636669397354, "l_c": 2.8783171474933624, "z_loss": 26.051554203033447, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20303061604499817, "tokens_per_second": 1438983.5266279809, "eta_seconds": 15068.989318323136, "data_wait_seconds": 1.8775463104248047e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08236}}
{"step": 31130, "total_steps": 72479, "tokens": 16321085440, "loss": 4.880403757095337, "l_mix": 3.025958925485611, "l_a": 3.1111835837364197, "l_c": 3.0702988505363464, "z_loss": 25.303097248077393, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13773038983345032, "tokens_per_second": 1333096.859497078, "eta_seconds": 16261.972532272339, "data_wait_seconds": 3.325939178466797e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20768, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 31140, "total_steps": 72479, "tokens": 16326328320, "loss": 4.994812846183777, "l_mix": 3.0973786115646362, "l_a": 3.1791152358055115, "l_c": 3.145665407180786, "z_loss": 25.258825540542603, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1612887680530548, "tokens_per_second": 1380506.5592841883, "eta_seconds": 15699.702030563354, "data_wait_seconds": 1.868605613708496e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20768, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 31150, "total_steps": 72479, "tokens": 16331571200, "loss": 4.821859180927277, "l_mix": 2.9896941781044006, "l_a": 3.0751820504665375, "l_c": 3.0320343375205994, "z_loss": 25.416505813598633, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13277028501033783, "tokens_per_second": 1448925.1418679722, "eta_seconds": 14954.73998336792, "data_wait_seconds": 1.7076730728149414e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 31160, "total_steps": 72479, "tokens": 16336814080, "loss": 4.904562830924988, "l_mix": 3.0412363409996033, "l_a": 3.124464213848114, "l_c": 3.086623936891556, "z_loss": 25.326573371887207, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15524908900260925, "tokens_per_second": 1285616.65979745, "eta_seconds": 16850.32292239666, "data_wait_seconds": 3.102421760559082e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 31170, "total_steps": 72479, "tokens": 16342056960, "loss": 4.8890910148620605, "l_mix": 3.031864643096924, "l_a": 3.115466982126236, "l_c": 3.075287252664566, "z_loss": 24.8456609249115, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15719486773014069, "tokens_per_second": 1411131.88107345, "eta_seconds": 15347.830548286438, "data_wait_seconds": 2.855062484741211e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 31180, "total_steps": 72479, "tokens": 16347299840, "loss": 3.297077029943466, "l_mix": 2.026440352201462, "l_a": 2.1525884568691254, "l_c": 2.082867056131363, "z_loss": 26.29780125617981, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18995410203933716, "tokens_per_second": 1463433.2269912, "eta_seconds": 14795.734928417205, "data_wait_seconds": 2.562999725341797e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20045, "finemath": 0.08233}}
{"step": 31190, "total_steps": 72479, "tokens": 16352542720, "loss": 4.5229490995407104, "l_mix": 2.801701068878174, "l_a": 2.8948738276958466, "l_c": 2.8426193296909332, "z_loss": 25.693947315216064, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11769922822713852, "tokens_per_second": 1457045.542794413, "eta_seconds": 14857.001099967956, "data_wait_seconds": 2.8252601623535156e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 31200, "total_steps": 72479, "tokens": 16357785600, "loss": 4.6732242703437805, "l_mix": 2.8949641287326813, "l_a": 2.987155258655548, "l_c": 2.9403785467147827, "z_loss": 25.809107303619385, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1406233161687851, "tokens_per_second": 1460802.7750787423, "eta_seconds": 14815.199369287491, "data_wait_seconds": 3.325939178466797e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.29275256, "gate_a2b/L1_absmax": 0.66571683, "gate_a2b/L1_rms": 0.30722553, "gate_a2b/L3_absmean": 0.30630437, "gate_a2b/L3_absmax": 0.60980207, "gate_a2b/L3_rms": 0.31178772, "gate_a2b/L5_absmean": 0.47553015, "gate_a2b/L5_absmax": 0.65225989, "gate_a2b/L5_rms": 0.481213, "gate_c2b/L1_absmean": 0.01635112, "gate_c2b/L1_absmax": 0.53016275, "gate_c2b/L1_rms": 0.06375569, "gate_c2b/L3_absmean": 0.21852282, "gate_c2b/L3_absmax": 0.47632852, "gate_c2b/L3_rms": 0.22274376, "gate_c2b/L5_absmean": 0.57712716, "gate_c2b/L5_absmax": 0.79586375, "gate_c2b/L5_rms": 0.59038419, "gate_b2a/L7_absmean": 0.07344538, "gate_b2a/L7_absmax": 0.32225138, "gate_b2a/L7_rms": 0.07813977, "gate_b2a/L11_absmean": 0.11292672, "gate_b2a/L11_absmax": 0.56666738, "gate_b2a/L11_rms": 0.11853849, "gate_b2a/L14_absmean": 0.2866438, "gate_b2a/L14_absmax": 0.67866772, "gate_b2a/L14_rms": 0.2910904, "gate_b2c/L3_absmean": 0.07104489, "gate_b2c/L3_absmax": 0.68216157, "gate_b2c/L3_rms": 0.08502024, "gate_b2c/L5_absmean": 0.08248364, "gate_b2c/L5_absmax": 0.79818606, "gate_b2c/L5_rms": 0.09399196, "gate_b2c/L6_absmean": 0.12083532, "gate_b2c/L6_absmax": 0.55222565, "gate_b2c/L6_rms": 0.12929139}, "streams": {"cos_a2b/round0": 0.404521, "cos_c2b/round0": 0.301288, "cos_b2a/round0": 0.124698, "cos_b2c/round0": 0.141646, "rms_a/round0": 1.846208, "rms_b/round0": 3.469828, "rms_c/round0": 1.996308, "cos_a2b/round1": 0.428834, "cos_c2b/round1": 0.292244, "cos_b2a/round1": 0.242957, "cos_b2c/round1": 0.124179, "rms_a/round1": 3.310675, "rms_b/round1": 7.317408, "rms_c/round1": 3.702472, "cos_a2b/round2": 0.557224, "cos_c2b/round2": 0.51447, "cos_b2a/round2": 0.090346, "cos_b2c/round2": 0.162871, "rms_a/round2": 7.933063, "rms_b/round2": 17.261608, "rms_c/round2": 6.446029}, "alpha_mean": 0.44677343449376394, "alpha_hist": [0.01207, 0.05954, 0.15565, 0.19974, 0.18968, 0.16315, 0.13034, 0.07846, 0.01132, 5e-05], "alpha_by_type": {"special": {"share": 0.00156, "mean": 0.56701, "std": 0.07943}, "whitespace": {"share": 0.0248, "mean": 0.52055, "std": 0.11528}, "digit": {"share": 0.03168, "mean": 0.46208, "std": 0.10369}, "word_start": {"share": 0.59196, "mean": 0.46463, "std": 0.16734}, "word_cont": {"share": 0.24895, "mean": 0.33199, "std": 0.13497}, "punct": {"share": 0.10106, "mean": 0.60021, "std": 0.15869}}, "alpha_p05": 0.17357, "alpha_p50": 0.43254, "alpha_p95": 0.72855}
{"step": 31210, "total_steps": 72479, "tokens": 16363028480, "loss": 4.584255516529083, "l_mix": 2.839129149913788, "l_a": 2.934285670518875, "l_c": 2.8828020989894867, "z_loss": 25.655378341674805, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12499493360519409, "tokens_per_second": 1446234.486883535, "eta_seconds": 14960.811450862884, "data_wait_seconds": 2.428889274597168e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 31220, "total_steps": 72479, "tokens": 16368271360, "loss": 4.501131474971771, "l_mix": 2.7888309955596924, "l_a": 2.879372715950012, "l_c": 2.82829549908638, "z_loss": 25.688165426254272, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1735425889492035, "tokens_per_second": 1329273.333156018, "eta_seconds": 16273.250995445253, "data_wait_seconds": 2.5480985641479492e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 31230, "total_steps": 72479, "tokens": 16373514240, "loss": 4.627014338970184, "l_mix": 2.867236375808716, "l_a": 2.959463447332382, "l_c": 2.9064628779888153, "z_loss": 25.0490300655365, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14948107302188873, "tokens_per_second": 1412233.418427364, "eta_seconds": 15313.584447026253, "data_wait_seconds": 1.6808509826660156e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08236}}
{"step": 31240, "total_steps": 72479, "tokens": 16378757120, "loss": 3.2880941331386566, "l_mix": 2.0223782062530518, "l_a": 2.14084929227829, "l_c": 2.07820363342762, "z_loss": 25.99324893951416, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17869587242603302, "tokens_per_second": 1447730.3794626442, "eta_seconds": 14934.488588976861, "data_wait_seconds": 3.7223100662231445e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20045, "finemath": 0.08233}}
{"step": 31250, "total_steps": 72479, "tokens": 16384000000, "loss": 3.2963052093982697, "l_mix": 2.027644246816635, "l_a": 2.147028833627701, "l_c": 2.0818409621715546, "z_loss": 25.697004318237305, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1539856344461441, "tokens_per_second": 1389510.711347399, "eta_seconds": 15556.461548280717, "data_wait_seconds": 3.409385681152344e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20045, "finemath": 0.08234}}
{"step": 31260, "total_steps": 72479, "tokens": 16389242880, "loss": 4.901853978633881, "l_mix": 3.0406011044979095, "l_a": 3.119904637336731, "l_c": 3.0842711329460144, "z_loss": 25.478646278381348, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14556223154067993, "tokens_per_second": 1414637.2008929553, "eta_seconds": 15276.444772100449, "data_wait_seconds": 3.573298454284668e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 31270, "total_steps": 72479, "tokens": 16394485760, "loss": 4.89178341627121, "l_mix": 3.0345770716667175, "l_a": 3.1145979166030884, "l_c": 3.076089918613434, "z_loss": 25.550848484039307, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12962548434734344, "tokens_per_second": 1447996.156859982, "eta_seconds": 14920.885038018227, "data_wait_seconds": 3.4749507904052734e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 31280, "total_steps": 72479, "tokens": 16399728640, "loss": 4.879921078681946, "l_mix": 3.0257033109664917, "l_a": 3.1100601851940155, "l_c": 3.0706655979156494, "z_loss": 25.751638412475586, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13723686337471008, "tokens_per_second": 1391474.493385567, "eta_seconds": 15523.203202557565, "data_wait_seconds": 2.9921531677246094e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 31290, "total_steps": 72479, "tokens": 16404971520, "loss": 3.309806376695633, "l_mix": 2.036821559071541, "l_a": 2.1551772356033325, "l_c": 2.088105320930481, "z_loss": 25.800230503082275, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20271967351436615, "tokens_per_second": 1370297.9961906243, "eta_seconds": 15759.271700048446, "data_wait_seconds": 1.8596649169921875e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20045, "finemath": 0.08233}}
{"step": 31300, "total_steps": 72479, "tokens": 16410214400, "loss": 4.497324347496033, "l_mix": 2.7865439355373383, "l_a": 2.879364252090454, "l_c": 2.8232370615005493, "z_loss": 25.927632808685303, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16628116369247437, "tokens_per_second": 1396088.2361300513, "eta_seconds": 15464.391858100891, "data_wait_seconds": 1.5527009963989258e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.29090399, "gate_a2b/L1_absmax": 0.66681653, "gate_a2b/L1_rms": 0.30552745, "gate_a2b/L3_absmean": 0.30539116, "gate_a2b/L3_absmax": 0.61533356, "gate_a2b/L3_rms": 0.31088188, "gate_a2b/L5_absmean": 0.47368297, "gate_a2b/L5_absmax": 0.65527821, "gate_a2b/L5_rms": 0.47941378, "gate_c2b/L1_absmean": 0.0175382, "gate_c2b/L1_absmax": 0.53372389, "gate_c2b/L1_rms": 0.06417791, "gate_c2b/L3_absmean": 0.21845017, "gate_c2b/L3_absmax": 0.47415709, "gate_c2b/L3_rms": 0.22272006, "gate_c2b/L5_absmean": 0.57767421, "gate_c2b/L5_absmax": 0.80986172, "gate_c2b/L5_rms": 0.59095699, "gate_b2a/L7_absmean": 0.07235638, "gate_b2a/L7_absmax": 0.32192031, "gate_b2a/L7_rms": 0.07719158, "gate_b2a/L11_absmean": 0.11228891, "gate_b2a/L11_absmax": 0.5675053, "gate_b2a/L11_rms": 0.11801136, "gate_b2a/L14_absmean": 0.2846756, "gate_b2a/L14_absmax": 0.68371224, "gate_b2a/L14_rms": 0.28925842, "gate_b2c/L3_absmean": 0.07016995, "gate_b2c/L3_absmax": 0.68308735, "gate_b2c/L3_rms": 0.08402304, "gate_b2c/L5_absmean": 0.08158673, "gate_b2c/L5_absmax": 0.79260641, "gate_b2c/L5_rms": 0.0930928, "gate_b2c/L6_absmean": 0.11964377, "gate_b2c/L6_absmax": 0.54976773, "gate_b2c/L6_rms": 0.12806889}, "streams": {"cos_a2b/round0": 0.407067, "cos_c2b/round0": 0.3047, "cos_b2a/round0": 0.122166, "cos_b2c/round0": 0.141041, "rms_a/round0": 1.864059, "rms_b/round0": 3.524534, "rms_c/round0": 2.021961, "cos_a2b/round1": 0.428359, "cos_c2b/round1": 0.283382, "cos_b2a/round1": 0.240264, "cos_b2c/round1": 0.114186, "rms_a/round1": 3.344839, "rms_b/round1": 7.412236, "rms_c/round1": 3.720058, "cos_a2b/round2": 0.559764, "cos_c2b/round2": 0.517746, "cos_b2a/round2": 0.095506, "cos_b2c/round2": 0.160143, "rms_a/round2": 8.077916, "rms_b/round2": 17.681358, "rms_c/round2": 6.405005}, "alpha_mean": 0.44596315578839096, "alpha_hist": [0.0061, 0.0472, 0.15868, 0.21721, 0.20241, 0.16255, 0.12794, 0.06956, 0.00833, 2e-05], "alpha_by_type": {"special": {"share": 0.00066, "mean": 0.55578, "std": 0.10221}, "whitespace": {"share": 0.03291, "mean": 0.53101, "std": 0.11618}, "digit": {"share": 0.08067, "mean": 0.43465, "std": 0.10431}, "word_start": {"share": 0.50679, "mean": 0.4656, "std": 0.16222}, "word_cont": {"share": 0.25024, "mean": 0.3304, "std": 0.12135}, "punct": {"share": 0.12872, "mean": 0.5781, "std": 0.14623}}, "alpha_p05": 0.19652, "alpha_p50": 0.43818, "alpha_p95": 0.7235}
{"step": 31310, "total_steps": 72479, "tokens": 16415457280, "loss": 4.553795874118805, "l_mix": 2.820721387863159, "l_a": 2.9126907885074615, "l_c": 2.8642241060733795, "z_loss": 26.06428599357605, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14710620045661926, "tokens_per_second": 1415393.119544937, "eta_seconds": 15249.765152835846, "data_wait_seconds": 2.0503997802734375e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 31320, "total_steps": 72479, "tokens": 16420700160, "loss": 4.627672731876373, "l_mix": 2.866964817047119, "l_a": 2.9591686725616455, "l_c": 2.9098576307296753, "z_loss": 26.373288869857788, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14012140035629272, "tokens_per_second": 1471352.9837853129, "eta_seconds": 14666.208605146408, "data_wait_seconds": 1.5527009963989258e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 31330, "total_steps": 72479, "tokens": 16425943040, "loss": 4.651672065258026, "l_mix": 2.882415384054184, "l_a": 2.972260892391205, "l_c": 2.9252611100673676, "z_loss": 26.09906840324402, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12051291018724442, "tokens_per_second": 1376648.4427334704, "eta_seconds": 15671.34080300331, "data_wait_seconds": 2.804398536682129e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 31340, "total_steps": 72479, "tokens": 16431185920, "loss": 4.580929636955261, "l_mix": 2.838152676820755, "l_a": 2.9293723106384277, "l_c": 2.8798839151859283, "z_loss": 25.887622356414795, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12734685838222504, "tokens_per_second": 1445040.072694144, "eta_seconds": 14926.01100797653, "data_wait_seconds": 2.0056962966918945e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 31350, "total_steps": 72479, "tokens": 16436428800, "loss": 3.3626244962215424, "l_mix": 2.0682581961154938, "l_a": 2.187633067369461, "l_c": 2.1269207894802094, "z_loss": 26.287970304489136, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17065609991550446, "tokens_per_second": 1402250.2310767886, "eta_seconds": 15377.741200613975, "data_wait_seconds": 2.810359001159668e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20045, "finemath": 0.08233}}
{"step": 31360, "total_steps": 72479, "tokens": 16441671680, "loss": 3.2914372384548187, "l_mix": 2.02418452501297, "l_a": 2.1443810760974884, "l_c": 2.079794406890869, "z_loss": 26.225457668304443, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1711721122264862, "tokens_per_second": 1323067.7371942357, "eta_seconds": 16294.100193023682, "data_wait_seconds": 2.110004425048828e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20045, "finemath": 0.08233}}
{"step": 31370, "total_steps": 72479, "tokens": 16446914560, "loss": 3.302471935749054, "l_mix": 2.030907854437828, "l_a": 2.1554584205150604, "l_c": 2.0830882638692856, "z_loss": 26.98089337348938, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16088725626468658, "tokens_per_second": 1468580.9187556102, "eta_seconds": 14676.042100739478, "data_wait_seconds": 3.466010093688965e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20045, "finemath": 0.08234}}
{"step": 31380, "total_steps": 72479, "tokens": 16452157440, "loss": 3.2868541479110718, "l_mix": 2.0210607796907425, "l_a": 2.142062187194824, "l_c": 2.0772488713264465, "z_loss": 27.08468222618103, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15225516259670258, "tokens_per_second": 1424288.0419828263, "eta_seconds": 15128.760388946534, "data_wait_seconds": 2.148747444152832e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20045, "finemath": 0.08235}}
{"step": 31390, "total_steps": 72479, "tokens": 16457400320, "loss": 3.303595781326294, "l_mix": 2.032595708966255, "l_a": 2.151284247636795, "l_c": 2.0853825211524963, "z_loss": 26.04921793937683, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1803206354379654, "tokens_per_second": 1279122.7878881441, "eta_seconds": 16841.596315836905, "data_wait_seconds": 1.665949821472168e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20045, "finemath": 0.08235}}
{"step": 31400, "total_steps": 72479, "tokens": 16462643200, "loss": 4.744318246841431, "l_mix": 2.9417138397693634, "l_a": 3.020800828933716, "l_c": 2.9878802597522736, "z_loss": 26.120481252670288, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1606181412935257, "tokens_per_second": 1466526.4415135568, "eta_seconds": 14685.876873636245, "data_wait_seconds": 1.7762184143066406e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20768, "cosmopedia_v2": 0.20041, "finemath": 0.08236}, "gates": {"gate_a2b/L1_absmean": 0.28995809, "gate_a2b/L1_absmax": 0.67141527, "gate_a2b/L1_rms": 0.3044087, "gate_a2b/L3_absmean": 0.30339894, "gate_a2b/L3_absmax": 0.6176722, "gate_a2b/L3_rms": 0.30893031, "gate_a2b/L5_absmean": 0.47198668, "gate_a2b/L5_absmax": 0.64948171, "gate_a2b/L5_rms": 0.4776473, "gate_c2b/L1_absmean": 0.01645953, "gate_c2b/L1_absmax": 0.53645903, "gate_c2b/L1_rms": 0.0642119, "gate_c2b/L3_absmean": 0.21857877, "gate_c2b/L3_absmax": 0.47115606, "gate_c2b/L3_rms": 0.22287399, "gate_c2b/L5_absmean": 0.57859612, "gate_c2b/L5_absmax": 0.79773599, "gate_c2b/L5_rms": 0.59187198, "gate_b2a/L7_absmean": 0.07166526, "gate_b2a/L7_absmax": 0.32576832, "gate_b2a/L7_rms": 0.07662613, "gate_b2a/L11_absmean": 0.11170922, "gate_b2a/L11_absmax": 0.57050908, "gate_b2a/L11_rms": 0.11751298, "gate_b2a/L14_absmean": 0.28264511, "gate_b2a/L14_absmax": 0.68678093, "gate_b2a/L14_rms": 0.28733221, "gate_b2c/L3_absmean": 0.06911649, "gate_b2c/L3_absmax": 0.68244994, "gate_b2c/L3_rms": 0.08306541, "gate_b2c/L5_absmean": 0.07970512, "gate_b2c/L5_absmax": 0.78902757, "gate_b2c/L5_rms": 0.09128091, "gate_b2c/L6_absmean": 0.11733659, "gate_b2c/L6_absmax": 0.5407244, "gate_b2c/L6_rms": 0.12576337}, "streams": {"cos_a2b/round0": 0.397912, "cos_c2b/round0": 0.29617, "cos_b2a/round0": 0.12933, "cos_b2c/round0": 0.138046, "rms_a/round0": 1.904813, "rms_b/round0": 3.616984, "rms_c/round0": 2.03668, "cos_a2b/round1": 0.434162, "cos_c2b/round1": 0.300743, "cos_b2a/round1": 0.247519, "cos_b2c/round1": 0.128176, "rms_a/round1": 3.403025, "rms_b/round1": 7.487803, "rms_c/round1": 3.751215, "cos_a2b/round2": 0.570175, "cos_c2b/round2": 0.52209, "cos_b2a/round2": 0.098036, "cos_b2c/round2": 0.167843, "rms_a/round2": 8.222288, "rms_b/round2": 17.857697, "rms_c/round2": 6.479379}, "alpha_mean": 0.45325452993019155, "alpha_hist": [0.00545, 0.05315, 0.14938, 0.18498, 0.20248, 0.19128, 0.14062, 0.0658, 0.00676, 9e-05], "alpha_by_type": {"special": {"share": 0.00046, "mean": 0.54211, "std": 0.16753}, "whitespace": {"share": 0.04199, "mean": 0.54184, "std": 0.11276}, "digit": {"share": 0.02788, "mean": 0.49721, "std": 0.12097}, "word_start": {"share": 0.59801, "mean": 0.46876, "std": 0.15573}, "word_cont": {"share": 0.23672, "mean": 0.33651, "std": 0.12956}, "punct": {"share": 0.09494, "mean": 0.59416, "std": 0.14745}}, "alpha_p05": 0.18243, "alpha_p50": 0.45375, "alpha_p95": 0.73791}
{"step": 31410, "total_steps": 72479, "tokens": 16467886080, "loss": 4.614525258541107, "l_mix": 2.859509527683258, "l_a": 2.9488691687583923, "l_c": 2.9011832773685455, "z_loss": 26.04605484008789, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12886416912078857, "tokens_per_second": 1378944.8791282, "eta_seconds": 15614.825652503967, "data_wait_seconds": 2.73287296295166e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08233}}
{"step": 31420, "total_steps": 72479, "tokens": 16473128960, "loss": 4.551391541957855, "l_mix": 2.8197268545627594, "l_a": 2.9095559418201447, "l_c": 2.8626593351364136, "z_loss": 26.319591760635376, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1353706419467926, "tokens_per_second": 1346913.096819365, "eta_seconds": 15982.279066729545, "data_wait_seconds": 1.5944242477416992e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08234}}
{"step": 31430, "total_steps": 72479, "tokens": 16478371840, "loss": 4.611637353897095, "l_mix": 2.857621192932129, "l_a": 2.94533309340477, "l_c": 2.9013871252536774, "z_loss": 25.92599868774414, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12998977303504944, "tokens_per_second": 1364400.6721243358, "eta_seconds": 15773.590963196753, "data_wait_seconds": 1.7970800399780273e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08234}}
{"step": 31440, "total_steps": 72479, "tokens": 16483614720, "loss": 4.689046382904053, "l_mix": 2.9064362347126007, "l_a": 2.9942069053649902, "l_c": 2.9478265941143036, "z_loss": 26.230628490447998, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1212896779179573, "tokens_per_second": 1380224.3468824239, "eta_seconds": 15588.954998946188, "data_wait_seconds": 1.621246337890625e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 31450, "total_steps": 72479, "tokens": 16488857600, "loss": 4.5763267278671265, "l_mix": 2.8353550136089325, "l_a": 2.9256105720996857, "l_c": 2.8776284754276276, "z_loss": 26.59856605529785, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18348948657512665, "tokens_per_second": 1482274.3736695312, "eta_seconds": 14512.166393828393, "data_wait_seconds": 2.2649765014648438e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 31460, "total_steps": 72479, "tokens": 16494100480, "loss": 3.3103047907352448, "l_mix": 2.03688682615757, "l_a": 2.1542475521564484, "l_c": 2.090478867292404, "z_loss": 26.02965807914734, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18292923271656036, "tokens_per_second": 1449501.4305366578, "eta_seconds": 14836.666607522966, "data_wait_seconds": 2.8789043426513672e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20045, "finemath": 0.08233}}
{"step": 31470, "total_steps": 72479, "tokens": 16499343360, "loss": 3.3036484122276306, "l_mix": 2.0323283672332764, "l_a": 2.1495417952537537, "l_c": 2.0881913900375366, "z_loss": 26.328338623046875, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1634804755449295, "tokens_per_second": 1433732.998024612, "eta_seconds": 14996.18591580391, "data_wait_seconds": 5.1528215408325195e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 31480, "total_steps": 72479, "tokens": 16504586240, "loss": 3.2954567074775696, "l_mix": 2.0273326486349106, "l_a": 2.145268201828003, "l_c": 2.0818119943141937, "z_loss": 26.116645574569702, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18256928026676178, "tokens_per_second": 1381542.1236798214, "eta_seconds": 15558.905764484405, "data_wait_seconds": 1.8209218978881836e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 31490, "total_steps": 72479, "tokens": 16509829120, "loss": 3.3003082871437073, "l_mix": 2.0292771607637405, "l_a": 2.15068456530571, "l_c": 2.0860857367515564, "z_loss": 26.724474668502808, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15243473649024963, "tokens_per_second": 1337462.2032872394, "eta_seconds": 16067.774311065672, "data_wait_seconds": 2.8371810913085938e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 31500, "total_steps": 72479, "tokens": 16515072000, "loss": 3.307264119386673, "l_mix": 2.034361854195595, "l_a": 2.1542137265205383, "l_c": 2.088793784379959, "z_loss": 26.46008014678955, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15104396641254425, "tokens_per_second": 1426722.0564303806, "eta_seconds": 15058.853162860869, "data_wait_seconds": 1.6033649444580078e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.28837526, "gate_a2b/L1_absmax": 0.67110831, "gate_a2b/L1_rms": 0.30283353, "gate_a2b/L3_absmean": 0.30228588, "gate_a2b/L3_absmax": 0.61780477, "gate_a2b/L3_rms": 0.30770829, "gate_a2b/L5_absmean": 0.46969506, "gate_a2b/L5_absmax": 0.65194064, "gate_a2b/L5_rms": 0.4754256, "gate_c2b/L1_absmean": 0.01662568, "gate_c2b/L1_absmax": 0.53536797, "gate_c2b/L1_rms": 0.06410135, "gate_c2b/L3_absmean": 0.21845925, "gate_c2b/L3_absmax": 0.47395867, "gate_c2b/L3_rms": 0.22277509, "gate_c2b/L5_absmean": 0.58001578, "gate_c2b/L5_absmax": 0.80781537, "gate_c2b/L5_rms": 0.59331173, "gate_b2a/L7_absmean": 0.0710701, "gate_b2a/L7_absmax": 0.32528785, "gate_b2a/L7_rms": 0.07603516, "gate_b2a/L11_absmean": 0.11044765, "gate_b2a/L11_absmax": 0.5697102, "gate_b2a/L11_rms": 0.11634071, "gate_b2a/L14_absmean": 0.28086877, "gate_b2a/L14_absmax": 0.68963516, "gate_b2a/L14_rms": 0.28565031, "gate_b2c/L3_absmean": 0.06778397, "gate_b2c/L3_absmax": 0.67940581, "gate_b2c/L3_rms": 0.0815339, "gate_b2c/L5_absmean": 0.07892271, "gate_b2c/L5_absmax": 0.7816841, "gate_b2c/L5_rms": 0.09039766, "gate_b2c/L6_absmean": 0.11581844, "gate_b2c/L6_absmax": 0.54358196, "gate_b2c/L6_rms": 0.12423792}, "streams": {"cos_a2b/round0": 0.391105, "cos_c2b/round0": 0.266442, "cos_b2a/round0": 0.11529, "cos_b2c/round0": 0.106874, "rms_a/round0": 2.011852, "rms_b/round0": 3.697336, "rms_c/round0": 2.094162, "cos_a2b/round1": 0.430365, "cos_c2b/round1": 0.290388, "cos_b2a/round1": 0.231752, "cos_b2c/round1": 0.12632, "rms_a/round1": 3.697165, "rms_b/round1": 8.169839, "rms_c/round1": 3.990001, "cos_a2b/round2": 0.522244, "cos_c2b/round2": 0.502841, "cos_b2a/round2": 0.063023, "cos_b2c/round2": 0.144627, "rms_a/round2": 8.620677, "rms_b/round2": 19.257679, "rms_c/round2": 6.975771}, "alpha_mean": 0.4135183286095412, "alpha_hist": [0.03531, 0.15865, 0.19714, 0.15144, 0.11095, 0.10156, 0.09833, 0.09175, 0.05138, 0.00349], "alpha_by_type": {"special": {"share": 0.00107, "mean": 0.5889, "std": 0.11276}, "whitespace": {"share": 0.03757, "mean": 0.61236, "std": 0.16841}, "digit": {"share": 0.00903, "mean": 0.36223, "std": 0.17486}, "word_start": {"share": 0.55759, "mean": 0.42505, "std": 0.21263}, "word_cont": {"share": 0.27252, "mean": 0.26619, "std": 0.13863}, "punct": {"share": 0.12222, "mean": 0.63056, "std": 0.18717}}, "alpha_p05": 0.1255, "alpha_p50": 0.36388, "alpha_p95": 0.81785}
{"step": 31510, "total_steps": 72479, "tokens": 16520314880, "loss": 4.489514470100403, "l_mix": 2.7815033197402954, "l_a": 2.872655898332596, "l_c": 2.820714443922043, "z_loss": 26.12492036819458, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15197482705116272, "tokens_per_second": 1442825.0724993888, "eta_seconds": 14887.15124335289, "data_wait_seconds": 1.9788742065429688e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 31520, "total_steps": 72479, "tokens": 16525557760, "loss": 4.550472617149353, "l_mix": 2.8180915117263794, "l_a": 2.91454941034317, "l_c": 2.860054075717926, "z_loss": 26.36818027496338, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18473191559314728, "tokens_per_second": 1456277.3768821226, "eta_seconds": 14746.03158223629, "data_wait_seconds": 2.1129846572875977e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 31530, "total_steps": 72479, "tokens": 16530800640, "loss": 4.6283605098724365, "l_mix": 2.868312180042267, "l_a": 2.9563629627227783, "l_c": 2.9104648530483246, "z_loss": 25.718477487564087, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17998114228248596, "tokens_per_second": 1454599.8044361186, "eta_seconds": 14759.433657646181, "data_wait_seconds": 1.8298625946044922e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 31540, "total_steps": 72479, "tokens": 16536043520, "loss": 4.706956088542938, "l_mix": 2.917099267244339, "l_a": 3.0059918463230133, "l_c": 2.9601976573467255, "z_loss": 26.038730144500732, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13027696311473846, "tokens_per_second": 1365268.5170745542, "eta_seconds": 15721.322335910796, "data_wait_seconds": 1.77919864654541e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08234}}
{"step": 31550, "total_steps": 72479, "tokens": 16541286400, "loss": 4.600174725055695, "l_mix": 2.8499611020088196, "l_a": 2.942155569791794, "l_c": 2.8918894827365875, "z_loss": 25.947999000549316, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13072791695594788, "tokens_per_second": 1357265.6771335495, "eta_seconds": 15810.157077956199, "data_wait_seconds": 2.1219253540039062e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 31560, "total_steps": 72479, "tokens": 16546529280, "loss": 4.618997931480408, "l_mix": 2.8624207079410553, "l_a": 2.9516128301620483, "l_c": 2.903644323348999, "z_loss": 25.674499988555908, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17218227684497833, "tokens_per_second": 1440286.1361461428, "eta_seconds": 14895.193485236168, "data_wait_seconds": 2.4050474166870117e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 31570, "total_steps": 72479, "tokens": 16551772160, "loss": 3.321781039237976, "l_mix": 2.0433685034513474, "l_a": 2.1652619540691376, "l_c": 2.0961129665374756, "z_loss": 26.761162996292114, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1852228194475174, "tokens_per_second": 1434772.8467066798, "eta_seconds": 14948.775927305222, "data_wait_seconds": 2.136826515197754e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08232}}
{"step": 31580, "total_steps": 72479, "tokens": 16557015040, "loss": 3.289379894733429, "l_mix": 2.022544115781784, "l_a": 2.144396871328354, "l_c": 2.07838898897171, "z_loss": 26.721654653549194, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1563541293144226, "tokens_per_second": 1480440.7206916567, "eta_seconds": 14484.102343511582, "data_wait_seconds": 1.5676021575927734e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 31590, "total_steps": 72479, "tokens": 16562257920, "loss": 3.2816597521305084, "l_mix": 2.018609717488289, "l_a": 2.1386740505695343, "l_c": 2.0714928805828094, "z_loss": 27.25421643257141, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16415882110595703, "tokens_per_second": 1455556.360877521, "eta_seconds": 14728.12225496769, "data_wait_seconds": 3.567337989807129e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 31600, "total_steps": 72479, "tokens": 16567500800, "loss": 3.329852521419525, "l_mix": 2.047984689474106, "l_a": 2.1693834960460663, "l_c": 2.1035090684890747, "z_loss": 27.46010708808899, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18606793880462646, "tokens_per_second": 1420639.4279877478, "eta_seconds": 15086.42427470684, "data_wait_seconds": 2.4318695068359375e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.28782088, "gate_a2b/L1_absmax": 0.67442721, "gate_a2b/L1_rms": 0.30218551, "gate_a2b/L3_absmean": 0.30011344, "gate_a2b/L3_absmax": 0.61961544, "gate_a2b/L3_rms": 0.30569872, "gate_a2b/L5_absmean": 0.46737152, "gate_a2b/L5_absmax": 0.64775562, "gate_a2b/L5_rms": 0.47308472, "gate_c2b/L1_absmean": 0.01674867, "gate_c2b/L1_absmax": 0.54008204, "gate_c2b/L1_rms": 0.06438661, "gate_c2b/L3_absmean": 0.21844414, "gate_c2b/L3_absmax": 0.47265026, "gate_c2b/L3_rms": 0.22273418, "gate_c2b/L5_absmean": 0.58055753, "gate_c2b/L5_absmax": 0.81396466, "gate_c2b/L5_rms": 0.59385425, "gate_b2a/L7_absmean": 0.0707048, "gate_b2a/L7_absmax": 0.32391918, "gate_b2a/L7_rms": 0.07561488, "gate_b2a/L11_absmean": 0.11008105, "gate_b2a/L11_absmax": 0.5725832, "gate_b2a/L11_rms": 0.11604541, "gate_b2a/L14_absmean": 0.27877459, "gate_b2a/L14_absmax": 0.69329977, "gate_b2a/L14_rms": 0.28363246, "gate_b2c/L3_absmean": 0.06704772, "gate_b2c/L3_absmax": 0.67849046, "gate_b2c/L3_rms": 0.08098832, "gate_b2c/L5_absmean": 0.07738344, "gate_b2c/L5_absmax": 0.78462416, "gate_b2c/L5_rms": 0.08914734, "gate_b2c/L6_absmean": 0.11386521, "gate_b2c/L6_absmax": 0.54509008, "gate_b2c/L6_rms": 0.12241686}, "streams": {"cos_a2b/round0": 0.395722, "cos_c2b/round0": 0.26883, "cos_b2a/round0": 0.11477, "cos_b2c/round0": 0.108517, "rms_a/round0": 2.060821, "rms_b/round0": 3.774442, "rms_c/round0": 2.113392, "cos_a2b/round1": 0.429944, "cos_c2b/round1": 0.291376, "cos_b2a/round1": 0.22723, "cos_b2c/round1": 0.126832, "rms_a/round1": 3.800225, "rms_b/round1": 8.311294, "rms_c/round1": 4.074965, "cos_a2b/round2": 0.52963, "cos_c2b/round2": 0.494855, "cos_b2a/round2": 0.064497, "cos_b2c/round2": 0.144098, "rms_a/round2": 8.833665, "rms_b/round2": 19.53908, "rms_c/round2": 7.094718}, "alpha_mean": 0.427781632839622, "alpha_hist": [0.02925, 0.14166, 0.1841, 0.1575, 0.12212, 0.10611, 0.10255, 0.09804, 0.05548, 0.00319], "alpha_by_type": {"special": {"share": 0.00108, "mean": 0.60784, "std": 0.10379}, "whitespace": {"share": 0.03867, "mean": 0.60115, "std": 0.16943}, "digit": {"share": 0.01009, "mean": 0.37006, "std": 0.17449}, "word_start": {"share": 0.56856, "mean": 0.43948, "std": 0.21302}, "word_cont": {"share": 0.26094, "mean": 0.28303, "std": 0.14355}, "punct": {"share": 0.12067, "mean": 0.63336, "std": 0.18587}}, "alpha_p05": 0.11517, "alpha_p50": 0.3928, "alpha_p95": 0.80837}
{"step": 31610, "total_steps": 72479, "tokens": 16572743680, "loss": 3.3261934518814087, "l_mix": 2.0469950437545776, "l_a": 2.163240283727646, "l_c": 2.10075443983078, "z_loss": 25.77287769317627, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20241957902908325, "tokens_per_second": 1429391.631323561, "eta_seconds": 14990.381783723831, "data_wait_seconds": 2.664327621459961e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 31620, "total_steps": 72479, "tokens": 16577986560, "loss": 4.539598882198334, "l_mix": 2.8116718530654907, "l_a": 2.9065274596214294, "l_c": 2.853228807449341, "z_loss": 26.960559368133545, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17795708775520325, "tokens_per_second": 1441467.2208327372, "eta_seconds": 14861.165819382666, "data_wait_seconds": 2.0503997802734375e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 31630, "total_steps": 72479, "tokens": 16583229440, "loss": 4.634527623653412, "l_mix": 2.8711273968219757, "l_a": 2.961406797170639, "l_c": 2.916593611240387, "z_loss": 26.262630701065063, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1437319815158844, "tokens_per_second": 1463068.590945451, "eta_seconds": 14638.165732312202, "data_wait_seconds": 1.6629695892333984e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 31640, "total_steps": 72479, "tokens": 16588472320, "loss": 4.6356483697891235, "l_mix": 2.8722048103809357, "l_a": 2.964087724685669, "l_c": 2.9140570759773254, "z_loss": 25.575361013412476, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1687995046377182, "tokens_per_second": 1316594.2560767308, "eta_seconds": 16262.715360617636, "data_wait_seconds": 1.9282102584838867e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 31650, "total_steps": 72479, "tokens": 16593715200, "loss": 4.545573592185974, "l_mix": 2.816186398267746, "l_a": 2.905909389257431, "l_c": 2.858714073896408, "z_loss": 26.43168020248413, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15647368133068085, "tokens_per_second": 1351639.083575036, "eta_seconds": 15837.182434368133, "data_wait_seconds": 1.6421079635620117e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 31660, "total_steps": 72479, "tokens": 16598958080, "loss": 3.3197214007377625, "l_mix": 2.0417188853025436, "l_a": 2.1617604196071625, "l_c": 2.098247766494751, "z_loss": 27.24414873123169, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15346579253673553, "tokens_per_second": 1385289.7738864364, "eta_seconds": 15448.689707684516, "data_wait_seconds": 2.276897430419922e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 31670, "total_steps": 72479, "tokens": 16604200960, "loss": 3.2933057844638824, "l_mix": 2.025997072458267, "l_a": 2.146110862493515, "l_c": 2.0782515704631805, "z_loss": 26.364700078964233, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1751125156879425, "tokens_per_second": 1458244.0351566386, "eta_seconds": 14672.214304447174, "data_wait_seconds": 4.7653913497924805e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 31680, "total_steps": 72479, "tokens": 16609443840, "loss": 4.76769745349884, "l_mix": 2.9564853608608246, "l_a": 3.0399847626686096, "l_c": 2.9973886609077454, "z_loss": 26.852470636367798, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15512967109680176, "tokens_per_second": 1305075.4289133407, "eta_seconds": 16390.18376877308, "data_wait_seconds": 2.574920654296875e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 31690, "total_steps": 72479, "tokens": 16614686720, "loss": 3.323293000459671, "l_mix": 2.043701931834221, "l_a": 2.1649232506752014, "l_c": 2.1003802567720413, "z_loss": 26.66340470314026, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2070332169532776, "tokens_per_second": 1469154.2047443492, "eta_seconds": 14556.118862771988, "data_wait_seconds": 2.6613473892211914e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 31700, "total_steps": 72479, "tokens": 16619929600, "loss": 3.2877380549907684, "l_mix": 2.022873356938362, "l_a": 2.139954149723053, "l_c": 2.076261192560196, "z_loss": 26.49920678138733, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16635257005691528, "tokens_per_second": 1456246.6130883067, "eta_seconds": 14681.538250350954, "data_wait_seconds": 1.704692840576172e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.28597856, "gate_a2b/L1_absmax": 0.67338675, "gate_a2b/L1_rms": 0.30041218, "gate_a2b/L3_absmean": 0.29810128, "gate_a2b/L3_absmax": 0.6201365, "gate_a2b/L3_rms": 0.30367851, "gate_a2b/L5_absmean": 0.46510062, "gate_a2b/L5_absmax": 0.64333344, "gate_a2b/L5_rms": 0.47089112, "gate_c2b/L1_absmean": 0.01672137, "gate_c2b/L1_absmax": 0.54138482, "gate_c2b/L1_rms": 0.06457596, "gate_c2b/L3_absmean": 0.21842308, "gate_c2b/L3_absmax": 0.46841866, "gate_c2b/L3_rms": 0.22268096, "gate_c2b/L5_absmean": 0.58168709, "gate_c2b/L5_absmax": 0.80691916, "gate_c2b/L5_rms": 0.59497219, "gate_b2a/L7_absmean": 0.06973557, "gate_b2a/L7_absmax": 0.3253119, "gate_b2a/L7_rms": 0.07477213, "gate_b2a/L11_absmean": 0.10882791, "gate_b2a/L11_absmax": 0.57152683, "gate_b2a/L11_rms": 0.11490443, "gate_b2a/L14_absmean": 0.27763936, "gate_b2a/L14_absmax": 0.69442087, "gate_b2a/L14_rms": 0.28254691, "gate_b2c/L3_absmean": 0.06591851, "gate_b2c/L3_absmax": 0.67761505, "gate_b2c/L3_rms": 0.07960825, "gate_b2c/L5_absmean": 0.07626504, "gate_b2c/L5_absmax": 0.78212112, "gate_b2c/L5_rms": 0.08793631, "gate_b2c/L6_absmean": 0.11229274, "gate_b2c/L6_absmax": 0.5415321, "gate_b2c/L6_rms": 0.12085033}, "streams": {"cos_a2b/round0": 0.396115, "cos_c2b/round0": 0.265608, "cos_b2a/round0": 0.114498, "cos_b2c/round0": 0.105349, "rms_a/round0": 2.104117, "rms_b/round0": 3.85493, "rms_c/round0": 2.149197, "cos_a2b/round1": 0.429127, "cos_c2b/round1": 0.293178, "cos_b2a/round1": 0.233794, "cos_b2c/round1": 0.124288, "rms_a/round1": 3.835273, "rms_b/round1": 8.50649, "rms_c/round1": 4.088682, "cos_a2b/round2": 0.523038, "cos_c2b/round2": 0.505967, "cos_b2a/round2": 0.06356, "cos_b2c/round2": 0.148015, "rms_a/round2": 9.010304, "rms_b/round2": 20.042103, "rms_c/round2": 7.189446}, "alpha_mean": 0.40501187109908443, "alpha_hist": [0.03882, 0.16411, 0.1972, 0.15515, 0.11562, 0.09796, 0.09325, 0.09138, 0.0443, 0.00221], "alpha_by_type": {"special": {"share": 0.00104, "mean": 0.60246, "std": 0.09281}, "whitespace": {"share": 0.03108, "mean": 0.59264, "std": 0.17384}, "digit": {"share": 0.00883, "mean": 0.36247, "std": 0.18443}, "word_start": {"share": 0.56918, "mean": 0.41449, "std": 0.21188}, "word_cont": {"share": 0.27187, "mean": 0.27145, "std": 0.14197}, "punct": {"share": 0.118, "mean": 0.61907, "std": 0.19057}}, "alpha_p05": 0.09912, "alpha_p50": 0.35265, "alpha_p95": 0.801}
{"step": 31710, "total_steps": 72479, "tokens": 16625172480, "loss": 3.324285387992859, "l_mix": 2.044902056455612, "l_a": 2.160100996494293, "l_c": 2.1045098900794983, "z_loss": 26.840410709381104, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1916443556547165, "tokens_per_second": 1236778.1743704928, "eta_seconds": 17282.56361160278, "data_wait_seconds": 1.8268823623657227e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 31720, "total_steps": 72479, "tokens": 16630415360, "loss": 3.3120969235897064, "l_mix": 2.037399783730507, "l_a": 2.1542890071868896, "l_c": 2.0947013199329376, "z_loss": 27.298508167266846, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19993358850479126, "tokens_per_second": 1397383.657476737, "eta_seconds": 15292.474960374831, "data_wait_seconds": 7.292628288269043e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 31730, "total_steps": 72479, "tokens": 16635658240, "loss": 4.5509490966796875, "l_mix": 2.8186499178409576, "l_a": 2.912544459104538, "l_c": 2.8617859482765198, "z_loss": 26.156229734420776, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17668741941452026, "tokens_per_second": 1371490.118890533, "eta_seconds": 15577.371953129768, "data_wait_seconds": 3.0368566513061523e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 31740, "total_steps": 72479, "tokens": 16640901120, "loss": 4.579576373100281, "l_mix": 2.837669312953949, "l_a": 2.9282405078411102, "l_c": 2.878116339445114, "z_loss": 26.4038667678833, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1203494444489479, "tokens_per_second": 1286907.418349982, "eta_seconds": 16597.129309725762, "data_wait_seconds": 2.065300941467285e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 31750, "total_steps": 72479, "tokens": 16646144000, "loss": 4.605640470981598, "l_mix": 2.8536826968193054, "l_a": 2.9437651932239532, "l_c": 2.896093726158142, "z_loss": 26.797640562057495, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17145821452140808, "tokens_per_second": 1429385.5920574062, "eta_seconds": 14939.094160914421, "data_wait_seconds": 2.300739288330078e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 31760, "total_steps": 72479, "tokens": 16651386880, "loss": 4.588127315044403, "l_mix": 2.841968446969986, "l_a": 2.934820294380188, "l_c": 2.8857091069221497, "z_loss": 27.596542358398438, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17472225427627563, "tokens_per_second": 1281193.7404017814, "eta_seconds": 16662.96235985756, "data_wait_seconds": 1.9431114196777344e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 31770, "total_steps": 72479, "tokens": 16656629760, "loss": 3.2888692915439606, "l_mix": 2.021601527929306, "l_a": 2.143899977207184, "l_c": 2.0803258419036865, "z_loss": 28.105695486068726, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18054284155368805, "tokens_per_second": 1297242.1393072305, "eta_seconds": 16452.780514359474, "data_wait_seconds": 2.6285648345947266e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 31780, "total_steps": 72479, "tokens": 16661872640, "loss": 3.3382769227027893, "l_mix": 2.0550440549850464, "l_a": 2.1703093349933624, "l_c": 2.1071333587169647, "z_loss": 26.50136685371399, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18980750441551208, "tokens_per_second": 1387917.2168533655, "eta_seconds": 15374.113854122163, "data_wait_seconds": 3.382563591003418e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 31790, "total_steps": 72479, "tokens": 16667115520, "loss": 3.2927451729774475, "l_mix": 2.025606796145439, "l_a": 2.1434969902038574, "l_c": 2.0802973210811615, "z_loss": 26.91618537902832, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.21209774911403656, "tokens_per_second": 1337485.2244430352, "eta_seconds": 15949.899140667914, "data_wait_seconds": 2.5540590286254883e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 31800, "total_steps": 72479, "tokens": 16672358400, "loss": 4.861476957798004, "l_mix": 3.0147309601306915, "l_a": 3.0971370935440063, "l_c": 3.058682471513748, "z_loss": 26.21329355239868, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14301817119121552, "tokens_per_second": 1436967.9627179042, "eta_seconds": 14842.022999358178, "data_wait_seconds": 1.901388168334961e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.28460798, "gate_a2b/L1_absmax": 0.67592424, "gate_a2b/L1_rms": 0.29901782, "gate_a2b/L3_absmean": 0.29705784, "gate_a2b/L3_absmax": 0.61856985, "gate_a2b/L3_rms": 0.30263752, "gate_a2b/L5_absmean": 0.46305442, "gate_a2b/L5_absmax": 0.64177376, "gate_a2b/L5_rms": 0.4687964, "gate_c2b/L1_absmean": 0.01658593, "gate_c2b/L1_absmax": 0.5415259, "gate_c2b/L1_rms": 0.06455033, "gate_c2b/L3_absmean": 0.21854046, "gate_c2b/L3_absmax": 0.47439712, "gate_c2b/L3_rms": 0.22280385, "gate_c2b/L5_absmean": 0.58247244, "gate_c2b/L5_absmax": 0.80381864, "gate_c2b/L5_rms": 0.59577304, "gate_b2a/L7_absmean": 0.06938943, "gate_b2a/L7_absmax": 0.32460883, "gate_b2a/L7_rms": 0.07442642, "gate_b2a/L11_absmean": 0.10821156, "gate_b2a/L11_absmax": 0.57141596, "gate_b2a/L11_rms": 0.11427548, "gate_b2a/L14_absmean": 0.27607888, "gate_b2a/L14_absmax": 0.69301265, "gate_b2a/L14_rms": 0.28102592, "gate_b2c/L3_absmean": 0.06538973, "gate_b2c/L3_absmax": 0.6753431, "gate_b2c/L3_rms": 0.078965, "gate_b2c/L5_absmean": 0.07542939, "gate_b2c/L5_absmax": 0.78267443, "gate_b2c/L5_rms": 0.08723332, "gate_b2c/L6_absmean": 0.11079823, "gate_b2c/L6_absmax": 0.53807127, "gate_b2c/L6_rms": 0.11937573}, "streams": {"cos_a2b/round0": 0.397839, "cos_c2b/round0": 0.306828, "cos_b2a/round0": 0.131895, "cos_b2c/round0": 0.140665, "rms_a/round0": 2.043345, "rms_b/round0": 3.898611, "rms_c/round0": 2.108336, "cos_a2b/round1": 0.427148, "cos_c2b/round1": 0.293756, "cos_b2a/round1": 0.245297, "cos_b2c/round1": 0.124857, "rms_a/round1": 3.676225, "rms_b/round1": 8.148792, "rms_c/round1": 3.980319, "cos_a2b/round2": 0.570796, "cos_c2b/round2": 0.517292, "cos_b2a/round2": 0.104613, "cos_b2c/round2": 0.168841, "rms_a/round2": 8.959725, "rms_b/round2": 19.358997, "rms_c/round2": 6.960006}, "alpha_mean": 0.4577172707663948, "alpha_hist": [0.00604, 0.04472, 0.14128, 0.19577, 0.20393, 0.19279, 0.13689, 0.06844, 0.00983, 0.00031], "alpha_by_type": {"special": {"share": 0.00072, "mean": 0.53057, "std": 0.15516}, "whitespace": {"share": 0.0696, "mean": 0.55379, "std": 0.11042}, "digit": {"share": 0.02718, "mean": 0.47342, "std": 0.11108}, "word_start": {"share": 0.55516, "mean": 0.47192, "std": 0.15781}, "word_cont": {"share": 0.2464, "mean": 0.34563, "std": 0.12662}, "punct": {"share": 0.10095, "mean": 0.58219, "std": 0.15661}}, "alpha_p05": 0.17838, "alpha_p50": 0.45241, "alpha_p95": 0.71661}
{"step": 31810, "total_steps": 72479, "tokens": 16677601280, "loss": 4.854948282241821, "l_mix": 3.010632663965225, "l_a": 3.094257891178131, "l_c": 3.0534606277942657, "z_loss": 26.259087085723877, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18616478145122528, "tokens_per_second": 1318355.076900535, "eta_seconds": 16173.388372826576, "data_wait_seconds": 2.9206275939941406e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 31820, "total_steps": 72479, "tokens": 16682844160, "loss": 3.266065537929535, "l_mix": 2.010665252804756, "l_a": 2.1255203783512115, "l_c": 2.0591471791267395, "z_loss": 26.67417860031128, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.24092620611190796, "tokens_per_second": 1468099.911956329, "eta_seconds": 14520.146495747567, "data_wait_seconds": 2.4765729904174805e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 31830, "total_steps": 72479, "tokens": 16688087040, "loss": 3.293113112449646, "l_mix": 2.0258000642061234, "l_a": 2.1455499827861786, "l_c": 2.0788265764713287, "z_loss": 27.387933254241943, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16101475059986115, "tokens_per_second": 1468483.1427744515, "eta_seconds": 14512.786896371841, "data_wait_seconds": 1.5944242477416992e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 31840, "total_steps": 72479, "tokens": 16693329920, "loss": 4.932827472686768, "l_mix": 3.051821917295456, "l_a": 3.1219295859336853, "l_c": 3.148088663816452, "z_loss": 26.352458000183105, "lr": 0.0023, "wd": 0.01, "grad_norm": 2.2266764640808105, "tokens_per_second": 1434301.191945822, "eta_seconds": 14854.997089624405, "data_wait_seconds": 1.9848346710205078e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 31850, "total_steps": 72479, "tokens": 16698572800, "loss": 4.6205397844314575, "l_mix": 2.862359404563904, "l_a": 2.9554405212402344, "l_c": 2.9051606953144073, "z_loss": 27.35341215133667, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13779591023921967, "tokens_per_second": 1466680.2031694097, "eta_seconds": 14523.47765107155, "data_wait_seconds": 1.5527009963989258e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08232}}
{"step": 31860, "total_steps": 72479, "tokens": 16703815680, "loss": 4.479682266712189, "l_mix": 2.7745796740055084, "l_a": 2.866550177335739, "l_c": 2.8171247839927673, "z_loss": 27.332205772399902, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15987971425056458, "tokens_per_second": 1433798.435208449, "eta_seconds": 14852.892672395707, "data_wait_seconds": 2.384185791015625e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 31870, "total_steps": 72479, "tokens": 16709058560, "loss": 4.46286803483963, "l_mix": 2.7633751034736633, "l_a": 2.858134776353836, "l_c": 2.806841015815735, "z_loss": 27.281131267547607, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1392178237438202, "tokens_per_second": 1448910.439801274, "eta_seconds": 14694.359849405288, "data_wait_seconds": 4.13358211517334e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 31880, "total_steps": 72479, "tokens": 16714301440, "loss": 3.348062187433243, "l_mix": 2.058621406555176, "l_a": 2.179590493440628, "l_c": 2.118545413017273, "z_loss": 27.552491426467896, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18922272324562073, "tokens_per_second": 1472870.0445057906, "eta_seconds": 14451.762795639039, "data_wait_seconds": 3.510713577270508e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 31890, "total_steps": 72479, "tokens": 16719544320, "loss": 3.3059093058109283, "l_mix": 2.034143939614296, "l_a": 2.1535531282424927, "l_c": 2.085664838552475, "z_loss": 27.60632300376892, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1594814956188202, "tokens_per_second": 1471265.174010894, "eta_seconds": 14463.963402318956, "data_wait_seconds": 3.784894943237305e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 31900, "total_steps": 72479, "tokens": 16724787200, "loss": 3.3157874941825867, "l_mix": 2.039068877696991, "l_a": 2.1611944437026978, "l_c": 2.0945341885089874, "z_loss": 26.720492362976074, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20316042006015778, "tokens_per_second": 1383072.5506349204, "eta_seconds": 15382.477761006356, "data_wait_seconds": 2.0295381546020508e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.28445825, "gate_a2b/L1_absmax": 0.68095177, "gate_a2b/L1_rms": 0.29883218, "gate_a2b/L3_absmean": 0.29528448, "gate_a2b/L3_absmax": 0.61973453, "gate_a2b/L3_rms": 0.30100676, "gate_a2b/L5_absmean": 0.46071443, "gate_a2b/L5_absmax": 0.64016289, "gate_a2b/L5_rms": 0.46643075, "gate_c2b/L1_absmean": 0.0178383, "gate_c2b/L1_absmax": 0.54547143, "gate_c2b/L1_rms": 0.06451701, "gate_c2b/L3_absmean": 0.21902589, "gate_c2b/L3_absmax": 0.48272547, "gate_c2b/L3_rms": 0.22336252, "gate_c2b/L5_absmean": 0.58296615, "gate_c2b/L5_absmax": 0.82150984, "gate_c2b/L5_rms": 0.59625304, "gate_b2a/L7_absmean": 0.06939219, "gate_b2a/L7_absmax": 0.32606134, "gate_b2a/L7_rms": 0.07433565, "gate_b2a/L11_absmean": 0.10833229, "gate_b2a/L11_absmax": 0.57379448, "gate_b2a/L11_rms": 0.11439542, "gate_b2a/L14_absmean": 0.27516395, "gate_b2a/L14_absmax": 0.70386755, "gate_b2a/L14_rms": 0.28023767, "gate_b2c/L3_absmean": 0.06471512, "gate_b2c/L3_absmax": 0.68200332, "gate_b2c/L3_rms": 0.07833148, "gate_b2c/L5_absmean": 0.07436261, "gate_b2c/L5_absmax": 0.78890473, "gate_b2c/L5_rms": 0.08648247, "gate_b2c/L6_absmean": 0.10980821, "gate_b2c/L6_absmax": 0.53243822, "gate_b2c/L6_rms": 0.11844208}, "streams": {"cos_a2b/round0": 0.393305, "cos_c2b/round0": 0.262027, "cos_b2a/round0": 0.112844, "cos_b2c/round0": 0.103274, "rms_a/round0": 2.186247, "rms_b/round0": 4.010148, "rms_c/round0": 2.198454, "cos_a2b/round1": 0.426766, "cos_c2b/round1": 0.288265, "cos_b2a/round1": 0.229593, "cos_b2c/round1": 0.124103, "rms_a/round1": 3.959589, "rms_b/round1": 8.850852, "rms_c/round1": 4.174516, "cos_a2b/round2": 0.520816, "cos_c2b/round2": 0.505838, "cos_b2a/round2": 0.068028, "cos_b2c/round2": 0.144377, "rms_a/round2": 9.317307, "rms_b/round2": 20.927557, "rms_c/round2": 7.322244}, "alpha_mean": 0.4060586973091276, "alpha_hist": [0.03571, 0.16252, 0.19658, 0.15468, 0.11734, 0.10115, 0.10367, 0.08882, 0.03662, 0.00291], "alpha_by_type": {"special": {"share": 0.00105, "mean": 0.57188, "std": 0.09059}, "whitespace": {"share": 0.03621, "mean": 0.61608, "std": 0.1871}, "digit": {"share": 0.01039, "mean": 0.31597, "std": 0.16027}, "word_start": {"share": 0.56834, "mean": 0.41559, "std": 0.20793}, "word_cont": {"share": 0.26976, "mean": 0.27391, "std": 0.14298}, "punct": {"share": 0.11424, "mean": 0.61078, "std": 0.18312}}, "alpha_p05": 0.11556, "alpha_p50": 0.37846, "alpha_p95": 0.7902}
{"step": 31910, "total_steps": 72479, "tokens": 16730030080, "loss": 3.2710762321949005, "l_mix": 2.024402067065239, "l_a": 2.0849864035844803, "l_c": 2.0705940276384354, "z_loss": 29.32891607284546, "lr": 0.0023, "wd": 0.01, "grad_norm": 1.2624419927597046, "tokens_per_second": 1475018.426167288, "eta_seconds": 14420.050281858443, "data_wait_seconds": 2.1845102310180664e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08236}}
{"step": 31920, "total_steps": 72479, "tokens": 16735272960, "loss": 4.961085677146912, "l_mix": 3.0776147544384003, "l_a": 3.161531090736389, "l_c": 3.116705060005188, "z_loss": 25.108716011047363, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20302900671958923, "tokens_per_second": 1443984.6301983965, "eta_seconds": 14726.331947922705, "data_wait_seconds": 2.2470951080322266e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 31930, "total_steps": 72479, "tokens": 16740515840, "loss": 4.775653421878815, "l_mix": 2.961587280035019, "l_a": 3.0433050990104675, "l_c": 3.0035820603370667, "z_loss": 26.502394199371338, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16113674640655518, "tokens_per_second": 1400777.0220674512, "eta_seconds": 15176.829557514191, "data_wait_seconds": 2.03549861907959e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 31940, "total_steps": 72479, "tokens": 16745758720, "loss": 4.600574731826782, "l_mix": 2.8493382930755615, "l_a": 2.945594012737274, "l_c": 2.8918603658676147, "z_loss": 27.5311119556427, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15955762565135956, "tokens_per_second": 1451858.8155532246, "eta_seconds": 14639.241091704369, "data_wait_seconds": 2.1904706954956055e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 31950, "total_steps": 72479, "tokens": 16751001600, "loss": 4.447935521602631, "l_mix": 2.7552099525928497, "l_a": 2.845649003982544, "l_c": 2.796769469976425, "z_loss": 27.069223642349243, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16396567225456238, "tokens_per_second": 1338258.1537502995, "eta_seconds": 15878.00402519703, "data_wait_seconds": 2.1696090698242188e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 31960, "total_steps": 72479, "tokens": 16756244480, "loss": 4.603136122226715, "l_mix": 2.8508993089199066, "l_a": 2.946043521165848, "l_c": 2.8947455883026123, "z_loss": 26.860058307647705, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11402682214975357, "tokens_per_second": 1344600.9100094237, "eta_seconds": 15799.20503835678, "data_wait_seconds": 3.0934810638427734e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 31970, "total_steps": 72479, "tokens": 16761487360, "loss": 4.643034994602203, "l_mix": 2.876350998878479, "l_a": 2.9677965342998505, "l_c": 2.9211499094963074, "z_loss": 26.980048179626465, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1629694700241089, "tokens_per_second": 1447620.1123931166, "eta_seconds": 14671.240341424942, "data_wait_seconds": 4.121661186218262e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 31980, "total_steps": 72479, "tokens": 16766730240, "loss": 4.497548341751099, "l_mix": 2.785776972770691, "l_a": 2.879187762737274, "l_c": 2.826716810464859, "z_loss": 27.972355127334595, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18931494653224945, "tokens_per_second": 1346623.257286156, "eta_seconds": 15767.691221070289, "data_wait_seconds": 2.828240394592285e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 31990, "total_steps": 72479, "tokens": 16771973120, "loss": 3.302282303571701, "l_mix": 2.0304936170578003, "l_a": 2.151347190141678, "l_c": 2.087948352098465, "z_loss": 27.708011865615845, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18869243562221527, "tokens_per_second": 1455028.871696924, "eta_seconds": 14589.330318403245, "data_wait_seconds": 2.2023916244506836e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 32000, "total_steps": 72479, "tokens": 16777216000, "loss": 3.3235671520233154, "l_mix": 2.0444774478673935, "l_a": 2.165105104446411, "l_c": 2.0985269248485565, "z_loss": 28.18579077720642, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17457878589630127, "tokens_per_second": 1223038.0419495944, "eta_seconds": 17352.407058548928, "data_wait_seconds": 3.057718276977539e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.28214183, "gate_a2b/L1_absmax": 0.67774945, "gate_a2b/L1_rms": 0.29662254, "gate_a2b/L3_absmean": 0.29338673, "gate_a2b/L3_absmax": 0.61927003, "gate_a2b/L3_rms": 0.29912683, "gate_a2b/L5_absmean": 0.45837361, "gate_a2b/L5_absmax": 0.63977242, "gate_a2b/L5_rms": 0.46409321, "gate_c2b/L1_absmean": 0.0161896, "gate_c2b/L1_absmax": 0.54578102, "gate_c2b/L1_rms": 0.06447874, "gate_c2b/L3_absmean": 0.21889627, "gate_c2b/L3_absmax": 0.48681897, "gate_c2b/L3_rms": 0.22323559, "gate_c2b/L5_absmean": 0.58375466, "gate_c2b/L5_absmax": 0.82027078, "gate_c2b/L5_rms": 0.5970208, "gate_b2a/L7_absmean": 0.06863533, "gate_b2a/L7_absmax": 0.32419819, "gate_b2a/L7_rms": 0.07370986, "gate_b2a/L11_absmean": 0.10761929, "gate_b2a/L11_absmax": 0.56704521, "gate_b2a/L11_rms": 0.11371081, "gate_b2a/L14_absmean": 0.27351889, "gate_b2a/L14_absmax": 0.70141345, "gate_b2a/L14_rms": 0.27859572, "gate_b2c/L3_absmean": 0.06377913, "gate_b2c/L3_absmax": 0.6812458, "gate_b2c/L3_rms": 0.07740785, "gate_b2c/L5_absmean": 0.07380398, "gate_b2c/L5_absmax": 0.7800082, "gate_b2c/L5_rms": 0.08559895, "gate_b2c/L6_absmean": 0.10829707, "gate_b2c/L6_absmax": 0.53416443, "gate_b2c/L6_rms": 0.11692607}, "streams": {"cos_a2b/round0": 0.394365, "cos_c2b/round0": 0.265888, "cos_b2a/round0": 0.112566, "cos_b2c/round0": 0.105525, "rms_a/round0": 2.236545, "rms_b/round0": 4.077602, "rms_c/round0": 2.210039, "cos_a2b/round1": 0.425204, "cos_c2b/round1": 0.28884, "cos_b2a/round1": 0.229911, "cos_b2c/round1": 0.125748, "rms_a/round1": 4.056024, "rms_b/round1": 9.015201, "rms_c/round1": 4.24697, "cos_a2b/round2": 0.521486, "cos_c2b/round2": 0.502017, "cos_b2a/round2": 0.068997, "cos_b2c/round2": 0.148343, "rms_a/round2": 9.515907, "rms_b/round2": 21.347744, "rms_c/round2": 7.485837}, "alpha_mean": 0.423352652737762, "alpha_hist": [0.02933, 0.13911, 0.19131, 0.16287, 0.12257, 0.10625, 0.10017, 0.09222, 0.05147, 0.00468], "alpha_by_type": {"special": {"share": 0.00107, "mean": 0.5945, "std": 0.09666}, "whitespace": {"share": 0.04025, "mean": 0.63421, "std": 0.17061}, "digit": {"share": 0.00932, "mean": 0.36434, "std": 0.16804}, "word_start": {"share": 0.55296, "mean": 0.4301, "std": 0.20834}, "word_cont": {"share": 0.27191, "mean": 0.2835, "std": 0.1401}, "punct": {"share": 0.12448, "mean": 0.63365, "std": 0.18225}}, "alpha_p05": 0.12963, "alpha_p50": 0.38792, "alpha_p95": 0.82587}
{"step": 32010, "total_steps": 72479, "tokens": 16782458880, "loss": 3.3242445588111877, "l_mix": 2.045784905552864, "l_a": 2.1593310236930847, "l_c": 2.1022010445594788, "z_loss": 27.04724955558777, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1976885050535202, "tokens_per_second": 1465647.9192031971, "eta_seconds": 14476.472005319596, "data_wait_seconds": 3.2007694244384766e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 32020, "total_steps": 72479, "tokens": 16787701760, "loss": 3.0029499232769012, "l_mix": 1.8586150258779526, "l_a": 1.9230762720108032, "l_c": 1.8913732022047043, "z_loss": 30.770015239715576, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.3023732006549835, "tokens_per_second": 1428131.089456367, "eta_seconds": 14853.09601380825, "data_wait_seconds": 2.86102294921875e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 32030, "total_steps": 72479, "tokens": 16792944640, "loss": 4.474071264266968, "l_mix": 2.770506739616394, "l_a": 2.8661283254623413, "l_c": 2.812419831752777, "z_loss": 27.692522048950195, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1451537162065506, "tokens_per_second": 1308664.2449703102, "eta_seconds": 16205.01621673107, "data_wait_seconds": 1.9103288650512695e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 32040, "total_steps": 72479, "tokens": 16798187520, "loss": 4.559465944766998, "l_mix": 2.8238186836242676, "l_a": 2.9202851951122284, "l_c": 2.865205407142639, "z_loss": 27.126128673553467, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20205093920230865, "tokens_per_second": 1346812.3729235637, "eta_seconds": 15742.120326662063, "data_wait_seconds": 1.5556812286376953e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 32050, "total_steps": 72479, "tokens": 16803430400, "loss": 4.508600354194641, "l_mix": 2.7939435243606567, "l_a": 2.881167471408844, "l_c": 2.8343547582626343, "z_loss": 27.225479125976562, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14002041518688202, "tokens_per_second": 1381576.84285072, "eta_seconds": 15342.208188915254, "data_wait_seconds": 1.722574234008789e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 32060, "total_steps": 72479, "tokens": 16808673280, "loss": 4.494635045528412, "l_mix": 2.783389002084732, "l_a": 2.8785506188869476, "l_c": 2.825602561235428, "z_loss": 27.736008644104004, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14745956659317017, "tokens_per_second": 1378285.0852277607, "eta_seconds": 15375.046062040328, "data_wait_seconds": 1.6480684280395508e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 32070, "total_steps": 72479, "tokens": 16813916160, "loss": 4.53621244430542, "l_mix": 2.809577465057373, "l_a": 2.9014490842819214, "l_c": 2.8540006577968597, "z_loss": 27.413787364959717, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1621142476797104, "tokens_per_second": 1479336.6342252044, "eta_seconds": 14321.252716827394, "data_wait_seconds": 1.9729137420654297e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 32080, "total_steps": 72479, "tokens": 16819159040, "loss": 4.447959244251251, "l_mix": 2.7556761503219604, "l_a": 2.844387799501419, "l_c": 2.7965554296970367, "z_loss": 27.48882222175598, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17143601179122925, "tokens_per_second": 1238042.5192744685, "eta_seconds": 17108.225753355026, "data_wait_seconds": 2.5570392608642578e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 32090, "total_steps": 72479, "tokens": 16824401920, "loss": 4.587861120700836, "l_mix": 2.8424348533153534, "l_a": 2.933922231197357, "l_c": 2.8841652274131775, "z_loss": 27.580317735671997, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13650785386562347, "tokens_per_second": 1388025.9352661595, "eta_seconds": 15255.815827345848, "data_wait_seconds": 1.7017126083374023e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 32100, "total_steps": 72479, "tokens": 16829644800, "loss": 3.303387999534607, "l_mix": 2.0310309678316116, "l_a": 2.151122897863388, "l_c": 2.090067356824875, "z_loss": 27.720991134643555, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20122551918029785, "tokens_per_second": 1292049.8618755136, "eta_seconds": 16384.990840268136, "data_wait_seconds": 2.2292137145996094e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.28042945, "gate_a2b/L1_absmax": 0.68037635, "gate_a2b/L1_rms": 0.29470286, "gate_a2b/L3_absmean": 0.2919569, "gate_a2b/L3_absmax": 0.61776233, "gate_a2b/L3_rms": 0.29762399, "gate_a2b/L5_absmean": 0.456209, "gate_a2b/L5_absmax": 0.63476467, "gate_a2b/L5_rms": 0.46197981, "gate_c2b/L1_absmean": 0.0164748, "gate_c2b/L1_absmax": 0.54874432, "gate_c2b/L1_rms": 0.06474926, "gate_c2b/L3_absmean": 0.2190185, "gate_c2b/L3_absmax": 0.48489937, "gate_c2b/L3_rms": 0.22332905, "gate_c2b/L5_absmean": 0.58497429, "gate_c2b/L5_absmax": 0.81849951, "gate_c2b/L5_rms": 0.59828895, "gate_b2a/L7_absmean": 0.06801005, "gate_b2a/L7_absmax": 0.32869902, "gate_b2a/L7_rms": 0.07317922, "gate_b2a/L11_absmean": 0.10728627, "gate_b2a/L11_absmax": 0.57068706, "gate_b2a/L11_rms": 0.11340909, "gate_b2a/L14_absmean": 0.27191252, "gate_b2a/L14_absmax": 0.70234984, "gate_b2a/L14_rms": 0.27711189, "gate_b2c/L3_absmean": 0.06251077, "gate_b2c/L3_absmax": 0.6780715, "gate_b2c/L3_rms": 0.07629296, "gate_b2c/L5_absmean": 0.07238494, "gate_b2c/L5_absmax": 0.77438563, "gate_b2c/L5_rms": 0.08424376, "gate_b2c/L6_absmean": 0.1067462, "gate_b2c/L6_absmax": 0.52878135, "gate_b2c/L6_rms": 0.11535788}, "streams": {"cos_a2b/round0": 0.391623, "cos_c2b/round0": 0.266588, "cos_b2a/round0": 0.11365, "cos_b2c/round0": 0.106414, "rms_a/round0": 2.273688, "rms_b/round0": 4.159466, "rms_c/round0": 2.238043, "cos_a2b/round1": 0.426659, "cos_c2b/round1": 0.291097, "cos_b2a/round1": 0.226317, "cos_b2c/round1": 0.125046, "rms_a/round1": 4.160689, "rms_b/round1": 9.238759, "rms_c/round1": 4.284563, "cos_a2b/round2": 0.526408, "cos_c2b/round2": 0.501373, "cos_b2a/round2": 0.067578, "cos_b2c/round2": 0.143839, "rms_a/round2": 9.808217, "rms_b/round2": 21.854836, "rms_c/round2": 7.56188}, "alpha_mean": 0.44313818402137883, "alpha_hist": [0.02841, 0.1277, 0.17804, 0.15521, 0.12227, 0.10506, 0.10088, 0.1053, 0.07001, 0.00713], "alpha_by_type": {"special": {"share": 0.00102, "mean": 0.64147, "std": 0.08896}, "whitespace": {"share": 0.04048, "mean": 0.65148, "std": 0.16481}, "digit": {"share": 0.01207, "mean": 0.43662, "std": 0.16389}, "word_start": {"share": 0.55692, "mean": 0.44928, "std": 0.21704}, "word_cont": {"share": 0.2632, "mean": 0.2961, "std": 0.15248}, "punct": {"share": 0.12631, "mean": 0.65467, "std": 0.18579}}, "alpha_p05": 0.12745, "alpha_p50": 0.42585, "alpha_p95": 0.8472}
{"step": 32110, "total_steps": 72479, "tokens": 16834887680, "loss": 3.303368389606476, "l_mix": 2.032406508922577, "l_a": 2.154063194990158, "l_c": 2.082476317882538, "z_loss": 27.321720123291016, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18375252187252045, "tokens_per_second": 1377640.2490927656, "eta_seconds": 15363.214225149153, "data_wait_seconds": 2.3543834686279297e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 32120, "total_steps": 72479, "tokens": 16840130560, "loss": 3.329430729150772, "l_mix": 2.047066241502762, "l_a": 2.169475644826889, "l_c": 2.1050725281238556, "z_loss": 27.91937017440796, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19035924971103668, "tokens_per_second": 1432969.974189219, "eta_seconds": 14766.35224263668, "data_wait_seconds": 2.4884939193725586e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 32130, "total_steps": 72479, "tokens": 16845373440, "loss": 3.121185839176178, "l_mix": 1.9303387999534607, "l_a": 2.007596656680107, "l_c": 1.961893230676651, "z_loss": 32.3600697517395, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1941099315881729, "tokens_per_second": 1437769.092117591, "eta_seconds": 14713.417215585709, "data_wait_seconds": 2.9712915420532227e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 32140, "total_steps": 72479, "tokens": 16850616320, "loss": 4.501836717128754, "l_mix": 2.7883262634277344, "l_a": 2.883251190185547, "l_c": 2.82845002412796, "z_loss": 27.78471088409424, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14177431166172028, "tokens_per_second": 1407833.9500477116, "eta_seconds": 15022.548384547234, "data_wait_seconds": 2.2679567337036133e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 32150, "total_steps": 72479, "tokens": 16855859200, "loss": 4.519457280635834, "l_mix": 2.8000703155994415, "l_a": 2.8881244361400604, "l_c": 2.843165308237076, "z_loss": 27.114922523498535, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16066864132881165, "tokens_per_second": 1451309.864613163, "eta_seconds": 14568.915479421616, "data_wait_seconds": 2.378225326538086e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 32160, "total_steps": 72479, "tokens": 16861102080, "loss": 4.5002493262290955, "l_mix": 2.7870973646640778, "l_a": 2.8829789459705353, "l_c": 2.827527165412903, "z_loss": 27.256869792938232, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1421234905719757, "tokens_per_second": 1421100.11861246, "eta_seconds": 14874.932170605658, "data_wait_seconds": 1.996755599975586e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 32170, "total_steps": 72479, "tokens": 16866344960, "loss": 4.458245038986206, "l_mix": 2.761247307062149, "l_a": 2.8554573953151703, "l_c": 2.8012013733386993, "z_loss": 27.975709915161133, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15451544523239136, "tokens_per_second": 1414576.5948471618, "eta_seconds": 14939.823738765715, "data_wait_seconds": 1.8268823623657227e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 32180, "total_steps": 72479, "tokens": 16871587840, "loss": 4.5391223430633545, "l_mix": 2.81150358915329, "l_a": 2.9035394489765167, "l_c": 2.855189621448517, "z_loss": 27.65963578224182, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15093301236629486, "tokens_per_second": 1450034.6715857864, "eta_seconds": 14570.880632042885, "data_wait_seconds": 1.5109777450561523e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 32190, "total_steps": 72479, "tokens": 16876830720, "loss": 4.770324468612671, "l_mix": 2.958559572696686, "l_a": 3.0391324162483215, "l_c": 3.000083237886429, "z_loss": 27.68871545791626, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14688315987586975, "tokens_per_second": 1422987.216256179, "eta_seconds": 14844.152491807938, "data_wait_seconds": 1.8864870071411133e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08232}}
{"step": 32200, "total_steps": 72479, "tokens": 16882073600, "loss": 4.765467643737793, "l_mix": 2.9553496837615967, "l_a": 3.036391407251358, "l_c": 2.9973352551460266, "z_loss": 26.845438718795776, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.154183030128479, "tokens_per_second": 1469314.6047605558, "eta_seconds": 14372.54913520813, "data_wait_seconds": 1.537799835205078e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.27934977, "gate_a2b/L1_absmax": 0.68058938, "gate_a2b/L1_rms": 0.29357076, "gate_a2b/L3_absmean": 0.29008985, "gate_a2b/L3_absmax": 0.6167441, "gate_a2b/L3_rms": 0.29582185, "gate_a2b/L5_absmean": 0.45398304, "gate_a2b/L5_absmax": 0.63335675, "gate_a2b/L5_rms": 0.45982075, "gate_c2b/L1_absmean": 0.01680094, "gate_c2b/L1_absmax": 0.54961842, "gate_c2b/L1_rms": 0.06477618, "gate_c2b/L3_absmean": 0.21862002, "gate_c2b/L3_absmax": 0.47887897, "gate_c2b/L3_rms": 0.22295155, "gate_c2b/L5_absmean": 0.58609372, "gate_c2b/L5_absmax": 0.82265121, "gate_c2b/L5_rms": 0.59949088, "gate_b2a/L7_absmean": 0.0676965, "gate_b2a/L7_absmax": 0.32493147, "gate_b2a/L7_rms": 0.07281482, "gate_b2a/L11_absmean": 0.10623366, "gate_b2a/L11_absmax": 0.57197887, "gate_b2a/L11_rms": 0.11247082, "gate_b2a/L14_absmean": 0.2704103, "gate_b2a/L14_absmax": 0.70389861, "gate_b2a/L14_rms": 0.27565026, "gate_b2c/L3_absmean": 0.06241326, "gate_b2c/L3_absmax": 0.67470038, "gate_b2c/L3_rms": 0.0758813, "gate_b2c/L5_absmean": 0.07160502, "gate_b2c/L5_absmax": 0.77125573, "gate_b2c/L5_rms": 0.08344495, "gate_b2c/L6_absmean": 0.10516512, "gate_b2c/L6_absmax": 0.52970952, "gate_b2c/L6_rms": 0.113835}, "streams": {"cos_a2b/round0": 0.400418, "cos_c2b/round0": 0.303056, "cos_b2a/round0": 0.134833, "cos_b2c/round0": 0.138381, "rms_a/round0": 2.202686, "rms_b/round0": 4.163759, "rms_c/round0": 2.218277, "cos_a2b/round1": 0.433686, "cos_c2b/round1": 0.301548, "cos_b2a/round1": 0.248742, "cos_b2c/round1": 0.127468, "rms_a/round1": 3.944217, "rms_b/round1": 8.731518, "rms_c/round1": 4.133321, "cos_a2b/round2": 0.575397, "cos_c2b/round2": 0.517466, "cos_b2a/round2": 0.103695, "cos_b2c/round2": 0.174237, "rms_a/round2": 9.635424, "rms_b/round2": 20.749935, "rms_c/round2": 7.167117}, "alpha_mean": 0.4741627546289209, "alpha_hist": [0.00533, 0.03926, 0.12465, 0.17543, 0.19901, 0.2123, 0.15054, 0.0815, 0.0118, 0.0002], "alpha_by_type": {"special": {"share": 0.00067, "mean": 0.53254, "std": 0.15492}, "whitespace": {"share": 0.05254, "mean": 0.56787, "std": 0.10661}, "digit": {"share": 0.02257, "mean": 0.49294, "std": 0.12238}, "word_start": {"share": 0.58656, "mean": 0.49378, "std": 0.15518}, "word_cont": {"share": 0.23737, "mean": 0.35463, "std": 0.13195}, "punct": {"share": 0.1003, "mean": 0.58863, "std": 0.15739}}, "alpha_p05": 0.21174, "alpha_p50": 0.48542, "alpha_p95": 0.74017}
{"step": 32210, "total_steps": 72479, "tokens": 16887316480, "loss": 4.596393465995789, "l_mix": 2.849332571029663, "l_a": 2.9343017637729645, "l_c": 2.8892343640327454, "z_loss": 27.48619246482849, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17967066168785095, "tokens_per_second": 1438645.2773006877, "eta_seconds": 14675.301691889763, "data_wait_seconds": 2.428889274597168e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 32220, "total_steps": 72479, "tokens": 16892559360, "loss": 4.908633768558502, "l_mix": 3.044011414051056, "l_a": 3.127034157514572, "l_c": 3.088373214006424, "z_loss": 26.197420597076416, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15739314258098602, "tokens_per_second": 1465171.2720143986, "eta_seconds": 14406.036342072488, "data_wait_seconds": 3.3020973205566406e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 32230, "total_steps": 72479, "tokens": 16897802240, "loss": 3.314443051815033, "l_mix": 2.038185030221939, "l_a": 2.158706486225128, "l_c": 2.095486640930176, "z_loss": 27.992223501205444, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1611781269311905, "tokens_per_second": 1338987.2964098654, "eta_seconds": 15759.722118782996, "data_wait_seconds": 3.361701965332031e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 32240, "total_steps": 72479, "tokens": 16903045120, "loss": 3.251743882894516, "l_mix": 2.0001978874206543, "l_a": 2.1236062794923782, "l_c": 2.0482133328914642, "z_loss": 29.203426361083984, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.6840469837188721, "tokens_per_second": 1356680.3619923014, "eta_seconds": 15550.328156161308, "data_wait_seconds": 2.714991569519043e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 32250, "total_steps": 72479, "tokens": 16908288000, "loss": 4.642879068851471, "l_mix": 2.8774777352809906, "l_a": 2.9638457596302032, "l_c": 2.920825183391571, "z_loss": 27.0597562789917, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19150738418102264, "tokens_per_second": 1422719.677856712, "eta_seconds": 14824.833226299286, "data_wait_seconds": 1.8149614334106445e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 32260, "total_steps": 72479, "tokens": 16913530880, "loss": 4.457577347755432, "l_mix": 2.7601345777511597, "l_a": 2.855583757162094, "l_c": 2.80255851149559, "z_loss": 28.2878680229187, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18192318081855774, "tokens_per_second": 1463189.0127125096, "eta_seconds": 14411.220210647582, "data_wait_seconds": 1.5139579772949219e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 32270, "total_steps": 72479, "tokens": 16918773760, "loss": 4.540707349777222, "l_mix": 2.813052475452423, "l_a": 2.9044701159000397, "l_c": 2.8543792068958282, "z_loss": 27.767700672149658, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12981362640857697, "tokens_per_second": 1465536.1756694778, "eta_seconds": 14384.562143182753, "data_wait_seconds": 1.6033649444580078e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 32280, "total_steps": 72479, "tokens": 16924016640, "loss": 4.484892785549164, "l_mix": 2.7773576974868774, "l_a": 2.872196674346924, "l_c": 2.819586366415024, "z_loss": 27.000126600265503, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.169908344745636, "tokens_per_second": 1457662.5144286514, "eta_seconds": 14458.664542293549, "data_wait_seconds": 1.9043684005737305e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 32290, "total_steps": 72479, "tokens": 16929259520, "loss": 4.564907431602478, "l_mix": 2.827750265598297, "l_a": 2.9200905859470367, "l_c": 2.870433211326599, "z_loss": 27.98374032974243, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16323983669281006, "tokens_per_second": 1355407.3094894704, "eta_seconds": 15545.593036484719, "data_wait_seconds": 3.6329030990600586e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 32300, "total_steps": 72479, "tokens": 16934502400, "loss": 3.0764227211475372, "l_mix": 1.9035875648260117, "l_a": 1.9793365001678467, "l_c": 1.9301140308380127, "z_loss": 32.12138748168945, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.3113848865032196, "tokens_per_second": 1329505.431357375, "eta_seconds": 15844.514099121092, "data_wait_seconds": 3.263354301452637e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.27776167, "gate_a2b/L1_absmax": 0.67737675, "gate_a2b/L1_rms": 0.2921313, "gate_a2b/L3_absmean": 0.28849921, "gate_a2b/L3_absmax": 0.62482232, "gate_a2b/L3_rms": 0.29434776, "gate_a2b/L5_absmean": 0.4521291, "gate_a2b/L5_absmax": 0.63667136, "gate_a2b/L5_rms": 0.45788565, "gate_c2b/L1_absmean": 0.01698331, "gate_c2b/L1_absmax": 0.54828119, "gate_c2b/L1_rms": 0.06501886, "gate_c2b/L3_absmean": 0.21849428, "gate_c2b/L3_absmax": 0.49059656, "gate_c2b/L3_rms": 0.22288829, "gate_c2b/L5_absmean": 0.58655691, "gate_c2b/L5_absmax": 0.82224965, "gate_c2b/L5_rms": 0.59998655, "gate_b2a/L7_absmean": 0.06738929, "gate_b2a/L7_absmax": 0.32402021, "gate_b2a/L7_rms": 0.07258915, "gate_b2a/L11_absmean": 0.1056355, "gate_b2a/L11_absmax": 0.56886327, "gate_b2a/L11_rms": 0.11191089, "gate_b2a/L14_absmean": 0.26960626, "gate_b2a/L14_absmax": 0.6989035, "gate_b2a/L14_rms": 0.2749061, "gate_b2c/L3_absmean": 0.06167465, "gate_b2c/L3_absmax": 0.66390949, "gate_b2c/L3_rms": 0.07502112, "gate_b2c/L5_absmean": 0.07130834, "gate_b2c/L5_absmax": 0.76005274, "gate_b2c/L5_rms": 0.08285001, "gate_b2c/L6_absmean": 0.10417231, "gate_b2c/L6_absmax": 0.53599054, "gate_b2c/L6_rms": 0.11300527}, "streams": {"cos_a2b/round0": 0.409946, "cos_c2b/round0": 0.30748, "cos_b2a/round0": 0.123057, "cos_b2c/round0": 0.12973, "rms_a/round0": 2.300317, "rms_b/round0": 4.117379, "rms_c/round0": 2.208613, "cos_a2b/round1": 0.406953, "cos_c2b/round1": 0.262058, "cos_b2a/round1": 0.221674, "cos_b2c/round1": 0.110108, "rms_a/round1": 4.124869, "rms_b/round1": 9.028664, "rms_c/round1": 4.211598, "cos_a2b/round2": 0.520091, "cos_c2b/round2": 0.543634, "cos_b2a/round2": 0.084904, "cos_b2c/round2": 0.15401, "rms_a/round2": 9.575033, "rms_b/round2": 22.442533, "rms_c/round2": 7.247481}, "alpha_mean": 0.4365867141203239, "alpha_hist": [0.01891, 0.04318, 0.13301, 0.22183, 0.23969, 0.1844, 0.10866, 0.04399, 0.00604, 0.00029], "alpha_by_type": {"special": {"share": 0.00067, "mean": 0.56255, "std": 0.14401}, "whitespace": {"share": 0.08167, "mean": 0.51632, "std": 0.12731}, "digit": {"share": 0.10431, "mean": 0.42748, "std": 0.12022}, "word_start": {"share": 0.36765, "mean": 0.43049, "std": 0.15784}, "word_cont": {"share": 0.24011, "mean": 0.3356, "std": 0.12336}, "punct": {"share": 0.2056, "mean": 0.53797, "std": 0.138}}, "alpha_p05": 0.18215, "alpha_p50": 0.44119, "alpha_p95": 0.69451}
{"step": 32310, "total_steps": 72479, "tokens": 16939745280, "loss": 4.749606788158417, "l_mix": 2.945148289203644, "l_a": 3.026887387037277, "l_c": 2.9879740476608276, "z_loss": 26.82042932510376, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16559027135372162, "tokens_per_second": 1362161.07421649, "eta_seconds": 15460.818159198761, "data_wait_seconds": 5.2422285079956055e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 32320, "total_steps": 72479, "tokens": 16944988160, "loss": 4.9205238819122314, "l_mix": 3.0518061816692352, "l_a": 3.139624148607254, "l_c": 3.089434951543808, "z_loss": 26.98450207710266, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1607656180858612, "tokens_per_second": 1385413.4426599254, "eta_seconds": 15197.544028139113, "data_wait_seconds": 2.631545066833496e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 32330, "total_steps": 72479, "tokens": 16950231040, "loss": 4.877261400222778, "l_mix": 3.0242318511009216, "l_a": 3.1102125346660614, "l_c": 3.066552221775055, "z_loss": 27.67677927017212, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18860916793346405, "tokens_per_second": 1450587.5381744888, "eta_seconds": 14511.112468600273, "data_wait_seconds": 2.7120113372802734e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 32340, "total_steps": 72479, "tokens": 16955473920, "loss": 4.673838913440704, "l_mix": 2.899379312992096, "l_a": 2.974800229072571, "l_c": 2.9400647282600403, "z_loss": 27.643407583236694, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15719805657863617, "tokens_per_second": 1456629.7584073609, "eta_seconds": 14447.319856357573, "data_wait_seconds": 1.5854835510253906e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 32350, "total_steps": 72479, "tokens": 16960716800, "loss": 4.80527925491333, "l_mix": 2.980505108833313, "l_a": 3.056939661502838, "l_c": 3.025640219449997, "z_loss": 27.100461721420288, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1444094330072403, "tokens_per_second": 1433318.0783703183, "eta_seconds": 14678.63516793251, "data_wait_seconds": 2.485513687133789e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 32360, "total_steps": 72479, "tokens": 16965959680, "loss": 4.542013704776764, "l_mix": 2.8134535551071167, "l_a": 2.904233068227768, "l_c": 2.857634127140045, "z_loss": 28.24397897720337, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1302911937236786, "tokens_per_second": 1450633.2785423263, "eta_seconds": 14499.812311720847, "data_wait_seconds": 1.9729137420654297e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08232}}
{"step": 32370, "total_steps": 72479, "tokens": 16971202560, "loss": 4.580760478973389, "l_mix": 2.8379244208335876, "l_a": 2.9272442162036896, "l_c": 2.8822087943553925, "z_loss": 27.659658193588257, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11992128938436508, "tokens_per_second": 1361550.2857194513, "eta_seconds": 15444.649832296373, "data_wait_seconds": 2.1249055862426758e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 32380, "total_steps": 72479, "tokens": 16976445440, "loss": 4.47136253118515, "l_mix": 2.769623339176178, "l_a": 2.857139766216278, "l_c": 2.8153240084648132, "z_loss": 28.20068120956421, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1636444330215454, "tokens_per_second": 1382198.2638114025, "eta_seconds": 15210.136680412294, "data_wait_seconds": 1.7464160919189453e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 32390, "total_steps": 72479, "tokens": 16981688320, "loss": 4.5839157700538635, "l_mix": 2.8399368822574615, "l_a": 2.9355678260326385, "l_c": 2.877695143222809, "z_loss": 28.109017610549927, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.24629299342632294, "tokens_per_second": 1294977.6982653784, "eta_seconds": 16230.535599303244, "data_wait_seconds": 1.7255544662475586e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 32400, "total_steps": 72479, "tokens": 16986931200, "loss": 4.527627229690552, "l_mix": 2.8045747876167297, "l_a": 2.8947344720363617, "l_c": 2.848773241043091, "z_loss": 27.689086437225342, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18692360818386078, "tokens_per_second": 1439835.0784587236, "eta_seconds": 14593.990010643005, "data_wait_seconds": 2.0712614059448242e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.27600542, "gate_a2b/L1_absmax": 0.68458384, "gate_a2b/L1_rms": 0.29043457, "gate_a2b/L3_absmean": 0.28674135, "gate_a2b/L3_absmax": 0.62460113, "gate_a2b/L3_rms": 0.29260331, "gate_a2b/L5_absmean": 0.44992706, "gate_a2b/L5_absmax": 0.63901401, "gate_a2b/L5_rms": 0.45573458, "gate_c2b/L1_absmean": 0.01672752, "gate_c2b/L1_absmax": 0.55400294, "gate_c2b/L1_rms": 0.06513917, "gate_c2b/L3_absmean": 0.21886034, "gate_c2b/L3_absmax": 0.48458737, "gate_c2b/L3_rms": 0.22321346, "gate_c2b/L5_absmean": 0.58627981, "gate_c2b/L5_absmax": 0.82209897, "gate_c2b/L5_rms": 0.59955937, "gate_b2a/L7_absmean": 0.06693622, "gate_b2a/L7_absmax": 0.32510269, "gate_b2a/L7_rms": 0.07206836, "gate_b2a/L11_absmean": 0.10511457, "gate_b2a/L11_absmax": 0.5738799, "gate_b2a/L11_rms": 0.11150686, "gate_b2a/L14_absmean": 0.26794696, "gate_b2a/L14_absmax": 0.71038055, "gate_b2a/L14_rms": 0.27344278, "gate_b2c/L3_absmean": 0.06069244, "gate_b2c/L3_absmax": 0.66556191, "gate_b2c/L3_rms": 0.07413968, "gate_b2c/L5_absmean": 0.07007392, "gate_b2c/L5_absmax": 0.76603562, "gate_b2c/L5_rms": 0.08187, "gate_b2c/L6_absmean": 0.1026847, "gate_b2c/L6_absmax": 0.53398782, "gate_b2c/L6_rms": 0.1114913}, "streams": {"cos_a2b/round0": 0.400105, "cos_c2b/round0": 0.297198, "cos_b2a/round0": 0.120093, "cos_b2c/round0": 0.13136, "rms_a/round0": 2.301908, "rms_b/round0": 4.351659, "rms_c/round0": 2.276164, "cos_a2b/round1": 0.423054, "cos_c2b/round1": 0.28639, "cos_b2a/round1": 0.237463, "cos_b2c/round1": 0.116477, "rms_a/round1": 4.114508, "rms_b/round1": 9.30373, "rms_c/round1": 4.274082, "cos_a2b/round2": 0.542781, "cos_c2b/round2": 0.523673, "cos_b2a/round2": 0.091393, "cos_b2c/round2": 0.157296, "rms_a/round2": 10.002472, "rms_b/round2": 22.52088, "rms_c/round2": 7.468842}, "alpha_mean": 0.41981514237939876, "alpha_hist": [0.01031, 0.08481, 0.18637, 0.21051, 0.18469, 0.1497, 0.10454, 0.0591, 0.00984, 0.00012], "alpha_by_type": {"special": {"share": 0.00089, "mean": 0.58665, "std": 0.11202}, "whitespace": {"share": 0.0329, "mean": 0.52263, "std": 0.12951}, "digit": {"share": 0.06018, "mean": 0.4527, "std": 0.12084}, "word_start": {"share": 0.51303, "mean": 0.42587, "std": 0.17359}, "word_cont": {"share": 0.27032, "mean": 0.32685, "std": 0.12745}, "punct": {"share": 0.12268, "mean": 0.55444, "std": 0.16584}}, "alpha_p05": 0.17667, "alpha_p50": 0.39887, "alpha_p95": 0.73297}
{"step": 32410, "total_steps": 72479, "tokens": 16992174080, "loss": 3.053900361061096, "l_mix": 1.8879580944776535, "l_a": 1.9634362161159515, "l_c": 1.9230377823114395, "z_loss": 32.85974049568176, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.253958523273468, "tokens_per_second": 1441064.9059202306, "eta_seconds": 14577.89707159996, "data_wait_seconds": 2.3245811462402344e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 32420, "total_steps": 72479, "tokens": 16997416960, "loss": 3.227165788412094, "l_mix": 1.9973376095294952, "l_a": 2.0689492374658585, "l_c": 2.030477836728096, "z_loss": 32.461087703704834, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.21550162136554718, "tokens_per_second": 1374072.9826566204, "eta_seconds": 15284.816204881668, "data_wait_seconds": 1.9669532775878906e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08236}}
{"step": 32430, "total_steps": 72479, "tokens": 17002659840, "loss": 4.781286120414734, "l_mix": 2.9646142721176147, "l_a": 3.0435380935668945, "l_c": 3.0120344758033752, "z_loss": 27.61246418952942, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13650619983673096, "tokens_per_second": 1300726.9298212302, "eta_seconds": 16142.67347788811, "data_wait_seconds": 2.8192996978759766e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 32440, "total_steps": 72479, "tokens": 17007902720, "loss": 4.772217094898224, "l_mix": 2.9605570435523987, "l_a": 3.0399446189403534, "l_c": 2.998922199010849, "z_loss": 27.423147916793823, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14978760480880737, "tokens_per_second": 1430773.4550360611, "eta_seconds": 14671.761737060546, "data_wait_seconds": 1.8805265426635742e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 32450, "total_steps": 72479, "tokens": 17013145600, "loss": 4.832487344741821, "l_mix": 2.996703863143921, "l_a": 3.077776789665222, "l_c": 3.0415011942386627, "z_loss": 26.841989994049072, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16103938221931458, "tokens_per_second": 1353197.5310602933, "eta_seconds": 15508.988059973715, "data_wait_seconds": 2.110004425048828e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 32460, "total_steps": 72479, "tokens": 17018388480, "loss": 4.82965749502182, "l_mix": 2.9961642622947693, "l_a": 3.071379631757736, "l_c": 3.0402640402317047, "z_loss": 27.296910285949707, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1601286083459854, "tokens_per_second": 1452576.555921261, "eta_seconds": 14444.320601534844, "data_wait_seconds": 1.6003847122192383e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 32470, "total_steps": 72479, "tokens": 17023631360, "loss": 4.45841908454895, "l_mix": 2.761131525039673, "l_a": 2.853730171918869, "l_c": 2.803894966840744, "z_loss": 27.489020347595215, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13217346370220184, "tokens_per_second": 1461113.759261762, "eta_seconds": 14356.33499379158, "data_wait_seconds": 1.96993350982666e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08232}}
{"step": 32480, "total_steps": 72479, "tokens": 17028874240, "loss": 4.560022830963135, "l_mix": 2.8247642815113068, "l_a": 2.9175659716129303, "l_c": 2.866628736257553, "z_loss": 27.90755295753479, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11997632682323456, "tokens_per_second": 1403313.4922296247, "eta_seconds": 14943.913692927359, "data_wait_seconds": 2.1189451217651367e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 32490, "total_steps": 72479, "tokens": 17034117120, "loss": 4.501516819000244, "l_mix": 2.7882758378982544, "l_a": 2.8794179260730743, "l_c": 2.8313847482204437, "z_loss": 28.30060911178589, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17137005925178528, "tokens_per_second": 1375338.7763517033, "eta_seconds": 15244.064366173743, "data_wait_seconds": 2.3573637008666992e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 32500, "total_steps": 72479, "tokens": 17039360000, "loss": 4.599517405033112, "l_mix": 2.8494902849197388, "l_a": 2.937029093503952, "l_c": 2.8963943123817444, "z_loss": 28.70296835899353, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1895281821489334, "tokens_per_second": 1392074.8891349735, "eta_seconds": 15057.027546143532, "data_wait_seconds": 2.7745962142944336e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.27438903, "gate_a2b/L1_absmax": 0.68863153, "gate_a2b/L1_rms": 0.28888604, "gate_a2b/L3_absmean": 0.28532097, "gate_a2b/L3_absmax": 0.62683934, "gate_a2b/L3_rms": 0.29117605, "gate_a2b/L5_absmean": 0.44807339, "gate_a2b/L5_absmax": 0.63279516, "gate_a2b/L5_rms": 0.45389083, "gate_c2b/L1_absmean": 0.01725878, "gate_c2b/L1_absmax": 0.55632764, "gate_c2b/L1_rms": 0.06546255, "gate_c2b/L3_absmean": 0.2179433, "gate_c2b/L3_absmax": 0.47746959, "gate_c2b/L3_rms": 0.22232416, "gate_c2b/L5_absmean": 0.58739036, "gate_c2b/L5_absmax": 0.82298338, "gate_c2b/L5_rms": 0.60081941, "gate_b2a/L7_absmean": 0.06660219, "gate_b2a/L7_absmax": 0.32319224, "gate_b2a/L7_rms": 0.07168654, "gate_b2a/L11_absmean": 0.10412043, "gate_b2a/L11_absmax": 0.56988424, "gate_b2a/L11_rms": 0.11051422, "gate_b2a/L14_absmean": 0.26663995, "gate_b2a/L14_absmax": 0.70634019, "gate_b2a/L14_rms": 0.2721402, "gate_b2c/L3_absmean": 0.05993845, "gate_b2c/L3_absmax": 0.66525322, "gate_b2c/L3_rms": 0.07329197, "gate_b2c/L5_absmean": 0.06923997, "gate_b2c/L5_absmax": 0.76052076, "gate_b2c/L5_rms": 0.08103192, "gate_b2c/L6_absmean": 0.10177314, "gate_b2c/L6_absmax": 0.52569377, "gate_b2c/L6_rms": 0.11047596}, "streams": {"cos_a2b/round0": 0.40028, "cos_c2b/round0": 0.300541, "cos_b2a/round0": 0.129992, "cos_b2c/round0": 0.135379, "rms_a/round0": 2.333503, "rms_b/round0": 4.388701, "rms_c/round0": 2.297055, "cos_a2b/round1": 0.436747, "cos_c2b/round1": 0.294102, "cos_b2a/round1": 0.244717, "cos_b2c/round1": 0.117726, "rms_a/round1": 4.206565, "rms_b/round1": 9.376866, "rms_c/round1": 4.264312, "cos_a2b/round2": 0.565316, "cos_c2b/round2": 0.519408, "cos_b2a/round2": 0.09948, "cos_b2c/round2": 0.164261, "rms_a/round2": 10.295305, "rms_b/round2": 22.820175, "rms_c/round2": 7.429072}, "alpha_mean": 0.4440255145156691, "alpha_hist": [0.00864, 0.06311, 0.15739, 0.19435, 0.19485, 0.17685, 0.12663, 0.06851, 0.00952, 0.00014], "alpha_by_type": {"special": {"share": 0.00096, "mean": 0.56491, "std": 0.12723}, "whitespace": {"share": 0.02209, "mean": 0.50264, "std": 0.12407}, "digit": {"share": 0.0303, "mean": 0.47014, "std": 0.11735}, "word_start": {"share": 0.58881, "mean": 0.46059, "std": 0.16169}, "word_cont": {"share": 0.2532, "mean": 0.33633, "std": 0.13394}, "punct": {"share": 0.10463, "mean": 0.59037, "std": 0.1624}}, "alpha_p05": 0.19284, "alpha_p50": 0.44155, "alpha_p95": 0.72272}
{"step": 32510, "total_steps": 72479, "tokens": 17044602880, "loss": 4.417782753705978, "l_mix": 2.737124055624008, "l_a": 2.824859857559204, "l_c": 2.7773353159427643, "z_loss": 27.998294830322266, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15731249749660492, "tokens_per_second": 1469088.543115823, "eta_seconds": 14264.128033804893, "data_wait_seconds": 3.8623809814453125e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 32520, "total_steps": 72479, "tokens": 17049845760, "loss": 4.5054216384887695, "l_mix": 2.790172129869461, "l_a": 2.882512927055359, "l_c": 2.8349851965904236, "z_loss": 28.374635457992554, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18190470337867737, "tokens_per_second": 1430828.1951714905, "eta_seconds": 14641.886609935762, "data_wait_seconds": 2.0951032638549805e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 32530, "total_steps": 72479, "tokens": 17055088640, "loss": 3.2677435278892517, "l_mix": 2.0229551643133163, "l_a": 2.0958645045757294, "l_c": 2.0534296929836273, "z_loss": 31.666083335876465, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.21442176401615143, "tokens_per_second": 1418274.9505895388, "eta_seconds": 14767.78624856472, "data_wait_seconds": 2.3573637008666992e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 32540, "total_steps": 72479, "tokens": 17060331520, "loss": 4.556131660938263, "l_mix": 2.8232359290122986, "l_a": 2.9081427454948425, "l_c": 2.8681761622428894, "z_loss": 27.14957332611084, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1931852251291275, "tokens_per_second": 1467854.235774013, "eta_seconds": 14265.407232999802, "data_wait_seconds": 2.4139881134033203e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 32550, "total_steps": 72479, "tokens": 17065574400, "loss": 4.494496524333954, "l_mix": 2.7842239141464233, "l_a": 2.877295345067978, "l_c": 2.823613256216049, "z_loss": 28.206899642944336, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1367863416671753, "tokens_per_second": 1310956.6052031848, "eta_seconds": 15968.717399883271, "data_wait_seconds": 2.1338462829589844e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 32560, "total_steps": 72479, "tokens": 17070817280, "loss": 4.463788241147995, "l_mix": 2.7651434540748596, "l_a": 2.8517046868801117, "l_c": 2.8104443550109863, "z_loss": 29.331928491592407, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14613084495067596, "tokens_per_second": 1452535.682064637, "eta_seconds": 14408.632387089729, "data_wait_seconds": 3.2067298889160156e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 32570, "total_steps": 72479, "tokens": 17076060160, "loss": 4.623311936855316, "l_mix": 2.8654864728450775, "l_a": 2.9524835646152496, "l_c": 2.9069347381591797, "z_loss": 27.897390842437744, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1546327769756317, "tokens_per_second": 1414171.413995686, "eta_seconds": 14795.80875763893, "data_wait_seconds": 1.704692840576172e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 32580, "total_steps": 72479, "tokens": 17081303040, "loss": 4.537464737892151, "l_mix": 2.8114072680473328, "l_a": 2.89888197183609, "l_c": 2.854643017053604, "z_loss": 28.632118701934814, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15893979370594025, "tokens_per_second": 1344490.7494340849, "eta_seconds": 15558.728775787353, "data_wait_seconds": 1.8388032913208008e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 32590, "total_steps": 72479, "tokens": 17086545920, "loss": 4.621559798717499, "l_mix": 2.863447368144989, "l_a": 2.9575918316841125, "l_c": 2.902782827615738, "z_loss": 27.467185020446777, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13158418238162994, "tokens_per_second": 1468857.44909384, "eta_seconds": 14237.817321825027, "data_wait_seconds": 1.5854835510253906e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 32600, "total_steps": 72479, "tokens": 17091788800, "loss": 4.457262873649597, "l_mix": 2.7610033750534058, "l_a": 2.8509358167648315, "l_c": 2.8032622635364532, "z_loss": 27.86003541946411, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13210339844226837, "tokens_per_second": 1436307.7754584167, "eta_seconds": 14556.825152134896, "data_wait_seconds": 3.11434268951416e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.27392998, "gate_a2b/L1_absmax": 0.69243026, "gate_a2b/L1_rms": 0.28833926, "gate_a2b/L3_absmean": 0.28371426, "gate_a2b/L3_absmax": 0.62791014, "gate_a2b/L3_rms": 0.28968573, "gate_a2b/L5_absmean": 0.44583121, "gate_a2b/L5_absmax": 0.62600279, "gate_a2b/L5_rms": 0.45169088, "gate_c2b/L1_absmean": 0.01685168, "gate_c2b/L1_absmax": 0.55927914, "gate_c2b/L1_rms": 0.06571037, "gate_c2b/L3_absmean": 0.21797919, "gate_c2b/L3_absmax": 0.48254159, "gate_c2b/L3_rms": 0.22226042, "gate_c2b/L5_absmean": 0.58807641, "gate_c2b/L5_absmax": 0.82020521, "gate_c2b/L5_rms": 0.6014064, "gate_b2a/L7_absmean": 0.06611206, "gate_b2a/L7_absmax": 0.32469901, "gate_b2a/L7_rms": 0.07119545, "gate_b2a/L11_absmean": 0.10385189, "gate_b2a/L11_absmax": 0.57447988, "gate_b2a/L11_rms": 0.1103781, "gate_b2a/L14_absmean": 0.26480937, "gate_b2a/L14_absmax": 0.71574599, "gate_b2a/L14_rms": 0.27038714, "gate_b2c/L3_absmean": 0.05951445, "gate_b2c/L3_absmax": 0.66600358, "gate_b2c/L3_rms": 0.07287361, "gate_b2c/L5_absmean": 0.0682964, "gate_b2c/L5_absmax": 0.75445235, "gate_b2c/L5_rms": 0.08001054, "gate_b2c/L6_absmean": 0.10050222, "gate_b2c/L6_absmax": 0.5273791, "gate_b2c/L6_rms": 0.10930668}, "streams": {"cos_a2b/round0": 0.40018, "cos_c2b/round0": 0.288352, "cos_b2a/round0": 0.110177, "cos_b2c/round0": 0.126547, "rms_a/round0": 2.411515, "rms_b/round0": 4.531864, "rms_c/round0": 2.350716, "cos_a2b/round1": 0.420833, "cos_c2b/round1": 0.270594, "cos_b2a/round1": 0.234379, "cos_b2c/round1": 0.106446, "rms_a/round1": 4.33685, "rms_b/round1": 9.845379, "rms_c/round1": 4.436149, "cos_a2b/round2": 0.522783, "cos_c2b/round2": 0.533408, "cos_b2a/round2": 0.080331, "cos_b2c/round2": 0.144161, "rms_a/round2": 10.528516, "rms_b/round2": 24.121428, "rms_c/round2": 7.84679}, "alpha_mean": 0.3984863729243493, "alpha_hist": [0.01469, 0.10092, 0.22079, 0.2169, 0.16287, 0.12961, 0.09518, 0.05203, 0.00684, 0.00015], "alpha_by_type": {"special": {"share": 0.00055, "mean": 0.57656, "std": 0.15562}, "whitespace": {"share": 0.02698, "mean": 0.49659, "std": 0.12866}, "digit": {"share": 0.07179, "mean": 0.39359, "std": 0.13428}, "word_start": {"share": 0.45952, "mean": 0.42069, "std": 0.17451}, "word_cont": {"share": 0.31464, "mean": 0.30188, "std": 0.11854}, "punct": {"share": 0.12653, "mean": 0.53919, "std": 0.17097}}, "alpha_p05": 0.17385, "alpha_p50": 0.38353, "alpha_p95": 0.70882}
{"step": 32610, "total_steps": 72479, "tokens": 17097031680, "loss": 4.526390731334686, "l_mix": 2.805147171020508, "l_a": 2.8914250135421753, "l_c": 2.846053272485733, "z_loss": 28.352341413497925, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1843416541814804, "tokens_per_second": 1346760.3257741416, "eta_seconds": 15520.83015215397, "data_wait_seconds": 1.9073486328125e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 32620, "total_steps": 72479, "tokens": 17102274560, "loss": 4.839467227458954, "l_mix": 3.0000025033950806, "l_a": 3.0852697789669037, "l_c": 3.0462790727615356, "z_loss": 27.73328995704651, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20346960425376892, "tokens_per_second": 1433254.6467377509, "eta_seconds": 14580.518151164053, "data_wait_seconds": 2.8580427169799805e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 32630, "total_steps": 72479, "tokens": 17107517440, "loss": 4.831342041492462, "l_mix": 2.9962291717529297, "l_a": 3.0779095888137817, "l_c": 3.039132922887802, "z_loss": 28.31066393852234, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18261565268039703, "tokens_per_second": 1477039.9328212535, "eta_seconds": 14144.744531106948, "data_wait_seconds": 1.5735626220703125e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 32640, "total_steps": 72479, "tokens": 17112760320, "loss": 3.200780302286148, "l_mix": 1.980672225356102, "l_a": 2.055878460407257, "l_c": 2.0111484080553055, "z_loss": 32.049590826034546, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2017679661512375, "tokens_per_second": 1376148.855021842, "eta_seconds": 15177.94354569912, "data_wait_seconds": 2.0682811737060547e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 32650, "total_steps": 72479, "tokens": 17118003200, "loss": 4.445456445217133, "l_mix": 2.753075957298279, "l_a": 2.8447532951831818, "l_c": 2.7965148091316223, "z_loss": 27.55009126663208, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16166366636753082, "tokens_per_second": 1435616.7028496135, "eta_seconds": 14545.572443223, "data_wait_seconds": 1.862645149230957e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 32660, "total_steps": 72479, "tokens": 17123246080, "loss": 4.365054100751877, "l_mix": 2.7039974331855774, "l_a": 2.792078882455826, "l_c": 2.7447767853736877, "z_loss": 27.71851372718811, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.137378990650177, "tokens_per_second": 1470393.944763788, "eta_seconds": 14197.980035448074, "data_wait_seconds": 2.9921531677246094e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 32670, "total_steps": 72479, "tokens": 17128488960, "loss": 4.562735915184021, "l_mix": 2.8254132866859436, "l_a": 2.924664944410324, "l_c": 2.8664104640483856, "z_loss": 28.101237058639526, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20301300287246704, "tokens_per_second": 1428063.664568732, "eta_seconds": 14615.161431407927, "data_wait_seconds": 2.4884939193725586e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 32680, "total_steps": 72479, "tokens": 17133731840, "loss": 4.480055093765259, "l_mix": 2.775948941707611, "l_a": 2.8658586144447327, "l_c": 2.814495086669922, "z_loss": 27.443202257156372, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1572621464729309, "tokens_per_second": 1458937.3233832074, "eta_seconds": 14302.285490655899, "data_wait_seconds": 1.7344951629638672e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 32690, "total_steps": 72479, "tokens": 17138974720, "loss": 4.539956510066986, "l_mix": 2.8116160333156586, "l_a": 2.902983456850052, "l_c": 2.8581513464450836, "z_loss": 28.47762393951416, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1360142081975937, "tokens_per_second": 1408692.4279463685, "eta_seconds": 14808.694089746476, "data_wait_seconds": 1.7195940017700195e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 32700, "total_steps": 72479, "tokens": 17144217600, "loss": 4.583928823471069, "l_mix": 2.839771121740341, "l_a": 2.9326104521751404, "l_c": 2.8812483847141266, "z_loss": 29.071861743927002, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1872994601726532, "tokens_per_second": 1439264.1880901395, "eta_seconds": 14490.496272039414, "data_wait_seconds": 1.6868114471435547e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.27191934, "gate_a2b/L1_absmax": 0.69242227, "gate_a2b/L1_rms": 0.28637224, "gate_a2b/L3_absmean": 0.28197476, "gate_a2b/L3_absmax": 0.6270442, "gate_a2b/L3_rms": 0.28784868, "gate_a2b/L5_absmean": 0.44391462, "gate_a2b/L5_absmax": 0.62726617, "gate_a2b/L5_rms": 0.4497692, "gate_c2b/L1_absmean": 0.01707254, "gate_c2b/L1_absmax": 0.56049711, "gate_c2b/L1_rms": 0.06565891, "gate_c2b/L3_absmean": 0.21885645, "gate_c2b/L3_absmax": 0.48082381, "gate_c2b/L3_rms": 0.22325377, "gate_c2b/L5_absmean": 0.58887589, "gate_c2b/L5_absmax": 0.82201844, "gate_c2b/L5_rms": 0.60223836, "gate_b2a/L7_absmean": 0.06566397, "gate_b2a/L7_absmax": 0.32609862, "gate_b2a/L7_rms": 0.07084385, "gate_b2a/L11_absmean": 0.10343374, "gate_b2a/L11_absmax": 0.57666367, "gate_b2a/L11_rms": 0.11002757, "gate_b2a/L14_absmean": 0.26383129, "gate_b2a/L14_absmax": 0.72026253, "gate_b2a/L14_rms": 0.26953623, "gate_b2c/L3_absmean": 0.05890004, "gate_b2c/L3_absmax": 0.66504544, "gate_b2c/L3_rms": 0.07228009, "gate_b2c/L5_absmean": 0.06731662, "gate_b2c/L5_absmax": 0.7529124, "gate_b2c/L5_rms": 0.07914688, "gate_b2c/L6_absmean": 0.09953775, "gate_b2c/L6_absmax": 0.52956152, "gate_b2c/L6_rms": 0.1083222}, "streams": {"cos_a2b/round0": 0.391935, "cos_c2b/round0": 0.292124, "cos_b2a/round0": 0.128951, "cos_b2c/round0": 0.128219, "rms_a/round0": 2.393381, "rms_b/round0": 4.572574, "rms_c/round0": 2.346985, "cos_a2b/round1": 0.423785, "cos_c2b/round1": 0.292432, "cos_b2a/round1": 0.239805, "cos_b2c/round1": 0.123784, "rms_a/round1": 4.378582, "rms_b/round1": 9.861825, "rms_c/round1": 4.443518, "cos_a2b/round2": 0.56087, "cos_c2b/round2": 0.510292, "cos_b2a/round2": 0.093366, "cos_b2c/round2": 0.168814, "rms_a/round2": 10.773552, "rms_b/round2": 23.719927, "rms_c/round2": 7.762177}, "alpha_mean": 0.4334897506885227, "alpha_hist": [0.01558, 0.07399, 0.16914, 0.19411, 0.18053, 0.16664, 0.12479, 0.06529, 0.00966, 0.00027], "alpha_by_type": {"special": {"share": 0.00125, "mean": 0.54434, "std": 0.12524}, "whitespace": {"share": 0.02739, "mean": 0.51113, "std": 0.12911}, "digit": {"share": 0.02931, "mean": 0.46831, "std": 0.12166}, "word_start": {"share": 0.558, "mean": 0.45116, "std": 0.16911}, "word_cont": {"share": 0.27142, "mean": 0.3271, "std": 0.1382}, "punct": {"share": 0.11263, "mean": 0.57316, "std": 0.17023}}, "alpha_p05": 0.15662, "alpha_p50": 0.41135, "alpha_p95": 0.71522}
{"step": 32710, "total_steps": 72479, "tokens": 17149460480, "loss": 4.772277772426605, "l_mix": 2.959747225046158, "l_a": 3.038343459367752, "l_c": 3.0034248530864716, "z_loss": 27.847474575042725, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15970665216445923, "tokens_per_second": 1443925.8448186822, "eta_seconds": 14440.083295702934, "data_wait_seconds": 1.7374753952026367e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 32720, "total_steps": 72479, "tokens": 17154703360, "loss": 4.866810739040375, "l_mix": 3.0187588334083557, "l_a": 3.0999352633953094, "l_c": 3.0602376461029053, "z_loss": 27.449143648147583, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1482428014278412, "tokens_per_second": 1372005.246851955, "eta_seconds": 15193.212008357048, "data_wait_seconds": 1.7136335372924805e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 32730, "total_steps": 72479, "tokens": 17159946240, "loss": 4.788905382156372, "l_mix": 2.9693703353405, "l_a": 3.0500219762325287, "l_c": 3.0150949358940125, "z_loss": 27.8582820892334, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1859244555234909, "tokens_per_second": 1445609.0929967128, "eta_seconds": 14416.015929174422, "data_wait_seconds": 2.434849739074707e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 32740, "total_steps": 72479, "tokens": 17165189120, "loss": 4.778917729854584, "l_mix": 2.9638420045375824, "l_a": 3.0449257493019104, "l_c": 3.005326747894287, "z_loss": 26.948613166809082, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18918102979660034, "tokens_per_second": 1353493.8733365943, "eta_seconds": 15393.258323836326, "data_wait_seconds": 3.4481287002563477e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 32750, "total_steps": 72479, "tokens": 17170432000, "loss": 4.806702077388763, "l_mix": 2.9812537729740143, "l_a": 3.0614043474197388, "l_c": 3.023423135280609, "z_loss": 28.45221781730652, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19182078540325165, "tokens_per_second": 1381912.927807659, "eta_seconds": 15072.901868748664, "data_wait_seconds": 2.4706125259399414e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 32760, "total_steps": 72479, "tokens": 17175674880, "loss": 4.640035271644592, "l_mix": 2.8756607472896576, "l_a": 2.9632617235183716, "l_c": 2.9179867208004, "z_loss": 28.798776865005493, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11813928931951523, "tokens_per_second": 1345352.867857584, "eta_seconds": 15478.61202032566, "data_wait_seconds": 1.952052116394043e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 32770, "total_steps": 72479, "tokens": 17180917760, "loss": 4.493029773235321, "l_mix": 2.783743530511856, "l_a": 2.8708916902542114, "l_c": 2.826728731393814, "z_loss": 28.419039011001587, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.115780770778656, "tokens_per_second": 1473283.6045984454, "eta_seconds": 14130.987494206429, "data_wait_seconds": 1.570582389831543e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 32780, "total_steps": 72479, "tokens": 17186160640, "loss": 4.39136666059494, "l_mix": 2.720657557249069, "l_a": 2.805828809738159, "l_c": 2.763201057910919, "z_loss": 27.566115617752075, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13108111917972565, "tokens_per_second": 1407022.6101164303, "eta_seconds": 14792.732655715943, "data_wait_seconds": 1.9818544387817383e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 32790, "total_steps": 72479, "tokens": 17191403520, "loss": 4.566320955753326, "l_mix": 2.8283518850803375, "l_a": 2.920033276081085, "l_c": 2.87319678068161, "z_loss": 28.180471181869507, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18661709129810333, "tokens_per_second": 1442446.1263233598, "eta_seconds": 14425.818789529802, "data_wait_seconds": 1.6480684280395508e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 32800, "total_steps": 72479, "tokens": 17196646400, "loss": 4.513650894165039, "l_mix": 2.797513335943222, "l_a": 2.8848524689674377, "l_c": 2.8356058299541473, "z_loss": 28.053640365600586, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2110031694173813, "tokens_per_second": 1459667.4101346578, "eta_seconds": 14252.029885411263, "data_wait_seconds": 1.6421079635620117e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.27066007, "gate_a2b/L1_absmax": 0.69645423, "gate_a2b/L1_rms": 0.2851409, "gate_a2b/L3_absmean": 0.28095222, "gate_a2b/L3_absmax": 0.63366055, "gate_a2b/L3_rms": 0.28684881, "gate_a2b/L5_absmean": 0.44135746, "gate_a2b/L5_absmax": 0.62358493, "gate_a2b/L5_rms": 0.44718963, "gate_c2b/L1_absmean": 0.01690343, "gate_c2b/L1_absmax": 0.56555086, "gate_c2b/L1_rms": 0.06594343, "gate_c2b/L3_absmean": 0.21778469, "gate_c2b/L3_absmax": 0.4858233, "gate_c2b/L3_rms": 0.22216691, "gate_c2b/L5_absmean": 0.58921635, "gate_c2b/L5_absmax": 0.82813263, "gate_c2b/L5_rms": 0.60262507, "gate_b2a/L7_absmean": 0.06540559, "gate_b2a/L7_absmax": 0.32700074, "gate_b2a/L7_rms": 0.07059447, "gate_b2a/L11_absmean": 0.10273235, "gate_b2a/L11_absmax": 0.57453638, "gate_b2a/L11_rms": 0.10934647, "gate_b2a/L14_absmean": 0.26225513, "gate_b2a/L14_absmax": 0.71976846, "gate_b2a/L14_rms": 0.26800796, "gate_b2c/L3_absmean": 0.05834782, "gate_b2c/L3_absmax": 0.66396505, "gate_b2c/L3_rms": 0.07161347, "gate_b2c/L5_absmean": 0.06621823, "gate_b2c/L5_absmax": 0.75191218, "gate_b2c/L5_rms": 0.07807407, "gate_b2c/L6_absmean": 0.0977663, "gate_b2c/L6_absmax": 0.52594399, "gate_b2c/L6_rms": 0.10663769}, "streams": {"cos_a2b/round0": 0.402688, "cos_c2b/round0": 0.302521, "cos_b2a/round0": 0.131861, "cos_b2c/round0": 0.136083, "rms_a/round0": 2.46185, "rms_b/round0": 4.648785, "rms_c/round0": 2.351449, "cos_a2b/round1": 0.432585, "cos_c2b/round1": 0.290948, "cos_b2a/round1": 0.247024, "cos_b2c/round1": 0.120036, "rms_a/round1": 4.478741, "rms_b/round1": 9.986901, "rms_c/round1": 4.511281, "cos_a2b/round2": 0.563025, "cos_c2b/round2": 0.51697, "cos_b2a/round2": 0.097018, "cos_b2c/round2": 0.166347, "rms_a/round2": 10.97559, "rms_b/round2": 24.20437, "rms_c/round2": 7.994925}, "alpha_mean": 0.4409408471527172, "alpha_hist": [0.00742, 0.05301, 0.16531, 0.2191, 0.19524, 0.16196, 0.11946, 0.0675, 0.01079, 0.00021], "alpha_by_type": {"special": {"share": 0.00134, "mean": 0.55235, "std": 0.1158}, "whitespace": {"share": 0.03325, "mean": 0.51986, "std": 0.13606}, "digit": {"share": 0.04213, "mean": 0.44344, "std": 0.11795}, "word_start": {"share": 0.56281, "mean": 0.4565, "std": 0.16122}, "word_cont": {"share": 0.25681, "mean": 0.33715, "std": 0.12754}, "punct": {"share": 0.10367, "mean": 0.58578, "std": 0.1612}}, "alpha_p05": 0.18126, "alpha_p50": 0.42227, "alpha_p95": 0.71879}
{"step": 32810, "total_steps": 72479, "tokens": 17201889280, "loss": 4.710513055324554, "l_mix": 2.92016464471817, "l_a": 3.006023734807968, "l_c": 2.9618041813373566, "z_loss": 27.398982048034668, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16802242398262024, "tokens_per_second": 1328273.2971338895, "eta_seconds": 15657.9076887846, "data_wait_seconds": 2.1338462829589844e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 32820, "total_steps": 72479, "tokens": 17207132160, "loss": 4.714943587779999, "l_mix": 2.9244669377803802, "l_a": 3.0045996606349945, "l_c": 2.963655561208725, "z_loss": 28.426875591278076, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15425622463226318, "tokens_per_second": 1475817.5889637126, "eta_seconds": 14088.961906599998, "data_wait_seconds": 1.6182661056518555e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20768, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 32830, "total_steps": 72479, "tokens": 17212375040, "loss": 4.8846030831336975, "l_mix": 3.028685301542282, "l_a": 3.109212964773178, "l_c": 3.0771796107292175, "z_loss": 27.617783308029175, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1943991780281067, "tokens_per_second": 1464349.7570410406, "eta_seconds": 14195.717117476463, "data_wait_seconds": 2.0205974578857422e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 32840, "total_steps": 72479, "tokens": 17217617920, "loss": 4.776533901691437, "l_mix": 2.9622747600078583, "l_a": 3.046392500400543, "l_c": 3.001137912273407, "z_loss": 27.249204874038696, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1532551646232605, "tokens_per_second": 1417999.4886176598, "eta_seconds": 14656.036337685586, "data_wait_seconds": 3.37064266204834e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 32850, "total_steps": 72479, "tokens": 17222860800, "loss": 4.71846330165863, "l_mix": 2.9264741241931915, "l_a": 3.004337042570114, "l_c": 2.9689598977565765, "z_loss": 27.35868239402771, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15439166128635406, "tokens_per_second": 1357260.0644194859, "eta_seconds": 15308.053111314774, "data_wait_seconds": 2.205371856689453e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 32860, "total_steps": 72479, "tokens": 17228103680, "loss": 4.823505401611328, "l_mix": 2.991330921649933, "l_a": 3.0734591484069824, "l_c": 3.0337889790534973, "z_loss": 27.928870916366577, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1323605179786682, "tokens_per_second": 1428288.6863403397, "eta_seconds": 14543.114757299421, "data_wait_seconds": 1.7642974853515625e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 32870, "total_steps": 72479, "tokens": 17233346560, "loss": 4.5071792006492615, "l_mix": 2.79238760471344, "l_a": 2.8818549513816833, "l_c": 2.834116756916046, "z_loss": 28.539902448654175, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11981969326734543, "tokens_per_second": 1420897.7378319553, "eta_seconds": 14615.072456717491, "data_wait_seconds": 1.722574234008789e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08232}}
{"step": 32880, "total_steps": 72479, "tokens": 17238589440, "loss": 4.484457850456238, "l_mix": 2.7780369222164154, "l_a": 2.8686289191246033, "l_c": 2.819440573453903, "z_loss": 28.838597059249878, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13764886558055878, "tokens_per_second": 1410141.0161706002, "eta_seconds": 14722.839967012405, "data_wait_seconds": 1.8775463104248047e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 32890, "total_steps": 72479, "tokens": 17243832320, "loss": 4.508088767528534, "l_mix": 2.7919941842556, "l_a": 2.8827435076236725, "l_c": 2.837571531534195, "z_loss": 28.977489709854126, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12943808734416962, "tokens_per_second": 1459174.794984128, "eta_seconds": 14224.50394794941, "data_wait_seconds": 1.8894672393798828e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 32900, "total_steps": 72479, "tokens": 17249075200, "loss": 4.523419141769409, "l_mix": 2.8025735914707184, "l_a": 2.8930299878120422, "l_c": 2.8431217074394226, "z_loss": 28.06053638458252, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13315358757972717, "tokens_per_second": 1431218.479635096, "eta_seconds": 14498.691183257102, "data_wait_seconds": 1.6480684280395508e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.27003506, "gate_a2b/L1_absmax": 0.69514936, "gate_a2b/L1_rms": 0.28443435, "gate_a2b/L3_absmean": 0.27966148, "gate_a2b/L3_absmax": 0.63012969, "gate_a2b/L3_rms": 0.28558189, "gate_a2b/L5_absmean": 0.43962535, "gate_a2b/L5_absmax": 0.62315655, "gate_a2b/L5_rms": 0.44544435, "gate_c2b/L1_absmean": 0.01690509, "gate_c2b/L1_absmax": 0.5645017, "gate_c2b/L1_rms": 0.0660089, "gate_c2b/L3_absmean": 0.21826625, "gate_c2b/L3_absmax": 0.48539141, "gate_c2b/L3_rms": 0.22261168, "gate_c2b/L5_absmean": 0.58961052, "gate_c2b/L5_absmax": 0.83448142, "gate_c2b/L5_rms": 0.60309035, "gate_b2a/L7_absmean": 0.06470758, "gate_b2a/L7_absmax": 0.32803726, "gate_b2a/L7_rms": 0.069906, "gate_b2a/L11_absmean": 0.1019164, "gate_b2a/L11_absmax": 0.57588595, "gate_b2a/L11_rms": 0.10857022, "gate_b2a/L14_absmean": 0.2610794, "gate_b2a/L14_absmax": 0.72105521, "gate_b2a/L14_rms": 0.26684779, "gate_b2c/L3_absmean": 0.05796288, "gate_b2c/L3_absmax": 0.66188073, "gate_b2c/L3_rms": 0.07109842, "gate_b2c/L5_absmean": 0.06600835, "gate_b2c/L5_absmax": 0.75241435, "gate_b2c/L5_rms": 0.07783203, "gate_b2c/L6_absmean": 0.09673582, "gate_b2c/L6_absmax": 0.52144265, "gate_b2c/L6_rms": 0.10556903}, "streams": {"cos_a2b/round0": 0.396604, "cos_c2b/round0": 0.296215, "cos_b2a/round0": 0.129003, "cos_b2c/round0": 0.130109, "rms_a/round0": 2.457423, "rms_b/round0": 4.687459, "rms_c/round0": 2.384566, "cos_a2b/round1": 0.425597, "cos_c2b/round1": 0.290471, "cos_b2a/round1": 0.23673, "cos_b2c/round1": 0.1238, "rms_a/round1": 4.519221, "rms_b/round1": 10.18464, "rms_c/round1": 4.532594, "cos_a2b/round2": 0.556092, "cos_c2b/round2": 0.511813, "cos_b2a/round2": 0.093683, "cos_b2c/round2": 0.165719, "rms_a/round2": 11.066296, "rms_b/round2": 24.582783, "rms_c/round2": 8.027346}, "alpha_mean": 0.4363743946105032, "alpha_hist": [0.01099, 0.07278, 0.16994, 0.19586, 0.18883, 0.16055, 0.11536, 0.07263, 0.01282, 0.00024], "alpha_by_type": {"special": {"share": 0.00146, "mean": 0.52287, "std": 0.15317}, "whitespace": {"share": 0.03232, "mean": 0.51206, "std": 0.12715}, "digit": {"share": 0.04298, "mean": 0.45194, "std": 0.11136}, "word_start": {"share": 0.55527, "mean": 0.45011, "std": 0.17344}, "word_cont": {"share": 0.26692, "mean": 0.33807, "std": 0.13735}, "punct": {"share": 0.10104, "mean": 0.58848, "std": 0.16572}}, "alpha_p05": 0.16941, "alpha_p50": 0.44529, "alpha_p95": 0.73867}
{"step": 32910, "total_steps": 72479, "tokens": 17254318080, "loss": 4.513598680496216, "l_mix": 2.796442210674286, "l_a": 2.885134696960449, "l_c": 2.838720142841339, "z_loss": 28.48550772666931, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11852408200502396, "tokens_per_second": 1406871.201153328, "eta_seconds": 14745.878553056717, "data_wait_seconds": 1.7374753952026367e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 32920, "total_steps": 72479, "tokens": 17259560960, "loss": 4.578308999538422, "l_mix": 2.8364025950431824, "l_a": 2.926759034395218, "l_c": 2.8795954883098602, "z_loss": 27.966339111328125, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1505013406276703, "tokens_per_second": 1302588.8816785554, "eta_seconds": 15922.375266456605, "data_wait_seconds": 1.6242265701293945e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 32930, "total_steps": 72479, "tokens": 17264803840, "loss": 2.9666015207767487, "l_mix": 1.8363036066293716, "l_a": 1.9038738757371902, "l_c": 1.863785520195961, "z_loss": 33.57669973373413, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18200024962425232, "tokens_per_second": 1436221.6598562787, "eta_seconds": 14437.232560658456, "data_wait_seconds": 2.6553869247436523e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08236}}
{"step": 32940, "total_steps": 72479, "tokens": 17270046720, "loss": 4.730567991733551, "l_mix": 2.9338632822036743, "l_a": 3.016643673181534, "l_c": 2.97237166762352, "z_loss": 28.829660654067993, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.21660587191581726, "tokens_per_second": 1347435.1060007839, "eta_seconds": 15384.654251384734, "data_wait_seconds": 2.3812055587768555e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 32950, "total_steps": 72479, "tokens": 17275289600, "loss": 4.794354736804962, "l_mix": 2.972768545150757, "l_a": 3.0553890466690063, "l_c": 3.016564756631851, "z_loss": 28.06050443649292, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17867058515548706, "tokens_per_second": 1456908.9486270451, "eta_seconds": 14225.034702086448, "data_wait_seconds": 2.1070241928100586e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 32960, "total_steps": 72479, "tokens": 17280532480, "loss": 4.719622492790222, "l_mix": 2.9272844195365906, "l_a": 3.0074823200702667, "l_c": 2.9669778645038605, "z_loss": 28.733736991882324, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2511529326438904, "tokens_per_second": 1422558.7061593423, "eta_seconds": 14564.838261008263, "data_wait_seconds": 2.250075340270996e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 32970, "total_steps": 72479, "tokens": 17285775360, "loss": 4.798599421977997, "l_mix": 2.975324898958206, "l_a": 3.0585595965385437, "l_c": 3.019022047519684, "z_loss": 28.27019762992859, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17646539211273193, "tokens_per_second": 1422137.3518918462, "eta_seconds": 14565.466946244242, "data_wait_seconds": 1.615285873413086e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 32980, "total_steps": 72479, "tokens": 17291018240, "loss": 4.771601378917694, "l_mix": 2.9595534801483154, "l_a": 3.0407376885414124, "l_c": 2.9994218945503235, "z_loss": 27.23449397087097, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15249457955360413, "tokens_per_second": 1409680.2633785673, "eta_seconds": 14690.460134816169, "data_wait_seconds": 1.5616416931152344e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 32990, "total_steps": 72479, "tokens": 17296261120, "loss": 4.6892505288124084, "l_mix": 2.9068617820739746, "l_a": 2.99457848072052, "l_c": 2.946717143058777, "z_loss": 28.241771459579468, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12442474812269211, "tokens_per_second": 1457387.188590005, "eta_seconds": 14205.976966238022, "data_wait_seconds": 1.6033649444580078e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 33000, "total_steps": 72479, "tokens": 17301504000, "loss": 4.5038323402404785, "l_mix": 2.789638012647629, "l_a": 2.8821719586849213, "l_c": 2.831808865070343, "z_loss": 28.98920512199402, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11300677806138992, "tokens_per_second": 1417714.170926426, "eta_seconds": 14599.8159406662, "data_wait_seconds": 1.755356788635254e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.26903796, "gate_a2b/L1_absmax": 0.69457483, "gate_a2b/L1_rms": 0.28352877, "gate_a2b/L3_absmean": 0.2784079, "gate_a2b/L3_absmax": 0.63182658, "gate_a2b/L3_rms": 0.28428206, "gate_a2b/L5_absmean": 0.4381322, "gate_a2b/L5_absmax": 0.6248228, "gate_a2b/L5_rms": 0.44394189, "gate_c2b/L1_absmean": 0.01687315, "gate_c2b/L1_absmax": 0.56525278, "gate_c2b/L1_rms": 0.06590439, "gate_c2b/L3_absmean": 0.21780829, "gate_c2b/L3_absmax": 0.48122078, "gate_c2b/L3_rms": 0.22218643, "gate_c2b/L5_absmean": 0.58993739, "gate_c2b/L5_absmax": 0.82956153, "gate_c2b/L5_rms": 0.60343111, "gate_b2a/L7_absmean": 0.06457788, "gate_b2a/L7_absmax": 0.32405823, "gate_b2a/L7_rms": 0.06968386, "gate_b2a/L11_absmean": 0.10114492, "gate_b2a/L11_absmax": 0.57518882, "gate_b2a/L11_rms": 0.10787074, "gate_b2a/L14_absmean": 0.25963297, "gate_b2a/L14_absmax": 0.7216019, "gate_b2a/L14_rms": 0.26545402, "gate_b2c/L3_absmean": 0.05717922, "gate_b2c/L3_absmax": 0.66100812, "gate_b2c/L3_rms": 0.07041329, "gate_b2c/L5_absmean": 0.06527653, "gate_b2c/L5_absmax": 0.75075811, "gate_b2c/L5_rms": 0.0771342, "gate_b2c/L6_absmean": 0.09593157, "gate_b2c/L6_absmax": 0.51966912, "gate_b2c/L6_rms": 0.10472691}, "streams": {"cos_a2b/round0": 0.399147, "cos_c2b/round0": 0.298549, "cos_b2a/round0": 0.122816, "cos_b2c/round0": 0.134502, "rms_a/round0": 2.509416, "rms_b/round0": 4.740718, "rms_c/round0": 2.371249, "cos_a2b/round1": 0.426664, "cos_c2b/round1": 0.286431, "cos_b2a/round1": 0.227389, "cos_b2c/round1": 0.119094, "rms_a/round1": 4.586193, "rms_b/round1": 10.2332, "rms_c/round1": 4.524165, "cos_a2b/round2": 0.563763, "cos_c2b/round2": 0.501015, "cos_b2a/round2": 0.097838, "cos_b2c/round2": 0.164092, "rms_a/round2": 11.208604, "rms_b/round2": 24.612816, "rms_c/round2": 7.907576}, "alpha_mean": 0.4709101888838987, "alpha_hist": [0.00682, 0.04738, 0.13658, 0.17807, 0.18785, 0.18204, 0.15295, 0.09061, 0.01749, 0.00021], "alpha_by_type": {"special": {"share": 0.00099, "mean": 0.5518, "std": 0.16423}, "whitespace": {"share": 0.02864, "mean": 0.58845, "std": 0.15295}, "digit": {"share": 0.05452, "mean": 0.47869, "std": 0.10617}, "word_start": {"share": 0.56239, "mean": 0.48787, "std": 0.16352}, "word_cont": {"share": 0.2384, "mean": 0.34484, "std": 0.14092}, "punct": {"share": 0.11505, "mean": 0.61558, "std": 0.14556}}, "alpha_p05": 0.18074, "alpha_p50": 0.47731, "alpha_p95": 0.76101}
{"step": 33010, "total_steps": 72479, "tokens": 17306746880, "loss": 4.491357684135437, "l_mix": 2.7828208208084106, "l_a": 2.8712737262248993, "l_c": 2.8238489627838135, "z_loss": 28.493810892105103, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11578869819641113, "tokens_per_second": 1426846.012339784, "eta_seconds": 14502.702389073373, "data_wait_seconds": 1.806020736694336e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 33020, "total_steps": 72479, "tokens": 17311989760, "loss": 4.459984183311462, "l_mix": 2.762354612350464, "l_a": 2.854867398738861, "l_c": 2.803897738456726, "z_loss": 28.64714765548706, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12255873531103134, "tokens_per_second": 1421005.440572294, "eta_seconds": 14558.621382665635, "data_wait_seconds": 2.1338462829589844e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 33030, "total_steps": 72479, "tokens": 17317232640, "loss": 4.557611167430878, "l_mix": 2.8236949145793915, "l_a": 2.9147203266620636, "l_c": 2.865000367164612, "z_loss": 28.675186157226562, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1382737010717392, "tokens_per_second": 1400356.9666079849, "eta_seconds": 14769.5464836359, "data_wait_seconds": 2.8640031814575195e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 33040, "total_steps": 72479, "tokens": 17322475520, "loss": 2.9978663623332977, "l_mix": 1.8543246686458588, "l_a": 1.9276464730501175, "l_c": 1.884158805012703, "z_loss": 34.71343278884888, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17312170565128326, "tokens_per_second": 1420534.6254088515, "eta_seconds": 14556.065063214302, "data_wait_seconds": 4.938244819641113e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 33050, "total_steps": 72479, "tokens": 17327718400, "loss": 3.239778518676758, "l_mix": 2.005765423178673, "l_a": 2.078817829489708, "l_c": 2.034558981657028, "z_loss": 32.22448372840881, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20721690356731415, "tokens_per_second": 1408207.4608667833, "eta_seconds": 14679.762837839125, "data_wait_seconds": 4.723668098449707e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08236}}
{"step": 33060, "total_steps": 72479, "tokens": 17332961280, "loss": 4.475981175899506, "l_mix": 2.7732705175876617, "l_a": 2.860113650560379, "l_c": 2.815588265657425, "z_loss": 29.183172702789307, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11218719184398651, "tokens_per_second": 1384285.096734003, "eta_seconds": 14929.66204776764, "data_wait_seconds": 1.901388168334961e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 33070, "total_steps": 72479, "tokens": 17338204160, "loss": 4.421486914157867, "l_mix": 2.7384836673736572, "l_a": 2.8297895193099976, "l_c": 2.7802212238311768, "z_loss": 29.466073036193848, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12175917625427246, "tokens_per_second": 1379474.7961215526, "eta_seconds": 14977.921923685073, "data_wait_seconds": 2.372264862060547e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 33080, "total_steps": 72479, "tokens": 17343447040, "loss": 4.581470847129822, "l_mix": 2.8377476632595062, "l_a": 2.930841028690338, "l_c": 2.8815695345401764, "z_loss": 28.56948733329773, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14946284890174866, "tokens_per_second": 1453230.65854179, "eta_seconds": 14214.139228749274, "data_wait_seconds": 3.263354301452637e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 33090, "total_steps": 72479, "tokens": 17348689920, "loss": 4.513449013233185, "l_mix": 2.796257585287094, "l_a": 2.885146200656891, "l_c": 2.838824898004532, "z_loss": 29.13291907310486, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1424698680639267, "tokens_per_second": 1385751.8336611253, "eta_seconds": 14902.509619951246, "data_wait_seconds": 1.71661376953125e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 33100, "total_steps": 72479, "tokens": 17353932800, "loss": 4.5190112590789795, "l_mix": 2.8001619577407837, "l_a": 2.8902619779109955, "l_c": 2.8392354249954224, "z_loss": 28.354456424713135, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13277678191661835, "tokens_per_second": 1446763.4228336855, "eta_seconds": 14270.430691123009, "data_wait_seconds": 2.530217170715332e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20042, "finemath": 0.08236}, "gates": {"gate_a2b/L1_absmean": 0.26741949, "gate_a2b/L1_absmax": 0.69393128, "gate_a2b/L1_rms": 0.28188947, "gate_a2b/L3_absmean": 0.27704957, "gate_a2b/L3_absmax": 0.62530392, "gate_a2b/L3_rms": 0.28300145, "gate_a2b/L5_absmean": 0.43667874, "gate_a2b/L5_absmax": 0.62680942, "gate_a2b/L5_rms": 0.44248658, "gate_c2b/L1_absmean": 0.01741672, "gate_c2b/L1_absmax": 0.56299138, "gate_c2b/L1_rms": 0.06607591, "gate_c2b/L3_absmean": 0.21819626, "gate_c2b/L3_absmax": 0.48338935, "gate_c2b/L3_rms": 0.22254929, "gate_c2b/L5_absmean": 0.59074336, "gate_c2b/L5_absmax": 0.83436841, "gate_c2b/L5_rms": 0.60422826, "gate_b2a/L7_absmean": 0.06429638, "gate_b2a/L7_absmax": 0.32611567, "gate_b2a/L7_rms": 0.06949105, "gate_b2a/L11_absmean": 0.10137025, "gate_b2a/L11_absmax": 0.57880026, "gate_b2a/L11_rms": 0.1081447, "gate_b2a/L14_absmean": 0.25903091, "gate_b2a/L14_absmax": 0.72463799, "gate_b2a/L14_rms": 0.26488686, "gate_b2c/L3_absmean": 0.05701321, "gate_b2c/L3_absmax": 0.6593259, "gate_b2c/L3_rms": 0.07015195, "gate_b2c/L5_absmean": 0.06472545, "gate_b2c/L5_absmax": 0.74983895, "gate_b2c/L5_rms": 0.07665296, "gate_b2c/L6_absmean": 0.09471846, "gate_b2c/L6_absmax": 0.51880747, "gate_b2c/L6_rms": 0.10351568}, "streams": {"cos_a2b/round0": 0.394117, "cos_c2b/round0": 0.288613, "cos_b2a/round0": 0.133219, "cos_b2c/round0": 0.128008, "rms_a/round0": 2.537878, "rms_b/round0": 4.861596, "rms_c/round0": 2.418458, "cos_a2b/round1": 0.426055, "cos_c2b/round1": 0.291736, "cos_b2a/round1": 0.242006, "cos_b2c/round1": 0.122754, "rms_a/round1": 4.632694, "rms_b/round1": 10.439916, "rms_c/round1": 4.584866, "cos_a2b/round2": 0.562036, "cos_c2b/round2": 0.511315, "cos_b2a/round2": 0.096096, "cos_b2c/round2": 0.170717, "rms_a/round2": 11.454302, "rms_b/round2": 25.264862, "rms_c/round2": 8.069118}, "alpha_mean": 0.45459788804416945, "alpha_hist": [0.00648, 0.05182, 0.15237, 0.1927, 0.19278, 0.17964, 0.13713, 0.07565, 0.01131, 0.00011], "alpha_by_type": {"special": {"share": 0.00105, "mean": 0.57103, "std": 0.12778}, "whitespace": {"share": 0.02686, "mean": 0.54535, "std": 0.12167}, "digit": {"share": 0.03819, "mean": 0.49163, "std": 0.1065}, "word_start": {"share": 0.55803, "mean": 0.47654, "std": 0.16146}, "word_cont": {"share": 0.25294, "mean": 0.34254, "std": 0.13198}, "punct": {"share": 0.12292, "mean": 0.55325, "std": 0.18009}}, "alpha_p05": 0.18536, "alpha_p50": 0.45702, "alpha_p95": 0.73602}
{"step": 33110, "total_steps": 72479, "tokens": 17359175680, "loss": 4.605286955833435, "l_mix": 2.85355344414711, "l_a": 2.9415198266506195, "l_c": 2.897591471672058, "z_loss": 28.897096872329712, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.10700041055679321, "tokens_per_second": 1415883.7787098144, "eta_seconds": 14577.958009243011, "data_wait_seconds": 1.5944242477416992e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 33120, "total_steps": 72479, "tokens": 17364418560, "loss": 4.431240260601044, "l_mix": 2.7441126704216003, "l_a": 2.8362329304218292, "l_c": 2.787525624036789, "z_loss": 29.430764198303223, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12985815107822418, "tokens_per_second": 1379604.7857111846, "eta_seconds": 14957.509285068512, "data_wait_seconds": 1.5854835510253906e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 33130, "total_steps": 72479, "tokens": 17369661440, "loss": 4.497071385383606, "l_mix": 2.785669684410095, "l_a": 2.875617265701294, "l_c": 2.829054981470108, "z_loss": 28.742648124694824, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13641375303268433, "tokens_per_second": 1432484.0134142665, "eta_seconds": 14401.7024405241, "data_wait_seconds": 2.5928020477294922e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 33140, "total_steps": 72479, "tokens": 17374904320, "loss": 4.566191375255585, "l_mix": 2.8290654122829437, "l_a": 2.9176307916641235, "l_c": 2.8727890253067017, "z_loss": 28.843121767044067, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14624907076358795, "tokens_per_second": 1390944.380873104, "eta_seconds": 14828.030448675156, "data_wait_seconds": 2.0384788513183594e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 33150, "total_steps": 72479, "tokens": 17380147200, "loss": 3.12528994679451, "l_mix": 1.9347620606422424, "l_a": 2.0058739334344864, "l_c": 1.9625521004199982, "z_loss": 33.30095887184143, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15202441811561584, "tokens_per_second": 1385767.552430172, "eta_seconds": 14879.640323400497, "data_wait_seconds": 2.2530555725097656e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 33160, "total_steps": 72479, "tokens": 17385390080, "loss": 2.8574127852916718, "l_mix": 1.7673929184675217, "l_a": 1.8337800204753876, "l_c": 1.799619346857071, "z_loss": 34.866347312927246, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17051321268081665, "tokens_per_second": 1461465.2802667322, "eta_seconds": 14105.35039753914, "data_wait_seconds": 1.6689300537109375e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08236}}
{"step": 33170, "total_steps": 72479, "tokens": 17390632960, "loss": 4.59315150976181, "l_mix": 2.8457071781158447, "l_a": 2.935613662004471, "l_c": 2.889200657606125, "z_loss": 28.752673864364624, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11813735961914062, "tokens_per_second": 1444002.6460310211, "eta_seconds": 14272.298633694649, "data_wait_seconds": 1.8298625946044922e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 33180, "total_steps": 72479, "tokens": 17395875840, "loss": 4.481311023235321, "l_mix": 2.7760688960552216, "l_a": 2.86661097407341, "l_c": 2.817529112100601, "z_loss": 28.650121212005615, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.119706891477108, "tokens_per_second": 1472747.7281160976, "eta_seconds": 13990.172056388856, "data_wait_seconds": 1.704692840576172e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 33190, "total_steps": 72479, "tokens": 17401118720, "loss": 4.571967422962189, "l_mix": 2.8333118557929993, "l_a": 2.920572906732559, "l_c": 2.8749456107616425, "z_loss": 28.767634868621826, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.145030215382576, "tokens_per_second": 1371702.7120328057, "eta_seconds": 15016.920978069305, "data_wait_seconds": 1.5497207641601562e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 33200, "total_steps": 72479, "tokens": 17406361600, "loss": 4.490532994270325, "l_mix": 2.7816836833953857, "l_a": 2.87294739484787, "l_c": 2.823216885328293, "z_loss": 29.514080047607422, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15188361704349518, "tokens_per_second": 1433292.1071556695, "eta_seconds": 14367.977224731445, "data_wait_seconds": 2.5480985641479492e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.26676512, "gate_a2b/L1_absmax": 0.69485909, "gate_a2b/L1_rms": 0.28119826, "gate_a2b/L3_absmean": 0.27638498, "gate_a2b/L3_absmax": 0.62647289, "gate_a2b/L3_rms": 0.28224754, "gate_a2b/L5_absmean": 0.43464985, "gate_a2b/L5_absmax": 0.62730736, "gate_a2b/L5_rms": 0.44041416, "gate_c2b/L1_absmean": 0.01704599, "gate_c2b/L1_absmax": 0.56338012, "gate_c2b/L1_rms": 0.06616465, "gate_c2b/L3_absmean": 0.21764515, "gate_c2b/L3_absmax": 0.48786077, "gate_c2b/L3_rms": 0.22198112, "gate_c2b/L5_absmean": 0.59140378, "gate_c2b/L5_absmax": 0.82668561, "gate_c2b/L5_rms": 0.60497844, "gate_b2a/L7_absmean": 0.06393842, "gate_b2a/L7_absmax": 0.32615653, "gate_b2a/L7_rms": 0.06914046, "gate_b2a/L11_absmean": 0.1004535, "gate_b2a/L11_absmax": 0.57587856, "gate_b2a/L11_rms": 0.10720503, "gate_b2a/L14_absmean": 0.25784925, "gate_b2a/L14_absmax": 0.72346503, "gate_b2a/L14_rms": 0.26371562, "gate_b2c/L3_absmean": 0.05639476, "gate_b2c/L3_absmax": 0.65589547, "gate_b2c/L3_rms": 0.06948744, "gate_b2c/L5_absmean": 0.06393321, "gate_b2c/L5_absmax": 0.75100076, "gate_b2c/L5_rms": 0.07593432, "gate_b2c/L6_absmean": 0.09391215, "gate_b2c/L6_absmax": 0.51583326, "gate_b2c/L6_rms": 0.10262168}, "streams": {"cos_a2b/round0": 0.395576, "cos_c2b/round0": 0.297879, "cos_b2a/round0": 0.129033, "cos_b2c/round0": 0.130088, "rms_a/round0": 2.574669, "rms_b/round0": 4.925074, "rms_c/round0": 2.421813, "cos_a2b/round1": 0.425044, "cos_c2b/round1": 0.28888, "cos_b2a/round1": 0.235521, "cos_b2c/round1": 0.116817, "rms_a/round1": 4.728611, "rms_b/round1": 10.585312, "rms_c/round1": 4.649051, "cos_a2b/round2": 0.550528, "cos_c2b/round2": 0.515498, "cos_b2a/round2": 0.095792, "cos_b2c/round2": 0.16381, "rms_a/round2": 11.618753, "rms_b/round2": 25.94191, "rms_c/round2": 8.280028}, "alpha_mean": 0.4545462462901355, "alpha_hist": [0.0103, 0.06143, 0.1496, 0.18489, 0.18762, 0.17274, 0.13533, 0.08183, 0.01578, 0.00047], "alpha_by_type": {"special": {"share": 0.0013, "mean": 0.58578, "std": 0.13427}, "whitespace": {"share": 0.03183, "mean": 0.55262, "std": 0.1271}, "digit": {"share": 0.04613, "mean": 0.49657, "std": 0.11902}, "word_start": {"share": 0.55367, "mean": 0.46906, "std": 0.17082}, "word_cont": {"share": 0.26227, "mean": 0.3437, "std": 0.14013}, "punct": {"share": 0.10481, "mean": 0.60532, "std": 0.15769}}, "alpha_p05": 0.17024, "alpha_p50": 0.44867, "alpha_p95": 0.74205}
{"step": 33210, "total_steps": 72479, "tokens": 17411604480, "loss": 4.463921785354614, "l_mix": 2.765176683664322, "l_a": 2.8546603322029114, "l_c": 2.8078229427337646, "z_loss": 28.594116687774658, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.129947230219841, "tokens_per_second": 1454241.6722742664, "eta_seconds": 14157.389287161828, "data_wait_seconds": 2.1964311599731445e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 33220, "total_steps": 72479, "tokens": 17416847360, "loss": 4.7932217717170715, "l_mix": 2.9719933569431305, "l_a": 3.0597021281719208, "l_c": 3.0110589265823364, "z_loss": 27.78861379623413, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11500652134418488, "tokens_per_second": 1412508.186905937, "eta_seconds": 14571.966932868958, "data_wait_seconds": 1.6421079635620117e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20768, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 33230, "total_steps": 72479, "tokens": 17422090240, "loss": 4.734040021896362, "l_mix": 2.935551643371582, "l_a": 3.0201898515224457, "l_c": 2.974771350622177, "z_loss": 28.027427434921265, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13384461402893066, "tokens_per_second": 1419517.962587867, "eta_seconds": 14496.315125513076, "data_wait_seconds": 2.3484230041503906e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 33240, "total_steps": 72479, "tokens": 17427333120, "loss": 4.761119246482849, "l_mix": 2.952386111021042, "l_a": 3.034923255443573, "l_c": 2.994186908006668, "z_loss": 28.191785097122192, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1453096866607666, "tokens_per_second": 1417914.365810816, "eta_seconds": 14509.012199926377, "data_wait_seconds": 1.633167266845703e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 33250, "total_steps": 72479, "tokens": 17432576000, "loss": 4.7293238043785095, "l_mix": 2.931921511888504, "l_a": 3.0164527595043182, "l_c": 2.9748879969120026, "z_loss": 28.397257566452026, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1414329707622528, "tokens_per_second": 1364654.8546055993, "eta_seconds": 15071.425483584404, "data_wait_seconds": 2.3245811462402344e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 33260, "total_steps": 72479, "tokens": 17437818880, "loss": 4.612593412399292, "l_mix": 2.859512060880661, "l_a": 2.9458835422992706, "l_c": 2.897720605134964, "z_loss": 27.992156505584717, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14520452916622162, "tokens_per_second": 1462144.1359344816, "eta_seconds": 14062.943978405, "data_wait_seconds": 2.041459083557129e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 33270, "total_steps": 72479, "tokens": 17443061760, "loss": 2.854685455560684, "l_mix": 1.7671259045600891, "l_a": 1.832506313920021, "l_c": 1.7926921397447586, "z_loss": 32.41847777366638, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.22497186064720154, "tokens_per_second": 1318532.7778757457, "eta_seconds": 15590.669065594673, "data_wait_seconds": 1.7523765563964844e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08236}}
{"step": 33280, "total_steps": 72479, "tokens": 17448304640, "loss": 4.442872762680054, "l_mix": 2.751704454421997, "l_a": 2.840749144554138, "l_c": 2.7964786291122437, "z_loss": 29.197657108306885, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14409109950065613, "tokens_per_second": 1374918.0188334892, "eta_seconds": 14947.484163045883, "data_wait_seconds": 2.9921531677246094e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 33290, "total_steps": 72479, "tokens": 17453547520, "loss": 4.462916254997253, "l_mix": 2.765193670988083, "l_a": 2.853745549917221, "l_c": 2.8053296506404877, "z_loss": 28.958799123764038, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12059474736452103, "tokens_per_second": 1375212.083447412, "eta_seconds": 14940.475494146349, "data_wait_seconds": 2.9325485229492188e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 33300, "total_steps": 72479, "tokens": 17458790400, "loss": 4.564272403717041, "l_mix": 2.826507955789566, "l_a": 2.91942098736763, "l_c": 2.8731267750263214, "z_loss": 28.72726845741272, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.3484291434288025, "tokens_per_second": 1371423.1464542195, "eta_seconds": 14977.929755020143, "data_wait_seconds": 2.3812055587768555e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.26562452, "gate_a2b/L1_absmax": 0.69743806, "gate_a2b/L1_rms": 0.28005546, "gate_a2b/L3_absmean": 0.27542573, "gate_a2b/L3_absmax": 0.62890291, "gate_a2b/L3_rms": 0.28128335, "gate_a2b/L5_absmean": 0.43281958, "gate_a2b/L5_absmax": 0.63249987, "gate_a2b/L5_rms": 0.43864542, "gate_c2b/L1_absmean": 0.01721877, "gate_c2b/L1_absmax": 0.5651868, "gate_c2b/L1_rms": 0.06626446, "gate_c2b/L3_absmean": 0.21738367, "gate_c2b/L3_absmax": 0.48459604, "gate_c2b/L3_rms": 0.22178093, "gate_c2b/L5_absmean": 0.59181678, "gate_c2b/L5_absmax": 0.82656455, "gate_c2b/L5_rms": 0.6053139, "gate_b2a/L7_absmean": 0.06366526, "gate_b2a/L7_absmax": 0.32714188, "gate_b2a/L7_rms": 0.0688611, "gate_b2a/L11_absmean": 0.10008463, "gate_b2a/L11_absmax": 0.57758075, "gate_b2a/L11_rms": 0.10691751, "gate_b2a/L14_absmean": 0.25683147, "gate_b2a/L14_absmax": 0.72434819, "gate_b2a/L14_rms": 0.26275745, "gate_b2c/L3_absmean": 0.05586178, "gate_b2c/L3_absmax": 0.65627038, "gate_b2c/L3_rms": 0.06895874, "gate_b2c/L5_absmean": 0.06346943, "gate_b2c/L5_absmax": 0.7486276, "gate_b2c/L5_rms": 0.07546463, "gate_b2c/L6_absmean": 0.09297245, "gate_b2c/L6_absmax": 0.51481783, "gate_b2c/L6_rms": 0.10171995}, "streams": {"cos_a2b/round0": 0.395733, "cos_c2b/round0": 0.302175, "cos_b2a/round0": 0.130109, "cos_b2c/round0": 0.136814, "rms_a/round0": 2.576246, "rms_b/round0": 4.931895, "rms_c/round0": 2.430466, "cos_a2b/round1": 0.43492, "cos_c2b/round1": 0.296039, "cos_b2a/round1": 0.242564, "cos_b2c/round1": 0.116287, "rms_a/round1": 4.707931, "rms_b/round1": 10.567641, "rms_c/round1": 4.604187, "cos_a2b/round2": 0.577423, "cos_c2b/round2": 0.500702, "cos_b2a/round2": 0.107836, "cos_b2c/round2": 0.17024, "rms_a/round2": 11.886895, "rms_b/round2": 26.072615, "rms_c/round2": 8.012959}, "alpha_mean": 0.4710735932276151, "alpha_hist": [0.00714, 0.05569, 0.14177, 0.17427, 0.17345, 0.17293, 0.15503, 0.0992, 0.02028, 0.00024], "alpha_by_type": {"special": {"share": 0.00066, "mean": 0.53542, "std": 0.12778}, "whitespace": {"share": 0.01872, "mean": 0.54362, "std": 0.11886}, "digit": {"share": 0.03311, "mean": 0.41971, "std": 0.14502}, "word_start": {"share": 0.60709, "mean": 0.50063, "std": 0.16822}, "word_cont": {"share": 0.24077, "mean": 0.33346, "std": 0.13008}, "punct": {"share": 0.09966, "mean": 0.62651, "std": 0.15951}}, "alpha_p05": 0.1833, "alpha_p50": 0.47853, "alpha_p95": 0.75992}
{"step": 33310, "total_steps": 72479, "tokens": 17464033280, "loss": 4.608851671218872, "l_mix": 2.8560890853405, "l_a": 2.94458270072937, "l_c": 2.897959291934967, "z_loss": 29.011712312698364, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13768108189105988, "tokens_per_second": 1378466.4353567197, "eta_seconds": 14897.59644868374, "data_wait_seconds": 2.205371856689453e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 33320, "total_steps": 72479, "tokens": 17469276160, "loss": 4.608337044715881, "l_mix": 2.855268269777298, "l_a": 2.948876738548279, "l_c": 2.8946859538555145, "z_loss": 29.339025735855103, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18100687861442566, "tokens_per_second": 1412361.219188179, "eta_seconds": 14536.36188325882, "data_wait_seconds": 1.7821788787841797e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 33330, "total_steps": 72479, "tokens": 17474519040, "loss": 3.108124941587448, "l_mix": 1.9235988110303879, "l_a": 1.9947829097509384, "l_c": 1.9536373913288116, "z_loss": 34.399784564971924, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1768651008605957, "tokens_per_second": 1456772.7660914054, "eta_seconds": 14089.603670358658, "data_wait_seconds": 2.0265579223632812e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08236}}
{"step": 33340, "total_steps": 72479, "tokens": 17479761920, "loss": 4.697209060192108, "l_mix": 2.9125792384147644, "l_a": 2.9958223700523376, "l_c": 2.952943354845047, "z_loss": 27.86050581932068, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12448545545339584, "tokens_per_second": 1344911.8410449666, "eta_seconds": 15257.585966420173, "data_wait_seconds": 2.0205974578857422e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20768, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 33350, "total_steps": 72479, "tokens": 17485004800, "loss": 4.7905153632164, "l_mix": 2.9698741734027863, "l_a": 3.0576095581054688, "l_c": 3.0111943781375885, "z_loss": 28.50060749053955, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16145607829093933, "tokens_per_second": 1430245.7255532518, "eta_seconds": 14343.594800162315, "data_wait_seconds": 2.3633241653442383e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20768, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 33360, "total_steps": 72479, "tokens": 17490247680, "loss": 4.71375834941864, "l_mix": 2.922569215297699, "l_a": 3.0077323615550995, "l_c": 2.9628979563713074, "z_loss": 27.9499454498291, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12651367485523224, "tokens_per_second": 1465809.4108981611, "eta_seconds": 13992.011594080925, "data_wait_seconds": 2.434849739074707e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 33370, "total_steps": 72479, "tokens": 17495490560, "loss": 4.723825097084045, "l_mix": 2.928659290075302, "l_a": 3.0150641798973083, "l_c": 2.9688218235969543, "z_loss": 28.103550910949707, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14058326184749603, "tokens_per_second": 1251880.7932616165, "eta_seconds": 16378.859315013886, "data_wait_seconds": 2.899765968322754e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 33380, "total_steps": 72479, "tokens": 17500733440, "loss": 2.985512614250183, "l_mix": 1.847643420100212, "l_a": 1.9138416349887848, "l_c": 1.8790554851293564, "z_loss": 35.14227914810181, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20286695659160614, "tokens_per_second": 1314855.7429271943, "eta_seconds": 15590.407253623009, "data_wait_seconds": 3.075599670410156e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 33390, "total_steps": 72479, "tokens": 17505976320, "loss": 4.5216978788375854, "l_mix": 2.8012295365333557, "l_a": 2.8908608853816986, "l_c": 2.844033271074295, "z_loss": 28.568252325057983, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12312009930610657, "tokens_per_second": 1405528.0421942854, "eta_seconds": 14580.921203112603, "data_wait_seconds": 1.7583370208740234e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 33400, "total_steps": 72479, "tokens": 17511219200, "loss": 4.4591920375823975, "l_mix": 2.7628640234470367, "l_a": 2.851767063140869, "l_c": 2.8026594519615173, "z_loss": 28.72228693962097, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11718415468931198, "tokens_per_second": 1417285.8158659055, "eta_seconds": 14456.258944129942, "data_wait_seconds": 2.1070241928100586e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.26482916, "gate_a2b/L1_absmax": 0.69680083, "gate_a2b/L1_rms": 0.279376, "gate_a2b/L3_absmean": 0.27445999, "gate_a2b/L3_absmax": 0.62590522, "gate_a2b/L3_rms": 0.28034896, "gate_a2b/L5_absmean": 0.43069053, "gate_a2b/L5_absmax": 0.6320551, "gate_a2b/L5_rms": 0.43656212, "gate_c2b/L1_absmean": 0.01697149, "gate_c2b/L1_absmax": 0.56520981, "gate_c2b/L1_rms": 0.06602995, "gate_c2b/L3_absmean": 0.21729162, "gate_c2b/L3_absmax": 0.48995948, "gate_c2b/L3_rms": 0.22169617, "gate_c2b/L5_absmean": 0.59227711, "gate_c2b/L5_absmax": 0.82882303, "gate_c2b/L5_rms": 0.60578847, "gate_b2a/L7_absmean": 0.06331505, "gate_b2a/L7_absmax": 0.32415041, "gate_b2a/L7_rms": 0.0685107, "gate_b2a/L11_absmean": 0.0998102, "gate_b2a/L11_absmax": 0.57761794, "gate_b2a/L11_rms": 0.10667463, "gate_b2a/L14_absmean": 0.25554031, "gate_b2a/L14_absmax": 0.72657889, "gate_b2a/L14_rms": 0.26154256, "gate_b2c/L3_absmean": 0.05554764, "gate_b2c/L3_absmax": 0.65352118, "gate_b2c/L3_rms": 0.06856189, "gate_b2c/L5_absmean": 0.06273779, "gate_b2c/L5_absmax": 0.74455893, "gate_b2c/L5_rms": 0.0747394, "gate_b2c/L6_absmean": 0.09190695, "gate_b2c/L6_absmax": 0.51250941, "gate_b2c/L6_rms": 0.10063928}, "streams": {"cos_a2b/round0": 0.394269, "cos_c2b/round0": 0.306843, "cos_b2a/round0": 0.13727, "cos_b2c/round0": 0.132011, "rms_a/round0": 2.608308, "rms_b/round0": 4.977197, "rms_c/round0": 2.446783, "cos_a2b/round1": 0.430507, "cos_c2b/round1": 0.295841, "cos_b2a/round1": 0.241504, "cos_b2c/round1": 0.120118, "rms_a/round1": 4.828259, "rms_b/round1": 10.792359, "rms_c/round1": 4.698657, "cos_a2b/round2": 0.558459, "cos_c2b/round2": 0.519618, "cos_b2a/round2": 0.098696, "cos_b2c/round2": 0.167443, "rms_a/round2": 11.991653, "rms_b/round2": 26.497133, "rms_c/round2": 8.397745}, "alpha_mean": 0.44017438493085237, "alpha_hist": [0.0119, 0.06905, 0.16032, 0.19191, 0.18582, 0.17233, 0.13628, 0.06561, 0.00673, 5e-05], "alpha_by_type": {"special": {"share": 0.00122, "mean": 0.55843, "std": 0.09565}, "whitespace": {"share": 0.0257, "mean": 0.51983, "std": 0.11067}, "digit": {"share": 0.03513, "mean": 0.4848, "std": 0.12953}, "word_start": {"share": 0.59154, "mean": 0.46089, "std": 0.16136}, "word_cont": {"share": 0.23857, "mean": 0.31277, "std": 0.13645}, "punct": {"share": 0.10785, "mean": 0.57353, "std": 0.15806}}, "alpha_p05": 0.1781, "alpha_p50": 0.43542, "alpha_p95": 0.72155}
{"step": 33410, "total_steps": 72479, "tokens": 17516462080, "loss": 4.53354287147522, "l_mix": 2.808203786611557, "l_a": 2.9032898545265198, "l_c": 2.847840279340744, "z_loss": 28.83591055870056, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13798397779464722, "tokens_per_second": 1471462.9576681512, "eta_seconds": 13920.437320733072, "data_wait_seconds": 2.16066837310791e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 33420, "total_steps": 72479, "tokens": 17521704960, "loss": 4.569866716861725, "l_mix": 2.83180633187294, "l_a": 2.92036834359169, "l_c": 2.8731661438941956, "z_loss": 28.655728816986084, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1380905658006668, "tokens_per_second": 1468792.1083028691, "eta_seconds": 13942.180704975128, "data_wait_seconds": 1.6570091247558594e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 33430, "total_steps": 72479, "tokens": 17526947840, "loss": 4.509106159210205, "l_mix": 2.7935237884521484, "l_a": 2.886278510093689, "l_c": 2.8323293030261993, "z_loss": 28.193530797958374, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14304201304912567, "tokens_per_second": 1394038.5958560025, "eta_seconds": 14686.051141524316, "data_wait_seconds": 2.0682811737060547e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 33440, "total_steps": 72479, "tokens": 17532190720, "loss": 2.889336347579956, "l_mix": 1.7875260263681412, "l_a": 1.855859637260437, "l_c": 1.816841185092926, "z_loss": 34.38713073730469, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18956023454666138, "tokens_per_second": 1439905.5994723937, "eta_seconds": 14214.597984409333, "data_wait_seconds": 1.7374753952026367e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08236}}
{"step": 33450, "total_steps": 72479, "tokens": 17537433600, "loss": 3.2596899569034576, "l_mix": 2.005159094929695, "l_a": 2.125336676836014, "l_c": 2.0564325749874115, "z_loss": 30.262580394744873, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15795563161373138, "tokens_per_second": 1316465.8596850215, "eta_seconds": 15543.46145892143, "data_wait_seconds": 2.4706125259399414e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20045, "finemath": 0.08233}}
{"step": 33460, "total_steps": 72479, "tokens": 17542676480, "loss": 3.2595452070236206, "l_mix": 2.0052028596401215, "l_a": 2.122834265232086, "l_c": 2.058306723833084, "z_loss": 29.555489540100098, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1447824388742447, "tokens_per_second": 1367256.6422708132, "eta_seconds": 14962.219117856026, "data_wait_seconds": 2.396106719970703e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20045, "finemath": 0.08234}}
{"step": 33470, "total_steps": 72479, "tokens": 17547919360, "loss": 3.293381631374359, "l_mix": 2.02587328851223, "l_a": 2.145755261182785, "l_c": 2.079272449016571, "z_loss": 28.75096297264099, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16506431996822357, "tokens_per_second": 1357073.279671393, "eta_seconds": 15070.630966186523, "data_wait_seconds": 2.0176172256469727e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20045, "finemath": 0.08234}}
{"step": 33480, "total_steps": 72479, "tokens": 17553162240, "loss": 4.733820199966431, "l_mix": 2.935431033372879, "l_a": 3.0164873898029327, "l_c": 2.978143185377121, "z_loss": 28.583285570144653, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15559466183185577, "tokens_per_second": 1416357.627582909, "eta_seconds": 14436.119320297243, "data_wait_seconds": 2.3633241653442383e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 33490, "total_steps": 72479, "tokens": 17558405120, "loss": 4.678620100021362, "l_mix": 2.900862395763397, "l_a": 2.9854949712753296, "l_c": 2.9403636157512665, "z_loss": 27.645668506622314, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16766732931137085, "tokens_per_second": 1309166.6088008047, "eta_seconds": 15614.104953932763, "data_wait_seconds": 1.9431114196777344e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 33500, "total_steps": 72479, "tokens": 17563648000, "loss": 4.498507499694824, "l_mix": 2.7870754301548004, "l_a": 2.8785191476345062, "l_c": 2.8262541592121124, "z_loss": 28.76178216934204, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1308981478214264, "tokens_per_second": 1297258.057030023, "eta_seconds": 15753.397592139245, "data_wait_seconds": 1.7255544662475586e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.26307398, "gate_a2b/L1_absmax": 0.70359302, "gate_a2b/L1_rms": 0.2775642, "gate_a2b/L3_absmean": 0.27280647, "gate_a2b/L3_absmax": 0.63354492, "gate_a2b/L3_rms": 0.2787936, "gate_a2b/L5_absmean": 0.42821667, "gate_a2b/L5_absmax": 0.63752341, "gate_a2b/L5_rms": 0.43402877, "gate_c2b/L1_absmean": 0.01755617, "gate_c2b/L1_absmax": 0.57266766, "gate_c2b/L1_rms": 0.0662236, "gate_c2b/L3_absmean": 0.21828003, "gate_c2b/L3_absmax": 0.49510151, "gate_c2b/L3_rms": 0.22270063, "gate_c2b/L5_absmean": 0.59444112, "gate_c2b/L5_absmax": 0.82547367, "gate_c2b/L5_rms": 0.60806298, "gate_b2a/L7_absmean": 0.06319364, "gate_b2a/L7_absmax": 0.32646555, "gate_b2a/L7_rms": 0.06838255, "gate_b2a/L11_absmean": 0.09969318, "gate_b2a/L11_absmax": 0.57841057, "gate_b2a/L11_rms": 0.10658183, "gate_b2a/L14_absmean": 0.25550804, "gate_b2a/L14_absmax": 0.73017704, "gate_b2a/L14_rms": 0.26155239, "gate_b2c/L3_absmean": 0.05452628, "gate_b2c/L3_absmax": 0.64692688, "gate_b2c/L3_rms": 0.06749078, "gate_b2c/L5_absmean": 0.06181609, "gate_b2c/L5_absmax": 0.73924047, "gate_b2c/L5_rms": 0.07376649, "gate_b2c/L6_absmean": 0.09080899, "gate_b2c/L6_absmax": 0.51345509, "gate_b2c/L6_rms": 0.09954634}, "streams": {"cos_a2b/round0": 0.399852, "cos_c2b/round0": 0.302202, "cos_b2a/round0": 0.134513, "cos_b2c/round0": 0.131103, "rms_a/round0": 2.689395, "rms_b/round0": 5.138768, "rms_c/round0": 2.503435, "cos_a2b/round1": 0.428458, "cos_c2b/round1": 0.293454, "cos_b2a/round1": 0.246402, "cos_b2c/round1": 0.119413, "rms_a/round1": 4.91495, "rms_b/round1": 11.093291, "rms_c/round1": 4.799021, "cos_a2b/round2": 0.555647, "cos_c2b/round2": 0.52416, "cos_b2a/round2": 0.096578, "cos_b2c/round2": 0.167321, "rms_a/round2": 12.233235, "rms_b/round2": 27.041733, "rms_c/round2": 8.54531}, "alpha_mean": 0.41508474221782876, "alpha_hist": [0.01134, 0.08981, 0.19824, 0.19998, 0.18159, 0.14572, 0.11021, 0.05675, 0.0063, 5e-05], "alpha_by_type": {"special": {"share": 0.00124, "mean": 0.52622, "std": 0.11879}, "whitespace": {"share": 0.0278, "mean": 0.51732, "std": 0.12455}, "digit": {"share": 0.04382, "mean": 0.44458, "std": 0.1144}, "word_start": {"share": 0.57146, "mean": 0.43165, "std": 0.16825}, "word_cont": {"share": 0.25491, "mean": 0.30529, "std": 0.12662}, "punct": {"share": 0.10077, "mean": 0.55651, "std": 0.16562}}, "alpha_p05": 0.16426, "alpha_p50": 0.39934, "alpha_p95": 0.7068}
{"step": 33510, "total_steps": 72479, "tokens": 17568890880, "loss": 4.591497600078583, "l_mix": 2.8449549973011017, "l_a": 2.936604470014572, "l_c": 2.8852036893367767, "z_loss": 28.727715492248535, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11400279402732849, "tokens_per_second": 1433793.2934992083, "eta_seconds": 14249.59871456623, "data_wait_seconds": 2.086162567138672e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 33520, "total_steps": 72479, "tokens": 17574133760, "loss": 4.476517021656036, "l_mix": 2.772730976343155, "l_a": 2.864859163761139, "l_c": 2.8144274055957794, "z_loss": 28.568339824676514, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11419598758220673, "tokens_per_second": 1472302.9242626328, "eta_seconds": 13873.324473786353, "data_wait_seconds": 1.5974044799804688e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 33530, "total_steps": 72479, "tokens": 17579376640, "loss": 4.463645398616791, "l_mix": 2.7648760080337524, "l_a": 2.8585220277309418, "l_c": 2.804042309522629, "z_loss": 28.812856912612915, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12141548842191696, "tokens_per_second": 1448364.0019369277, "eta_seconds": 14099.006385612487, "data_wait_seconds": 1.582503318786621e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 33540, "total_steps": 72479, "tokens": 17584619520, "loss": 4.61855685710907, "l_mix": 2.86116623878479, "l_a": 2.9554682672023773, "l_c": 2.9024999141693115, "z_loss": 28.906283855438232, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17106927931308746, "tokens_per_second": 1341305.1533033361, "eta_seconds": 15220.436886954309, "data_wait_seconds": 2.9146671295166016e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 33550, "total_steps": 72479, "tokens": 17589862400, "loss": 3.054877609014511, "l_mix": 1.8897836804389954, "l_a": 1.9613356739282608, "l_c": 1.9223105013370514, "z_loss": 33.794862270355225, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19427214562892914, "tokens_per_second": 1454885.4361302124, "eta_seconds": 14028.601184082032, "data_wait_seconds": 1.7762184143066406e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 33560, "total_steps": 72479, "tokens": 17595105280, "loss": 3.271765947341919, "l_mix": 2.013334408402443, "l_a": 2.1307268142700195, "l_c": 2.064044862985611, "z_loss": 30.806169033050537, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2129286527633667, "tokens_per_second": 1286614.671495175, "eta_seconds": 15859.26627767086, "data_wait_seconds": 2.3365020751953125e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20045, "finemath": 0.08233}}
{"step": 33570, "total_steps": 72479, "tokens": 17600348160, "loss": 3.236895889043808, "l_mix": 1.992127776145935, "l_a": 2.102174252271652, "l_c": 2.0470528304576874, "z_loss": 30.478951692581177, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2244245857000351, "tokens_per_second": 1397475.6579041188, "eta_seconds": 14597.40760178566, "data_wait_seconds": 2.7358531951904297e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20045, "finemath": 0.08234}}
{"step": 33580, "total_steps": 72479, "tokens": 17605591040, "loss": 3.283395826816559, "l_mix": 2.0198107957839966, "l_a": 2.138149321079254, "l_c": 2.07380048930645, "z_loss": 29.060428142547607, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19025757908821106, "tokens_per_second": 1321373.3121058568, "eta_seconds": 15434.153789210319, "data_wait_seconds": 2.1278858184814453e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20045, "finemath": 0.08234}}
{"step": 33590, "total_steps": 72479, "tokens": 17610833920, "loss": 4.553694069385529, "l_mix": 2.8211827278137207, "l_a": 2.913667768239975, "l_c": 2.8613697290420532, "z_loss": 28.692772388458252, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13903284072875977, "tokens_per_second": 1436691.6689437455, "eta_seconds": 14191.657453536987, "data_wait_seconds": 3.4362077713012695e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 33600, "total_steps": 72479, "tokens": 17616076800, "loss": 4.533049762248993, "l_mix": 2.808509886264801, "l_a": 2.9009132087230682, "l_c": 2.8475527465343475, "z_loss": 28.74553394317627, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12173140794038773, "tokens_per_second": 1335288.0445450782, "eta_seconds": 15265.46518204212, "data_wait_seconds": 1.773238182067871e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.26224512, "gate_a2b/L1_absmax": 0.70294875, "gate_a2b/L1_rms": 0.27667955, "gate_a2b/L3_absmean": 0.27134395, "gate_a2b/L3_absmax": 0.62563223, "gate_a2b/L3_rms": 0.27724871, "gate_a2b/L5_absmean": 0.42617732, "gate_a2b/L5_absmax": 0.63698936, "gate_a2b/L5_rms": 0.43201831, "gate_c2b/L1_absmean": 0.01700749, "gate_c2b/L1_absmax": 0.57247394, "gate_c2b/L1_rms": 0.06623467, "gate_c2b/L3_absmean": 0.21823345, "gate_c2b/L3_absmax": 0.49207845, "gate_c2b/L3_rms": 0.2226309, "gate_c2b/L5_absmean": 0.59428018, "gate_c2b/L5_absmax": 0.83353055, "gate_c2b/L5_rms": 0.60784638, "gate_b2a/L7_absmean": 0.06298923, "gate_b2a/L7_absmax": 0.32660317, "gate_b2a/L7_rms": 0.06821129, "gate_b2a/L11_absmean": 0.09892178, "gate_b2a/L11_absmax": 0.57985353, "gate_b2a/L11_rms": 0.10590082, "gate_b2a/L14_absmean": 0.25400519, "gate_b2a/L14_absmax": 0.73034948, "gate_b2a/L14_rms": 0.26009136, "gate_b2c/L3_absmean": 0.05408195, "gate_b2c/L3_absmax": 0.64728379, "gate_b2c/L3_rms": 0.06700203, "gate_b2c/L5_absmean": 0.06142729, "gate_b2c/L5_absmax": 0.73585606, "gate_b2c/L5_rms": 0.07330439, "gate_b2c/L6_absmean": 0.09002139, "gate_b2c/L6_absmax": 0.50840873, "gate_b2c/L6_rms": 0.09870953}, "streams": {"cos_a2b/round0": 0.394847, "cos_c2b/round0": 0.300922, "cos_b2a/round0": 0.127711, "cos_b2c/round0": 0.130141, "rms_a/round0": 2.72277, "rms_b/round0": 5.163717, "rms_c/round0": 2.504421, "cos_a2b/round1": 0.420578, "cos_c2b/round1": 0.286048, "cos_b2a/round1": 0.238906, "cos_b2c/round1": 0.113889, "rms_a/round1": 4.937627, "rms_b/round1": 11.235329, "rms_c/round1": 4.802306, "cos_a2b/round2": 0.551782, "cos_c2b/round2": 0.515605, "cos_b2a/round2": 0.091866, "cos_b2c/round2": 0.164137, "rms_a/round2": 12.363899, "rms_b/round2": 27.638479, "rms_c/round2": 8.446436}, "alpha_mean": 0.43009580360045163, "alpha_hist": [0.01189, 0.07365, 0.18446, 0.20547, 0.17497, 0.1467, 0.1239, 0.06805, 0.01047, 0.00043], "alpha_by_type": {"special": {"share": 0.00101, "mean": 0.51403, "std": 0.09485}, "whitespace": {"share": 0.02509, "mean": 0.5399, "std": 0.14227}, "digit": {"share": 0.05855, "mean": 0.46581, "std": 0.13823}, "word_start": {"share": 0.52724, "mean": 0.45009, "std": 0.16555}, "word_cont": {"share": 0.27461, "mean": 0.30832, "std": 0.12796}, "punct": {"share": 0.11351, "mean": 0.58841, "std": 0.15947}}, "alpha_p05": 0.1833, "alpha_p50": 0.40969, "alpha_p95": 0.74354}
{"step": 33610, "total_steps": 72479, "tokens": 17621319680, "loss": 4.574513137340546, "l_mix": 2.833802103996277, "l_a": 2.926884263753891, "l_c": 2.8754857778549194, "z_loss": 29.339207410812378, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15839383006095886, "tokens_per_second": 1429188.4612690448, "eta_seconds": 14258.826476883889, "data_wait_seconds": 1.621246337890625e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08236}}
{"step": 33620, "total_steps": 72479, "tokens": 17626562560, "loss": 4.43597412109375, "l_mix": 2.748417615890503, "l_a": 2.8375722467899323, "l_c": 2.7876157462596893, "z_loss": 28.752623558044434, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.09955132007598877, "tokens_per_second": 1474267.272478852, "eta_seconds": 13819.276716184617, "data_wait_seconds": 1.6391277313232422e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 33630, "total_steps": 72479, "tokens": 17631805440, "loss": 4.512466132640839, "l_mix": 2.7961014807224274, "l_a": 2.8868633210659027, "l_c": 2.8343520760536194, "z_loss": 28.962724685668945, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11289732158184052, "tokens_per_second": 1456816.0985234508, "eta_seconds": 13981.21872255802, "data_wait_seconds": 1.874566078186035e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 33640, "total_steps": 72479, "tokens": 17637048320, "loss": 4.524427890777588, "l_mix": 2.8022190034389496, "l_a": 2.896300047636032, "l_c": 2.8443959057331085, "z_loss": 28.976455211639404, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1628485471010208, "tokens_per_second": 1345579.1708945797, "eta_seconds": 15133.127854871751, "data_wait_seconds": 2.086162567138672e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 33650, "total_steps": 72479, "tokens": 17642291200, "loss": 4.512013614177704, "l_mix": 2.7950950264930725, "l_a": 2.8864952623844147, "l_c": 2.83656644821167, "z_loss": 29.26346445083618, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13177336752414703, "tokens_per_second": 1462115.1652864153, "eta_seconds": 13923.375692510604, "data_wait_seconds": 1.8864870071411133e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 33660, "total_steps": 72479, "tokens": 17647534080, "loss": 3.0560709834098816, "l_mix": 1.889960989356041, "l_a": 1.9639144539833069, "l_c": 1.923118770122528, "z_loss": 34.28367280960083, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16433671116828918, "tokens_per_second": 1463870.0557476296, "eta_seconds": 13903.102800750732, "data_wait_seconds": 3.4868717193603516e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 33670, "total_steps": 72479, "tokens": 17652776960, "loss": 3.2207935750484467, "l_mix": 1.9819912165403366, "l_a": 2.095794767141342, "l_c": 2.0335463881492615, "z_loss": 29.391366720199585, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.212738037109375, "tokens_per_second": 1455492.2944127293, "eta_seconds": 13979.526425600052, "data_wait_seconds": 3.311038017272949e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20045, "finemath": 0.08233}}
{"step": 33680, "total_steps": 72479, "tokens": 17658019840, "loss": 3.2659032940864563, "l_mix": 2.0088405311107635, "l_a": 2.1262265145778656, "l_c": 2.063982367515564, "z_loss": 30.325125694274902, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18092761933803558, "tokens_per_second": 1195617.8408392312, "eta_seconds": 17013.672276520727, "data_wait_seconds": 2.905726432800293e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20045, "finemath": 0.08233}}
{"step": 33690, "total_steps": 72479, "tokens": 17663262720, "loss": 3.240444242954254, "l_mix": 1.9929437935352325, "l_a": 2.1120389103889465, "l_c": 2.0462957322597504, "z_loss": 30.805505514144897, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16778965294361115, "tokens_per_second": 1418971.5820565682, "eta_seconds": 14331.934119868278, "data_wait_seconds": 2.7745962142944336e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20045, "finemath": 0.08234}}
{"step": 33700, "total_steps": 72479, "tokens": 17668505600, "loss": 4.724994480609894, "l_mix": 2.9290818870067596, "l_a": 3.01800736784935, "l_c": 2.9683675467967987, "z_loss": 28.57170796394348, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12266983091831207, "tokens_per_second": 1386945.6402856081, "eta_seconds": 14659.092441296578, "data_wait_seconds": 2.5153160095214844e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.26096869, "gate_a2b/L1_absmax": 0.70143306, "gate_a2b/L1_rms": 0.27542371, "gate_a2b/L3_absmean": 0.27016166, "gate_a2b/L3_absmax": 0.62598115, "gate_a2b/L3_rms": 0.27614257, "gate_a2b/L5_absmean": 0.42497855, "gate_a2b/L5_absmax": 0.63884872, "gate_a2b/L5_rms": 0.43078575, "gate_c2b/L1_absmean": 0.01665009, "gate_c2b/L1_absmax": 0.57300955, "gate_c2b/L1_rms": 0.06632978, "gate_c2b/L3_absmean": 0.21796738, "gate_c2b/L3_absmax": 0.48945302, "gate_c2b/L3_rms": 0.22237653, "gate_c2b/L5_absmean": 0.59418553, "gate_c2b/L5_absmax": 0.83352602, "gate_c2b/L5_rms": 0.60761923, "gate_b2a/L7_absmean": 0.0627341, "gate_b2a/L7_absmax": 0.32463959, "gate_b2a/L7_rms": 0.06796516, "gate_b2a/L11_absmean": 0.09849244, "gate_b2a/L11_absmax": 0.57784331, "gate_b2a/L11_rms": 0.10548245, "gate_b2a/L14_absmean": 0.25292039, "gate_b2a/L14_absmax": 0.73091793, "gate_b2a/L14_rms": 0.25898337, "gate_b2c/L3_absmean": 0.05386927, "gate_b2c/L3_absmax": 0.64545977, "gate_b2c/L3_rms": 0.06669626, "gate_b2c/L5_absmean": 0.06081655, "gate_b2c/L5_absmax": 0.73489606, "gate_b2c/L5_rms": 0.07276099, "gate_b2c/L6_absmean": 0.08904265, "gate_b2c/L6_absmax": 0.50616342, "gate_b2c/L6_rms": 0.09772367}, "streams": {"cos_a2b/round0": 0.394593, "cos_c2b/round0": 0.304868, "cos_b2a/round0": 0.134306, "cos_b2c/round0": 0.130654, "rms_a/round0": 2.719532, "rms_b/round0": 5.229846, "rms_c/round0": 2.536307, "cos_a2b/round1": 0.427623, "cos_c2b/round1": 0.295655, "cos_b2a/round1": 0.245791, "cos_b2c/round1": 0.116385, "rms_a/round1": 4.972441, "rms_b/round1": 11.259315, "rms_c/round1": 4.860248, "cos_a2b/round2": 0.552599, "cos_c2b/round2": 0.526994, "cos_b2a/round2": 0.097597, "cos_b2c/round2": 0.166324, "rms_a/round2": 12.46922, "rms_b/round2": 27.965887, "rms_c/round2": 8.660626}, "alpha_mean": 0.41509732116052334, "alpha_hist": [0.00906, 0.07359, 0.19107, 0.2272, 0.1925, 0.1503, 0.10663, 0.04512, 0.00443, 9e-05], "alpha_by_type": {"special": {"share": 0.00099, "mean": 0.52167, "std": 0.0907}, "whitespace": {"share": 0.02634, "mean": 0.49983, "std": 0.11151}, "digit": {"share": 0.04024, "mean": 0.4335, "std": 0.11148}, "word_start": {"share": 0.57542, "mean": 0.4372, "std": 0.15855}, "word_cont": {"share": 0.26289, "mean": 0.3061, "std": 0.11545}, "punct": {"share": 0.09412, "mean": 0.55174, "std": 0.15448}}, "alpha_p05": 0.16668, "alpha_p50": 0.40051, "alpha_p95": 0.69884}
{"step": 33710, "total_steps": 72479, "tokens": 17673748480, "loss": 4.515300512313843, "l_mix": 2.7980997562408447, "l_a": 2.888382703065872, "l_c": 2.835619330406189, "z_loss": 28.57573652267456, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15954160690307617, "tokens_per_second": 1415976.0431183649, "eta_seconds": 14354.848424720763, "data_wait_seconds": 1.7821788787841797e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 33720, "total_steps": 72479, "tokens": 17678991360, "loss": 4.547552525997162, "l_mix": 2.8175641894340515, "l_a": 2.908287137746811, "l_c": 2.858340412378311, "z_loss": 29.819867372512817, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14367185533046722, "tokens_per_second": 1296226.432714122, "eta_seconds": 15676.95124797821, "data_wait_seconds": 2.0444393157958984e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 33730, "total_steps": 72479, "tokens": 17684234240, "loss": 4.443298161029816, "l_mix": 2.7527237832546234, "l_a": 2.843857526779175, "l_c": 2.791390120983124, "z_loss": 29.31415891647339, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14729106426239014, "tokens_per_second": 1458704.670387308, "eta_seconds": 13927.175338792802, "data_wait_seconds": 1.8984079360961914e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 33740, "total_steps": 72479, "tokens": 17689477120, "loss": 3.2593466341495514, "l_mix": 2.0055003315210342, "l_a": 2.123052269220352, "l_c": 2.056435286998749, "z_loss": 29.978237867355347, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16887082159519196, "tokens_per_second": 1459927.0240402757, "eta_seconds": 13911.923334217072, "data_wait_seconds": 1.531839370727539e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 33750, "total_steps": 72479, "tokens": 17694720000, "loss": 4.633619248867035, "l_mix": 2.873552620410919, "l_a": 2.9542577862739563, "l_c": 2.9126304984092712, "z_loss": 28.803191661834717, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13231635093688965, "tokens_per_second": 1455291.5572355022, "eta_seconds": 13952.6336499691, "data_wait_seconds": 2.473592758178711e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08234}}
{"step": 33760, "total_steps": 72479, "tokens": 17699962880, "loss": 4.703181624412537, "l_mix": 2.9158523082733154, "l_a": 2.999617964029312, "l_c": 2.9581462144851685, "z_loss": 29.04510736465454, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12589173018932343, "tokens_per_second": 1450993.2736067972, "eta_seconds": 13990.352292633057, "data_wait_seconds": 1.9490718841552734e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 33770, "total_steps": 72479, "tokens": 17705205760, "loss": 4.647466361522675, "l_mix": 2.8817842304706573, "l_a": 2.9638165831565857, "l_c": 2.921790212392807, "z_loss": 28.674583435058594, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.137316033244133, "tokens_per_second": 1478229.920436578, "eta_seconds": 13729.030857396125, "data_wait_seconds": 1.6450881958007812e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 33780, "total_steps": 72479, "tokens": 17710448640, "loss": 3.2581584453582764, "l_mix": 2.004378154873848, "l_a": 2.121832102537155, "l_c": 2.057435154914856, "z_loss": 30.302306413650513, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16717363893985748, "tokens_per_second": 1235218.2648327136, "eta_seconds": 16425.778252840042, "data_wait_seconds": 2.473592758178711e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 33790, "total_steps": 72479, "tokens": 17715691520, "loss": 3.2647959291934967, "l_mix": 2.0095849335193634, "l_a": 2.1225574016571045, "l_c": 2.0614791810512543, "z_loss": 30.1308171749115, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20611520111560822, "tokens_per_second": 1434657.8054717232, "eta_seconds": 14138.68753554821, "data_wait_seconds": 2.3543834686279297e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 33800, "total_steps": 72479, "tokens": 17720934400, "loss": 3.2536496222019196, "l_mix": 2.0014610588550568, "l_a": 2.1212200224399567, "l_c": 2.052741676568985, "z_loss": 30.042641401290894, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17177768051624298, "tokens_per_second": 1459091.240311312, "eta_seconds": 13898.332737350464, "data_wait_seconds": 1.6450881958007812e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.26023194, "gate_a2b/L1_absmax": 0.70195979, "gate_a2b/L1_rms": 0.27464256, "gate_a2b/L3_absmean": 0.26967272, "gate_a2b/L3_absmax": 0.62656933, "gate_a2b/L3_rms": 0.27554756, "gate_a2b/L5_absmean": 0.42306444, "gate_a2b/L5_absmax": 0.64139444, "gate_a2b/L5_rms": 0.42891714, "gate_c2b/L1_absmean": 0.01703408, "gate_c2b/L1_absmax": 0.57373834, "gate_c2b/L1_rms": 0.06628762, "gate_c2b/L3_absmean": 0.21769391, "gate_c2b/L3_absmax": 0.48500851, "gate_c2b/L3_rms": 0.22200909, "gate_c2b/L5_absmean": 0.59500378, "gate_c2b/L5_absmax": 0.83380342, "gate_c2b/L5_rms": 0.60852671, "gate_b2a/L7_absmean": 0.06204714, "gate_b2a/L7_absmax": 0.32709017, "gate_b2a/L7_rms": 0.06739271, "gate_b2a/L11_absmean": 0.09815783, "gate_b2a/L11_absmax": 0.57976341, "gate_b2a/L11_rms": 0.10520015, "gate_b2a/L14_absmean": 0.25247097, "gate_b2a/L14_absmax": 0.73056322, "gate_b2a/L14_rms": 0.25857195, "gate_b2c/L3_absmean": 0.05375376, "gate_b2c/L3_absmax": 0.64378303, "gate_b2c/L3_rms": 0.06654375, "gate_b2c/L5_absmean": 0.06068692, "gate_b2c/L5_absmax": 0.7320103, "gate_b2c/L5_rms": 0.07253628, "gate_b2c/L6_absmean": 0.08854606, "gate_b2c/L6_absmax": 0.50369334, "gate_b2c/L6_rms": 0.09722295}, "streams": {"cos_a2b/round0": 0.386253, "cos_c2b/round0": 0.265674, "cos_b2a/round0": 0.12448, "cos_b2c/round0": 0.101405, "rms_a/round0": 2.852558, "rms_b/round0": 5.352273, "rms_c/round0": 2.589404, "cos_a2b/round1": 0.427301, "cos_c2b/round1": 0.29594, "cos_b2a/round1": 0.232756, "cos_b2c/round1": 0.125374, "rms_a/round1": 5.340941, "rms_b/round1": 12.027742, "rms_c/round1": 5.15403, "cos_a2b/round2": 0.527315, "cos_c2b/round2": 0.50557, "cos_b2a/round2": 0.078036, "cos_b2c/round2": 0.154445, "rms_a/round2": 12.930782, "rms_b/round2": 28.914824, "rms_c/round2": 9.278826}, "alpha_mean": 0.43034695631237696, "alpha_hist": [0.03064, 0.1377, 0.18379, 0.15436, 0.12297, 0.10913, 0.10216, 0.09595, 0.05824, 0.00507], "alpha_by_type": {"special": {"share": 0.00108, "mean": 0.65006, "std": 0.104}, "whitespace": {"share": 0.03607, "mean": 0.6174, "std": 0.17204}, "digit": {"share": 0.00946, "mean": 0.37859, "std": 0.18163}, "word_start": {"share": 0.57304, "mean": 0.44288, "std": 0.21272}, "word_cont": {"share": 0.26385, "mean": 0.28589, "std": 0.14755}, "punct": {"share": 0.11649, "mean": 0.64012, "std": 0.19057}}, "alpha_p05": 0.11241, "alpha_p50": 0.39141, "alpha_p95": 0.81197}
{"step": 33810, "total_steps": 72479, "tokens": 17726177280, "loss": 4.628859877586365, "l_mix": 2.868415057659149, "l_a": 2.9608302414417267, "l_c": 2.907318949699402, "z_loss": 28.851540565490723, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14721675217151642, "tokens_per_second": 1440093.7207220877, "eta_seconds": 14078.036991810799, "data_wait_seconds": 2.22623348236084e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20041, "finemath": 0.08234}}
{"step": 33820, "total_steps": 72479, "tokens": 17731420160, "loss": 4.5474889278411865, "l_mix": 2.817124664783478, "l_a": 2.910207122564316, "l_c": 2.857673555612564, "z_loss": 29.649287700653076, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12037616223096848, "tokens_per_second": 1374271.862691757, "eta_seconds": 14748.500891447067, "data_wait_seconds": 2.4706125259399414e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 33830, "total_steps": 72479, "tokens": 17736663040, "loss": 4.500965237617493, "l_mix": 2.7885440587997437, "l_a": 2.8802222311496735, "l_c": 2.827848047018051, "z_loss": 29.866633892059326, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14005745947360992, "tokens_per_second": 1427850.3025835375, "eta_seconds": 14191.408493828774, "data_wait_seconds": 1.531839370727539e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 33840, "total_steps": 72479, "tokens": 17741905920, "loss": 4.46114307641983, "l_mix": 2.763993591070175, "l_a": 2.853158414363861, "l_c": 2.804006278514862, "z_loss": 29.030147790908813, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13486352562904358, "tokens_per_second": 1435863.9872809523, "eta_seconds": 14108.553603577613, "data_wait_seconds": 1.5676021575927734e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 33850, "total_steps": 72479, "tokens": 17747148800, "loss": 3.2721000611782074, "l_mix": 2.0131296515464783, "l_a": 2.131293773651123, "l_c": 2.065274178981781, "z_loss": 29.971886157989502, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1468125730752945, "tokens_per_second": 1363188.8086742277, "eta_seconds": 14856.871640324593, "data_wait_seconds": 2.4616718292236328e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 33860, "total_steps": 72479, "tokens": 17752391680, "loss": 3.2537107467651367, "l_mix": 2.002165198326111, "l_a": 2.1170847713947296, "l_c": 2.054733455181122, "z_loss": 30.84187912940979, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.23268134891986847, "tokens_per_second": 1302600.224106474, "eta_seconds": 15543.892820906638, "data_wait_seconds": 3.090500831604004e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 33870, "total_steps": 72479, "tokens": 17757634560, "loss": 3.254931926727295, "l_mix": 2.00255623459816, "l_a": 2.117331385612488, "l_c": 2.057253986597061, "z_loss": 30.08291244506836, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17070163786411285, "tokens_per_second": 1418356.548846709, "eta_seconds": 14271.61273972988, "data_wait_seconds": 2.5093555450439453e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 33880, "total_steps": 72479, "tokens": 17762877440, "loss": 3.299899995326996, "l_mix": 2.030973434448242, "l_a": 2.1497007608413696, "l_c": 2.0800544917583466, "z_loss": 30.594666242599487, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18037763237953186, "tokens_per_second": 1433976.548081299, "eta_seconds": 14112.498938059807, "data_wait_seconds": 3.844499588012695e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 33890, "total_steps": 72479, "tokens": 17768120320, "loss": 4.741639196872711, "l_mix": 2.940710037946701, "l_a": 3.021300882101059, "l_c": 2.9817959666252136, "z_loss": 28.149176597595215, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17816780507564545, "tokens_per_second": 1450531.658459937, "eta_seconds": 13947.816660189628, "data_wait_seconds": 3.510713577270508e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 33900, "total_steps": 72479, "tokens": 17773363200, "loss": 3.224380671977997, "l_mix": 1.9842291325330734, "l_a": 2.0998191833496094, "l_c": 2.0340189337730408, "z_loss": 30.34607982635498, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18287652730941772, "tokens_per_second": 1250202.7538804342, "eta_seconds": 16178.581185507775, "data_wait_seconds": 2.0116567611694336e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.25885725, "gate_a2b/L1_absmax": 0.70205718, "gate_a2b/L1_rms": 0.2731111, "gate_a2b/L3_absmean": 0.26863274, "gate_a2b/L3_absmax": 0.62514234, "gate_a2b/L3_rms": 0.27453607, "gate_a2b/L5_absmean": 0.42112717, "gate_a2b/L5_absmax": 0.64349359, "gate_a2b/L5_rms": 0.42698407, "gate_c2b/L1_absmean": 0.01715067, "gate_c2b/L1_absmax": 0.57498366, "gate_c2b/L1_rms": 0.06640642, "gate_c2b/L3_absmean": 0.21696194, "gate_c2b/L3_absmax": 0.48333785, "gate_c2b/L3_rms": 0.22134185, "gate_c2b/L5_absmean": 0.59439927, "gate_c2b/L5_absmax": 0.83996582, "gate_c2b/L5_rms": 0.60787296, "gate_b2a/L7_absmean": 0.06216359, "gate_b2a/L7_absmax": 0.32611945, "gate_b2a/L7_rms": 0.06746288, "gate_b2a/L11_absmean": 0.09768389, "gate_b2a/L11_absmax": 0.58067185, "gate_b2a/L11_rms": 0.10481719, "gate_b2a/L14_absmean": 0.25077462, "gate_b2a/L14_absmax": 0.73264289, "gate_b2a/L14_rms": 0.25694448, "gate_b2c/L3_absmean": 0.05296712, "gate_b2c/L3_absmax": 0.64287794, "gate_b2c/L3_rms": 0.06577841, "gate_b2c/L5_absmean": 0.05992299, "gate_b2c/L5_absmax": 0.73430258, "gate_b2c/L5_rms": 0.07196735, "gate_b2c/L6_absmean": 0.08777134, "gate_b2c/L6_absmax": 0.50088662, "gate_b2c/L6_rms": 0.09635118}, "streams": {"cos_a2b/round0": 0.385691, "cos_c2b/round0": 0.274195, "cos_b2a/round0": 0.126872, "cos_b2c/round0": 0.105098, "rms_a/round0": 2.894093, "rms_b/round0": 5.355435, "rms_c/round0": 2.608959, "cos_a2b/round1": 0.426318, "cos_c2b/round1": 0.29908, "cos_b2a/round1": 0.232334, "cos_b2c/round1": 0.124247, "rms_a/round1": 5.401041, "rms_b/round1": 12.078692, "rms_c/round1": 5.152702, "cos_a2b/round2": 0.532523, "cos_c2b/round2": 0.505999, "cos_b2a/round2": 0.07995, "cos_b2c/round2": 0.156016, "rms_a/round2": 13.146876, "rms_b/round2": 29.268995, "rms_c/round2": 9.307114}, "alpha_mean": 0.4282588938280085, "alpha_hist": [0.02426, 0.13235, 0.18327, 0.16064, 0.13625, 0.11707, 0.10359, 0.09331, 0.04721, 0.00204], "alpha_by_type": {"special": {"share": 0.00116, "mean": 0.63228, "std": 0.11465}, "whitespace": {"share": 0.03365, "mean": 0.61483, "std": 0.16313}, "digit": {"share": 0.00948, "mean": 0.38336, "std": 0.17359}, "word_start": {"share": 0.59241, "mean": 0.43309, "std": 0.20235}, "word_cont": {"share": 0.24594, "mean": 0.29291, "std": 0.14276}, "punct": {"share": 0.11737, "mean": 0.6356, "std": 0.18089}}, "alpha_p05": 0.12918, "alpha_p50": 0.40686, "alpha_p95": 0.7947}
{"step": 33910, "total_steps": 72479, "tokens": 17778606080, "loss": 3.2558139860630035, "l_mix": 2.0027266144752502, "l_a": 2.1192339658737183, "l_c": 2.057723581790924, "z_loss": 29.65976858139038, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17697855830192566, "tokens_per_second": 1414294.0172021096, "eta_seconds": 14297.77940516472, "data_wait_seconds": 2.0205974578857422e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 33920, "total_steps": 72479, "tokens": 17783848960, "loss": 4.4115447998046875, "l_mix": 2.733544200658798, "l_a": 2.821993261575699, "l_c": 2.771341860294342, "z_loss": 29.428359270095825, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14155687391757965, "tokens_per_second": 1444407.0775277135, "eta_seconds": 13996.068910574913, "data_wait_seconds": 2.3066997528076172e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08234}}
{"step": 33930, "total_steps": 72479, "tokens": 17789091840, "loss": 4.472773492336273, "l_mix": 2.7711499631404877, "l_a": 2.8625321090221405, "l_c": 2.809546023607254, "z_loss": 29.192845582962036, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.10263197869062424, "tokens_per_second": 1221085.3926980754, "eta_seconds": 16551.48627021313, "data_wait_seconds": 1.7315149307250977e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 33940, "total_steps": 72479, "tokens": 17794334720, "loss": 4.403735518455505, "l_mix": 2.728454679250717, "l_a": 2.817542165517807, "l_c": 2.7667271494865417, "z_loss": 29.01671600341797, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1190076395869255, "tokens_per_second": 1401968.5339822995, "eta_seconds": 14412.260148668289, "data_wait_seconds": 1.9311904907226562e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 33950, "total_steps": 72479, "tokens": 17799577600, "loss": 4.492703139781952, "l_mix": 2.782963275909424, "l_a": 2.8748416900634766, "l_c": 2.824290782213211, "z_loss": 30.154160261154175, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.155568465590477, "tokens_per_second": 1392060.172542736, "eta_seconds": 14511.07699971199, "data_wait_seconds": 1.576542854309082e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 33960, "total_steps": 72479, "tokens": 17804820480, "loss": 3.253506451845169, "l_mix": 2.001828983426094, "l_a": 2.11914199590683, "l_c": 2.0531161427497864, "z_loss": 31.43660259246826, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17319995164871216, "tokens_per_second": 1455530.4445906798, "eta_seconds": 13874.700832986833, "data_wait_seconds": 1.71661376953125e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 33970, "total_steps": 72479, "tokens": 17810063360, "loss": 3.1915145814418793, "l_mix": 1.9628154784440994, "l_a": 2.080400973558426, "l_c": 2.015262618660927, "z_loss": 30.16076111793518, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1942550241947174, "tokens_per_second": 1479242.993162391, "eta_seconds": 13648.742421174049, "data_wait_seconds": 2.142786979675293e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 33980, "total_steps": 72479, "tokens": 17815306240, "loss": 3.2346912622451782, "l_mix": 1.990044504404068, "l_a": 2.106855481863022, "l_c": 2.041966885328293, "z_loss": 30.152177572250366, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16377797722816467, "tokens_per_second": 1455795.720689048, "eta_seconds": 13864.969806647301, "data_wait_seconds": 2.2798776626586914e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 33990, "total_steps": 72479, "tokens": 17820549120, "loss": 3.2067067325115204, "l_mix": 1.9725871086120605, "l_a": 2.0904984772205353, "l_c": 2.023233324289322, "z_loss": 31.398889780044556, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16932322084903717, "tokens_per_second": 1415477.3932190016, "eta_seconds": 14256.194361472131, "data_wait_seconds": 3.153085708618164e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 34000, "total_steps": 72479, "tokens": 17825792000, "loss": 3.304900139570236, "l_mix": 2.0337578654289246, "l_a": 2.1490062177181244, "l_c": 2.088134527206421, "z_loss": 29.857069730758667, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15906977653503418, "tokens_per_second": 1429410.2140010518, "eta_seconds": 14113.567787885664, "data_wait_seconds": 4.607439041137695e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.25855142, "gate_a2b/L1_absmax": 0.70147532, "gate_a2b/L1_rms": 0.2726706, "gate_a2b/L3_absmean": 0.26761243, "gate_a2b/L3_absmax": 0.62526715, "gate_a2b/L3_rms": 0.27355775, "gate_a2b/L5_absmean": 0.41960797, "gate_a2b/L5_absmax": 0.64946479, "gate_a2b/L5_rms": 0.42543176, "gate_c2b/L1_absmean": 0.01728162, "gate_c2b/L1_absmax": 0.5750789, "gate_c2b/L1_rms": 0.0664576, "gate_c2b/L3_absmean": 0.21719471, "gate_c2b/L3_absmax": 0.48332405, "gate_c2b/L3_rms": 0.22154972, "gate_c2b/L5_absmean": 0.59424889, "gate_c2b/L5_absmax": 0.8362506, "gate_c2b/L5_rms": 0.60777783, "gate_b2a/L7_absmean": 0.0616509, "gate_b2a/L7_absmax": 0.32421392, "gate_b2a/L7_rms": 0.06697584, "gate_b2a/L11_absmean": 0.09778334, "gate_b2a/L11_absmax": 0.58024597, "gate_b2a/L11_rms": 0.1048824, "gate_b2a/L14_absmean": 0.24984285, "gate_b2a/L14_absmax": 0.73397553, "gate_b2a/L14_rms": 0.25607592, "gate_b2c/L3_absmean": 0.05309723, "gate_b2c/L3_absmax": 0.64215487, "gate_b2c/L3_rms": 0.06579453, "gate_b2c/L5_absmean": 0.05949252, "gate_b2c/L5_absmax": 0.7305814, "gate_b2c/L5_rms": 0.0714485, "gate_b2c/L6_absmean": 0.08680823, "gate_b2c/L6_absmax": 0.50216424, "gate_b2c/L6_rms": 0.0955455}, "streams": {"cos_a2b/round0": 0.388355, "cos_c2b/round0": 0.266977, "cos_b2a/round0": 0.124196, "cos_b2c/round0": 0.100995, "rms_a/round0": 2.941586, "rms_b/round0": 5.400239, "rms_c/round0": 2.620722, "cos_a2b/round1": 0.420995, "cos_c2b/round1": 0.297711, "cos_b2a/round1": 0.232801, "cos_b2c/round1": 0.121513, "rms_a/round1": 5.398344, "rms_b/round1": 12.191888, "rms_c/round1": 5.136944, "cos_a2b/round2": 0.528794, "cos_c2b/round2": 0.504265, "cos_b2a/round2": 0.073524, "cos_b2c/round2": 0.157082, "rms_a/round2": 13.161198, "rms_b/round2": 29.526258, "rms_c/round2": 9.11683}, "alpha_mean": 0.40845649176344523, "alpha_hist": [0.03102, 0.15208, 0.19456, 0.15765, 0.12642, 0.11621, 0.10696, 0.08081, 0.03223, 0.00204], "alpha_by_type": {"special": {"share": 0.00107, "mean": 0.60668, "std": 0.1048}, "whitespace": {"share": 0.03835, "mean": 0.61248, "std": 0.16357}, "digit": {"share": 0.01208, "mean": 0.39086, "std": 0.17043}, "word_start": {"share": 0.5546, "mean": 0.40812, "std": 0.19997}, "word_cont": {"share": 0.26405, "mean": 0.28448, "std": 0.14263}, "punct": {"share": 0.12985, "mean": 0.60178, "std": 0.18256}}, "alpha_p05": 0.11982, "alpha_p50": 0.37479, "alpha_p95": 0.78858}
{"step": 34010, "total_steps": 72479, "tokens": 17831034880, "loss": 4.73876166343689, "l_mix": 2.9383219480514526, "l_a": 3.025291293859482, "l_c": 2.9761742651462555, "z_loss": 29.324402809143066, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17996782064437866, "tokens_per_second": 1378287.5040674408, "eta_seconds": 14633.256858587265, "data_wait_seconds": 1.621246337890625e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 34020, "total_steps": 72479, "tokens": 17836277760, "loss": 4.85167384147644, "l_mix": 3.009196013212204, "l_a": 3.0911785066127777, "l_c": 3.0504139959812164, "z_loss": 27.745835542678833, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15701821446418762, "tokens_per_second": 1411165.5677908042, "eta_seconds": 14288.608404445647, "data_wait_seconds": 2.014636993408203e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 34030, "total_steps": 72479, "tokens": 17841520640, "loss": 4.504566967487335, "l_mix": 2.7911021411418915, "l_a": 2.8837114572525024, "l_c": 2.8278378546237946, "z_loss": 29.057506561279297, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1319485306739807, "tokens_per_second": 1453661.6089728645, "eta_seconds": 13867.291526150702, "data_wait_seconds": 2.485513687133789e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08234}}
{"step": 34040, "total_steps": 72479, "tokens": 17846763520, "loss": 4.535112261772156, "l_mix": 2.8100315034389496, "l_a": 2.8993712067604065, "l_c": 2.8508976995944977, "z_loss": 29.33825421333313, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1269233524799347, "tokens_per_second": 1463687.8497035448, "eta_seconds": 13768.718812608719, "data_wait_seconds": 1.6808509826660156e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08234}}
{"step": 34050, "total_steps": 72479, "tokens": 17852006400, "loss": 4.490519046783447, "l_mix": 2.7824667394161224, "l_a": 2.872596800327301, "l_c": 2.820910483598709, "z_loss": 29.654008150100708, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12874455749988556, "tokens_per_second": 1407946.622517752, "eta_seconds": 14310.104679942131, "data_wait_seconds": 2.4974346160888672e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 34060, "total_steps": 72479, "tokens": 17857249280, "loss": 4.472794532775879, "l_mix": 2.7717458307743073, "l_a": 2.8596136271953583, "l_c": 2.810548335313797, "z_loss": 28.89613652229309, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15261249244213104, "tokens_per_second": 1421384.6878167419, "eta_seconds": 14171.12541358471, "data_wait_seconds": 3.460049629211426e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 34070, "total_steps": 72479, "tokens": 17862492160, "loss": 3.2798737287521362, "l_mix": 2.0198877155780792, "l_a": 2.130536675453186, "l_c": 2.0694164633750916, "z_loss": 29.987375259399414, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.21473896503448486, "tokens_per_second": 1368814.1834490392, "eta_seconds": 14711.549628496168, "data_wait_seconds": 3.0308961868286133e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 34080, "total_steps": 72479, "tokens": 17867735040, "loss": 3.2956999838352203, "l_mix": 2.0274411737918854, "l_a": 2.1432995796203613, "l_c": 2.0842297077178955, "z_loss": 29.922837495803833, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19173766672611237, "tokens_per_second": 1439370.1709703642, "eta_seconds": 13986.766794276236, "data_wait_seconds": 3.0994415283203125e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 34090, "total_steps": 72479, "tokens": 17872977920, "loss": 3.247555732727051, "l_mix": 1.996891900897026, "l_a": 2.1185601353645325, "l_c": 2.050319164991379, "z_loss": 31.638892650604248, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1822076439857483, "tokens_per_second": 1434691.7823707147, "eta_seconds": 14028.721903419495, "data_wait_seconds": 2.2858381271362305e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 34100, "total_steps": 72479, "tokens": 17878220800, "loss": 3.248990476131439, "l_mix": 1.998498573899269, "l_a": 2.1168077886104584, "l_c": 2.0514984726905823, "z_loss": 31.06027603149414, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1674388200044632, "tokens_per_second": 1453369.6375933245, "eta_seconds": 13844.82559118271, "data_wait_seconds": 1.65402889251709e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.25767246, "gate_a2b/L1_absmax": 0.70261949, "gate_a2b/L1_rms": 0.27177387, "gate_a2b/L3_absmean": 0.26606321, "gate_a2b/L3_absmax": 0.62796211, "gate_a2b/L3_rms": 0.27193856, "gate_a2b/L5_absmean": 0.41759396, "gate_a2b/L5_absmax": 0.65033978, "gate_a2b/L5_rms": 0.42346248, "gate_c2b/L1_absmean": 0.01709961, "gate_c2b/L1_absmax": 0.57722169, "gate_c2b/L1_rms": 0.0668539, "gate_c2b/L3_absmean": 0.21665302, "gate_c2b/L3_absmax": 0.48307455, "gate_c2b/L3_rms": 0.22099112, "gate_c2b/L5_absmean": 0.59412444, "gate_c2b/L5_absmax": 0.83113748, "gate_c2b/L5_rms": 0.60765469, "gate_b2a/L7_absmean": 0.0617232, "gate_b2a/L7_absmax": 0.32546672, "gate_b2a/L7_rms": 0.06703791, "gate_b2a/L11_absmean": 0.09709616, "gate_b2a/L11_absmax": 0.57968795, "gate_b2a/L11_rms": 0.10424226, "gate_b2a/L14_absmean": 0.24893574, "gate_b2a/L14_absmax": 0.73393583, "gate_b2a/L14_rms": 0.25516629, "gate_b2c/L3_absmean": 0.05249205, "gate_b2c/L3_absmax": 0.64038336, "gate_b2c/L3_rms": 0.06515739, "gate_b2c/L5_absmean": 0.05903361, "gate_b2c/L5_absmax": 0.72566837, "gate_b2c/L5_rms": 0.07096525, "gate_b2c/L6_absmean": 0.08617727, "gate_b2c/L6_absmax": 0.49977958, "gate_b2c/L6_rms": 0.09481255}, "streams": {"cos_a2b/round0": 0.386547, "cos_c2b/round0": 0.272036, "cos_b2a/round0": 0.125005, "cos_b2c/round0": 0.10135, "rms_a/round0": 2.942689, "rms_b/round0": 5.479521, "rms_c/round0": 2.643932, "cos_a2b/round1": 0.42077, "cos_c2b/round1": 0.297109, "cos_b2a/round1": 0.230774, "cos_b2c/round1": 0.123314, "rms_a/round1": 5.47292, "rms_b/round1": 12.447571, "rms_c/round1": 5.211285, "cos_a2b/round2": 0.519678, "cos_c2b/round2": 0.511848, "cos_b2a/round2": 0.075098, "cos_b2c/round2": 0.153716, "rms_a/round2": 13.319821, "rms_b/round2": 30.116934, "rms_c/round2": 9.382845}, "alpha_mean": 0.4156647952133312, "alpha_hist": [0.03607, 0.14656, 0.19313, 0.1523, 0.11816, 0.11179, 0.10435, 0.09409, 0.04137, 0.00218], "alpha_by_type": {"special": {"share": 0.00107, "mean": 0.60867, "std": 0.1063}, "whitespace": {"share": 0.04576, "mean": 0.5986, "std": 0.1608}, "digit": {"share": 0.01109, "mean": 0.34867, "std": 0.16791}, "word_start": {"share": 0.56609, "mean": 0.42539, "std": 0.20873}, "word_cont": {"share": 0.25578, "mean": 0.27019, "std": 0.13969}, "punct": {"share": 0.12021, "mean": 0.61423, "std": 0.1847}}, "alpha_p05": 0.12148, "alpha_p50": 0.397, "alpha_p95": 0.80776}
{"step": 34110, "total_steps": 72479, "tokens": 17883463680, "loss": 3.252300351858139, "l_mix": 2.0006728172302246, "l_a": 2.1193389296531677, "l_c": 2.052752807736397, "z_loss": 30.09430170059204, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1644061952829361, "tokens_per_second": 1474264.0108419943, "eta_seconds": 13645.050088763237, "data_wait_seconds": 2.1517276763916016e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 34120, "total_steps": 72479, "tokens": 17888706560, "loss": 4.496797561645508, "l_mix": 2.7862452566623688, "l_a": 2.874582141637802, "l_c": 2.8272584676742554, "z_loss": 29.486153841018677, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.10955459624528885, "tokens_per_second": 1440759.470437717, "eta_seconds": 13958.723718047142, "data_wait_seconds": 1.9103288650512695e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 34130, "total_steps": 72479, "tokens": 17893949440, "loss": 4.564954996109009, "l_mix": 2.829047679901123, "l_a": 2.918682634830475, "l_c": 2.8676748871803284, "z_loss": 29.00421667098999, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1406790316104889, "tokens_per_second": 1426969.5268020523, "eta_seconds": 14089.943852591516, "data_wait_seconds": 1.576542854309082e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 34140, "total_steps": 72479, "tokens": 17899192320, "loss": 4.603924632072449, "l_mix": 2.8535247147083282, "l_a": 2.9417343735694885, "l_c": 2.892932027578354, "z_loss": 28.72170877456665, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14475001394748688, "tokens_per_second": 1427987.9930480644, "eta_seconds": 14076.22314043045, "data_wait_seconds": 1.9073486328125e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 34150, "total_steps": 72479, "tokens": 17904435200, "loss": 4.537801146507263, "l_mix": 2.8119834661483765, "l_a": 2.9011741280555725, "l_c": 2.8515513241291046, "z_loss": 29.122481107711792, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1480570286512375, "tokens_per_second": 1417414.898189241, "eta_seconds": 14177.524716067313, "data_wait_seconds": 1.9490718841552734e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 34160, "total_steps": 72479, "tokens": 17909678080, "loss": 4.500308573246002, "l_mix": 2.788322538137436, "l_a": 2.877624422311783, "l_c": 2.8289957642555237, "z_loss": 29.918431043624878, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1339447945356369, "tokens_per_second": 1428022.767567173, "eta_seconds": 14068.537510943412, "data_wait_seconds": 2.5659799575805664e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 34170, "total_steps": 72479, "tokens": 17914920960, "loss": 4.55185079574585, "l_mix": 2.8205814361572266, "l_a": 2.910525679588318, "l_c": 2.860371857881546, "z_loss": 29.51001501083374, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13283242285251617, "tokens_per_second": 1426429.5170306035, "eta_seconds": 14080.575837922095, "data_wait_seconds": 2.664327621459961e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 34180, "total_steps": 72479, "tokens": 17920163840, "loss": 3.2791293263435364, "l_mix": 2.0169576853513718, "l_a": 2.13555571436882, "l_c": 2.0716829895973206, "z_loss": 31.079657077789307, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14231130480766296, "tokens_per_second": 1384805.2609398302, "eta_seconds": 14500.021539759635, "data_wait_seconds": 2.294778823852539e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 34190, "total_steps": 72479, "tokens": 17925406720, "loss": 3.261438876390457, "l_mix": 2.0060189813375473, "l_a": 2.1268724501132965, "l_c": 2.057860255241394, "z_loss": 32.07836890220642, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.24735292792320251, "tokens_per_second": 1371008.0291042163, "eta_seconds": 14642.119379210471, "data_wait_seconds": 2.580881118774414e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 34200, "total_steps": 72479, "tokens": 17930649600, "loss": 3.2589160799980164, "l_mix": 2.005257412791252, "l_a": 2.120865672826767, "l_c": 2.0579964369535446, "z_loss": 30.582038402557373, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15232820808887482, "tokens_per_second": 1348676.3458482306, "eta_seconds": 14880.679426002502, "data_wait_seconds": 2.631545066833496e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.25683358, "gate_a2b/L1_absmax": 0.70239419, "gate_a2b/L1_rms": 0.27085608, "gate_a2b/L3_absmean": 0.26503426, "gate_a2b/L3_absmax": 0.62491089, "gate_a2b/L3_rms": 0.27102122, "gate_a2b/L5_absmean": 0.41594887, "gate_a2b/L5_absmax": 0.65004414, "gate_a2b/L5_rms": 0.42186207, "gate_c2b/L1_absmean": 0.01686077, "gate_c2b/L1_absmax": 0.57614231, "gate_c2b/L1_rms": 0.06654341, "gate_c2b/L3_absmean": 0.21682623, "gate_c2b/L3_absmax": 0.47853395, "gate_c2b/L3_rms": 0.22116266, "gate_c2b/L5_absmean": 0.59449607, "gate_c2b/L5_absmax": 0.83089638, "gate_c2b/L5_rms": 0.60802358, "gate_b2a/L7_absmean": 0.0612207, "gate_b2a/L7_absmax": 0.32482889, "gate_b2a/L7_rms": 0.0665772, "gate_b2a/L11_absmean": 0.09675424, "gate_b2a/L11_absmax": 0.57894522, "gate_b2a/L11_rms": 0.1038833, "gate_b2a/L14_absmean": 0.24834834, "gate_b2a/L14_absmax": 0.73237127, "gate_b2a/L14_rms": 0.25460047, "gate_b2c/L3_absmean": 0.05198389, "gate_b2c/L3_absmax": 0.63815987, "gate_b2c/L3_rms": 0.06470092, "gate_b2c/L5_absmean": 0.05856826, "gate_b2c/L5_absmax": 0.72574961, "gate_b2c/L5_rms": 0.07054248, "gate_b2c/L6_absmean": 0.08554407, "gate_b2c/L6_absmax": 0.49706706, "gate_b2c/L6_rms": 0.09410915}, "streams": {"cos_a2b/round0": 0.384301, "cos_c2b/round0": 0.264367, "cos_b2a/round0": 0.127495, "cos_b2c/round0": 0.103517, "rms_a/round0": 2.980505, "rms_b/round0": 5.600064, "rms_c/round0": 2.659483, "cos_a2b/round1": 0.423038, "cos_c2b/round1": 0.300043, "cos_b2a/round1": 0.233737, "cos_b2c/round1": 0.125476, "rms_a/round1": 5.575243, "rms_b/round1": 12.606107, "rms_c/round1": 5.283909, "cos_a2b/round2": 0.530489, "cos_c2b/round2": 0.50444, "cos_b2a/round2": 0.076272, "cos_b2c/round2": 0.15698, "rms_a/round2": 13.603116, "rms_b/round2": 30.344517, "rms_c/round2": 9.512041}, "alpha_mean": 0.43450166354254804, "alpha_hist": [0.02963, 0.13329, 0.17961, 0.15059, 0.12332, 0.11191, 0.112, 0.10411, 0.052, 0.00354], "alpha_by_type": {"special": {"share": 0.00119, "mean": 0.58643, "std": 0.08527}, "whitespace": {"share": 0.03, "mean": 0.64427, "std": 0.17265}, "digit": {"share": 0.01012, "mean": 0.37723, "std": 0.16181}, "word_start": {"share": 0.59691, "mean": 0.45191, "std": 0.20846}, "word_cont": {"share": 0.25076, "mean": 0.28062, "std": 0.14986}, "punct": {"share": 0.11102, "mean": 0.63539, "std": 0.18368}}, "alpha_p05": 0.1188, "alpha_p50": 0.41632, "alpha_p95": 0.80409}
{"step": 34210, "total_steps": 72479, "tokens": 17935892480, "loss": 3.198977053165436, "l_mix": 1.9674755334854126, "l_a": 2.0848986208438873, "l_c": 2.020106226205826, "z_loss": 30.45298671722412, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14161071181297302, "tokens_per_second": 1419552.5090741627, "eta_seconds": 14134.0157153368, "data_wait_seconds": 2.491474151611328e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 34220, "total_steps": 72479, "tokens": 17941135360, "loss": 3.238167554140091, "l_mix": 1.991700068116188, "l_a": 2.1119924783706665, "l_c": 2.0428989231586456, "z_loss": 31.4456627368927, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18371567130088806, "tokens_per_second": 1453797.8832106895, "eta_seconds": 13797.471315407753, "data_wait_seconds": 1.6242265701293945e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 34230, "total_steps": 72479, "tokens": 17946378240, "loss": 4.534216225147247, "l_mix": 2.8099229633808136, "l_a": 2.8982431888580322, "l_c": 2.8494008481502533, "z_loss": 29.136749505996704, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13989782333374023, "tokens_per_second": 1440716.1440213467, "eta_seconds": 13919.113626384735, "data_wait_seconds": 1.6033649444580078e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 34240, "total_steps": 72479, "tokens": 17951621120, "loss": 4.502937018871307, "l_mix": 2.790084421634674, "l_a": 2.8827053606510162, "l_c": 2.826803296804428, "z_loss": 29.588112831115723, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12915346026420593, "tokens_per_second": 1454566.5137008927, "eta_seconds": 13782.971519804001, "data_wait_seconds": 2.54213809967041e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 34250, "total_steps": 72479, "tokens": 17956864000, "loss": 4.534232139587402, "l_mix": 2.8098729848861694, "l_a": 2.8983347713947296, "l_c": 2.849528878927231, "z_loss": 29.028980493545532, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18840469419956207, "tokens_per_second": 1444058.497969146, "eta_seconds": 13879.635748958588, "data_wait_seconds": 1.7255544662475586e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 34260, "total_steps": 72479, "tokens": 17962106880, "loss": 4.406144917011261, "l_mix": 2.730712413787842, "l_a": 2.8189875781536102, "l_c": 2.7657869458198547, "z_loss": 29.459841012954712, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15347912907600403, "tokens_per_second": 1285258.2412744951, "eta_seconds": 15590.456787991523, "data_wait_seconds": 4.4912099838256836e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 34270, "total_steps": 72479, "tokens": 17967349760, "loss": 4.483256697654724, "l_mix": 2.7767433524131775, "l_a": 2.8683154582977295, "l_c": 2.8200617730617523, "z_loss": 30.421740531921387, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15561209619045258, "tokens_per_second": 1425791.9175972308, "eta_seconds": 14050.100820994378, "data_wait_seconds": 2.1964311599731445e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 34280, "total_steps": 72479, "tokens": 17972592640, "loss": 3.2096438705921173, "l_mix": 1.9745748937129974, "l_a": 2.0889446139335632, "l_c": 2.0279519259929657, "z_loss": 31.10763645172119, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15721569955348969, "tokens_per_second": 1422881.8828688774, "eta_seconds": 14075.150968694687, "data_wait_seconds": 1.704692840576172e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 34290, "total_steps": 72479, "tokens": 17977835520, "loss": 4.601780652999878, "l_mix": 2.852890759706497, "l_a": 2.9409609735012054, "l_c": 2.888671964406967, "z_loss": 29.650068283081055, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14560182392597198, "tokens_per_second": 1415007.4108874886, "eta_seconds": 14149.773547434806, "data_wait_seconds": 1.7642974853515625e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 34300, "total_steps": 72479, "tokens": 17983078400, "loss": 4.711710751056671, "l_mix": 2.922221213579178, "l_a": 3.0037162601947784, "l_c": 2.9612483084201813, "z_loss": 29.148065090179443, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12826451659202576, "tokens_per_second": 1392114.8986891026, "eta_seconds": 14378.692140173911, "data_wait_seconds": 1.6450881958007812e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.25567046, "gate_a2b/L1_absmax": 0.70265216, "gate_a2b/L1_rms": 0.26991272, "gate_a2b/L3_absmean": 0.2645483, "gate_a2b/L3_absmax": 0.61764145, "gate_a2b/L3_rms": 0.27045849, "gate_a2b/L5_absmean": 0.41417843, "gate_a2b/L5_absmax": 0.65177327, "gate_a2b/L5_rms": 0.42003512, "gate_c2b/L1_absmean": 0.01733166, "gate_c2b/L1_absmax": 0.57620949, "gate_c2b/L1_rms": 0.06677301, "gate_c2b/L3_absmean": 0.21678033, "gate_c2b/L3_absmax": 0.48096874, "gate_c2b/L3_rms": 0.22115391, "gate_c2b/L5_absmean": 0.59475398, "gate_c2b/L5_absmax": 0.83472985, "gate_c2b/L5_rms": 0.60829073, "gate_b2a/L7_absmean": 0.06124571, "gate_b2a/L7_absmax": 0.32749137, "gate_b2a/L7_rms": 0.06648967, "gate_b2a/L11_absmean": 0.09665997, "gate_b2a/L11_absmax": 0.58268446, "gate_b2a/L11_rms": 0.10382842, "gate_b2a/L14_absmean": 0.24730422, "gate_b2a/L14_absmax": 0.73214775, "gate_b2a/L14_rms": 0.25366119, "gate_b2c/L3_absmean": 0.0517777, "gate_b2c/L3_absmax": 0.64332002, "gate_b2c/L3_rms": 0.06458212, "gate_b2c/L5_absmean": 0.05844158, "gate_b2c/L5_absmax": 0.73076248, "gate_b2c/L5_rms": 0.07051827, "gate_b2c/L6_absmean": 0.08485703, "gate_b2c/L6_absmax": 0.48324168, "gate_b2c/L6_rms": 0.09330496}, "streams": {"cos_a2b/round0": 0.387407, "cos_c2b/round0": 0.308402, "cos_b2a/round0": 0.14073, "cos_b2c/round0": 0.130049, "rms_a/round0": 2.899891, "rms_b/round0": 5.525027, "rms_c/round0": 2.584235, "cos_a2b/round1": 0.428786, "cos_c2b/round1": 0.299802, "cos_b2a/round1": 0.246348, "cos_b2c/round1": 0.120895, "rms_a/round1": 5.284159, "rms_b/round1": 11.847308, "rms_c/round1": 4.973008, "cos_a2b/round2": 0.571884, "cos_c2b/round2": 0.526156, "cos_b2a/round2": 0.107112, "cos_b2c/round2": 0.173841, "rms_a/round2": 13.355, "rms_b/round2": 29.307627, "rms_c/round2": 8.862044}, "alpha_mean": 0.4607989405471926, "alpha_hist": [0.0058, 0.04308, 0.12923, 0.19133, 0.22601, 0.18936, 0.13568, 0.07028, 0.00922, 2e-05], "alpha_by_type": {"special": {"share": 0.00063, "mean": 0.53428, "std": 0.14768}, "whitespace": {"share": 0.05357, "mean": 0.51808, "std": 0.10867}, "digit": {"share": 0.02873, "mean": 0.44178, "std": 0.12548}, "word_start": {"share": 0.56656, "mean": 0.47811, "std": 0.15767}, "word_cont": {"share": 0.21504, "mean": 0.35546, "std": 0.13033}, "punct": {"share": 0.13547, "mean": 0.53668, "std": 0.1657}}, "alpha_p05": 0.19529, "alpha_p50": 0.45799, "alpha_p95": 0.71542}
{"step": 34310, "total_steps": 72479, "tokens": 17988321280, "loss": 3.233878970146179, "l_mix": 1.990235686302185, "l_a": 2.1036402881145477, "l_c": 2.0418370217084885, "z_loss": 30.289096117019653, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18056882917881012, "tokens_per_second": 1374560.9252853715, "eta_seconds": 14558.50250351429, "data_wait_seconds": 2.8133392333984375e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 34320, "total_steps": 72479, "tokens": 17993564160, "loss": 3.254602611064911, "l_mix": 2.0010382682085037, "l_a": 2.120885908603668, "l_c": 2.057662010192871, "z_loss": 31.13310194015503, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1733109951019287, "tokens_per_second": 1468154.1148619822, "eta_seconds": 13626.843115091326, "data_wait_seconds": 1.9997358322143555e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 34330, "total_steps": 72479, "tokens": 17998807040, "loss": 3.220925509929657, "l_mix": 1.982032597064972, "l_a": 2.0970500111579895, "l_c": 2.0325926542282104, "z_loss": 30.628669261932373, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16993579268455505, "tokens_per_second": 1440621.6656017709, "eta_seconds": 13883.633288025856, "data_wait_seconds": 1.5556812286376953e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 34340, "total_steps": 72479, "tokens": 18004049920, "loss": 4.462786912918091, "l_mix": 2.7657380998134613, "l_a": 2.8550581336021423, "l_c": 2.801770895719528, "z_loss": 29.902880907058716, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16729791462421417, "tokens_per_second": 1472117.3319009072, "eta_seconds": 13583.034177160262, "data_wait_seconds": 3.007054328918457e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08232}}
{"step": 34350, "total_steps": 72479, "tokens": 18009292800, "loss": 4.471851289272308, "l_mix": 2.770017236471176, "l_a": 2.8631047904491425, "l_c": 2.8096752762794495, "z_loss": 30.216603755950928, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14684176445007324, "tokens_per_second": 1474789.1225104227, "eta_seconds": 13554.871572399139, "data_wait_seconds": 2.428889274597168e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 34360, "total_steps": 72479, "tokens": 18014535680, "loss": 4.5361692905426025, "l_mix": 2.8111096918582916, "l_a": 2.8984963595867157, "l_c": 2.851702094078064, "z_loss": 28.780714511871338, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15484952926635742, "tokens_per_second": 1394133.603343958, "eta_seconds": 14335.307766819, "data_wait_seconds": 1.8805265426635742e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 34370, "total_steps": 72479, "tokens": 18019778560, "loss": 4.507437169551849, "l_mix": 2.7926759719848633, "l_a": 2.8857611417770386, "l_c": 2.830109119415283, "z_loss": 29.79324221611023, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15425899624824524, "tokens_per_second": 1364819.420940366, "eta_seconds": 14639.36626739502, "data_wait_seconds": 2.3365020751953125e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 34380, "total_steps": 72479, "tokens": 18025021440, "loss": 4.530286610126495, "l_mix": 2.807200402021408, "l_a": 2.8958219289779663, "l_c": 2.847798526287079, "z_loss": 29.70989441871643, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11715669184923172, "tokens_per_second": 1436773.4288140524, "eta_seconds": 13902.57371928692, "data_wait_seconds": 1.6450881958007812e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 34390, "total_steps": 72479, "tokens": 18030264320, "loss": 3.2079721689224243, "l_mix": 1.9737879633903503, "l_a": 2.087063491344452, "l_c": 2.0268836468458176, "z_loss": 31.58669638633728, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15120863914489746, "tokens_per_second": 1434731.658139488, "eta_seconds": 13918.704252958296, "data_wait_seconds": 3.167986869812012e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 34400, "total_steps": 72479, "tokens": 18035507200, "loss": 4.578091025352478, "l_mix": 2.8376345038414, "l_a": 2.9226392209529877, "l_c": 2.878882199525833, "z_loss": 29.63470983505249, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1284593939781189, "tokens_per_second": 1402123.8959121157, "eta_seconds": 14238.658088779448, "data_wait_seconds": 2.4765729904174805e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.25521255, "gate_a2b/L1_absmax": 0.70248902, "gate_a2b/L1_rms": 0.26924151, "gate_a2b/L3_absmean": 0.26301739, "gate_a2b/L3_absmax": 0.62195426, "gate_a2b/L3_rms": 0.26895705, "gate_a2b/L5_absmean": 0.41254827, "gate_a2b/L5_absmax": 0.65385139, "gate_a2b/L5_rms": 0.4183751, "gate_c2b/L1_absmean": 0.01727359, "gate_c2b/L1_absmax": 0.57668591, "gate_c2b/L1_rms": 0.06676909, "gate_c2b/L3_absmean": 0.21666217, "gate_c2b/L3_absmax": 0.47573808, "gate_c2b/L3_rms": 0.22097422, "gate_c2b/L5_absmean": 0.5947383, "gate_c2b/L5_absmax": 0.83525413, "gate_c2b/L5_rms": 0.60822839, "gate_b2a/L7_absmean": 0.06071535, "gate_b2a/L7_absmax": 0.32911146, "gate_b2a/L7_rms": 0.06617674, "gate_b2a/L11_absmean": 0.0961924, "gate_b2a/L11_absmax": 0.58026004, "gate_b2a/L11_rms": 0.10338791, "gate_b2a/L14_absmean": 0.24635337, "gate_b2a/L14_absmax": 0.7316978, "gate_b2a/L14_rms": 0.25279728, "gate_b2c/L3_absmean": 0.05129078, "gate_b2c/L3_absmax": 0.64090651, "gate_b2c/L3_rms": 0.06410656, "gate_b2c/L5_absmean": 0.057798, "gate_b2c/L5_absmax": 0.72199112, "gate_b2c/L5_rms": 0.06972686, "gate_b2c/L6_absmean": 0.08416458, "gate_b2c/L6_absmax": 0.48846385, "gate_b2c/L6_rms": 0.0926742}, "streams": {"cos_a2b/round0": 0.387842, "cos_c2b/round0": 0.310975, "cos_b2a/round0": 0.140355, "cos_b2c/round0": 0.131676, "rms_a/round0": 2.941186, "rms_b/round0": 5.567271, "rms_c/round0": 2.631841, "cos_a2b/round1": 0.430101, "cos_c2b/round1": 0.301481, "cos_b2a/round1": 0.243276, "cos_b2c/round1": 0.121096, "rms_a/round1": 5.424289, "rms_b/round1": 12.014957, "rms_c/round1": 5.068713, "cos_a2b/round2": 0.564682, "cos_c2b/round2": 0.524408, "cos_b2a/round2": 0.100306, "cos_b2c/round2": 0.173849, "rms_a/round2": 13.570202, "rms_b/round2": 29.746759, "rms_c/round2": 8.964427}, "alpha_mean": 0.4508307889870906, "alpha_hist": [0.00613, 0.0544, 0.14915, 0.18806, 0.19673, 0.20314, 0.13576, 0.06004, 0.00656, 2e-05], "alpha_by_type": {"special": {"share": 0.00053, "mean": 0.60404, "std": 0.11213}, "whitespace": {"share": 0.05836, "mean": 0.55412, "std": 0.10572}, "digit": {"share": 0.01746, "mean": 0.44678, "std": 0.13769}, "word_start": {"share": 0.58485, "mean": 0.46637, "std": 0.15313}, "word_cont": {"share": 0.23564, "mean": 0.33114, "std": 0.13038}, "punct": {"share": 0.10315, "mean": 0.57762, "std": 0.15221}}, "alpha_p05": 0.18359, "alpha_p50": 0.45048, "alpha_p95": 0.72701}
{"step": 34410, "total_steps": 72479, "tokens": 18040750080, "loss": 4.694774508476257, "l_mix": 2.911003530025482, "l_a": 2.991495966911316, "l_c": 2.9544070959091187, "z_loss": 28.38353729248047, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1868681162595749, "tokens_per_second": 1284978.4831500724, "eta_seconds": 15532.649093914033, "data_wait_seconds": 1.895427703857422e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 34420, "total_steps": 72479, "tokens": 18045992960, "loss": 4.695750117301941, "l_mix": 2.9128420650959015, "l_a": 2.9937679767608643, "l_c": 2.9492587447166443, "z_loss": 29.216254234313965, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1534363180398941, "tokens_per_second": 1405717.9806381175, "eta_seconds": 14194.793882441521, "data_wait_seconds": 2.0951032638549805e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 34430, "total_steps": 72479, "tokens": 18051235840, "loss": 4.614444613456726, "l_mix": 2.8610452711582184, "l_a": 2.945497751235962, "l_c": 2.899166524410248, "z_loss": 29.721529722213745, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14884473383426666, "tokens_per_second": 1386247.146247212, "eta_seconds": 14390.387865543365, "data_wait_seconds": 3.343820571899414e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 34440, "total_steps": 72479, "tokens": 18056478720, "loss": 4.644105672836304, "l_mix": 2.879987120628357, "l_a": 2.958810955286026, "l_c": 2.92158380150795, "z_loss": 28.895566940307617, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1574573963880539, "tokens_per_second": 1458837.633521041, "eta_seconds": 13670.740851306915, "data_wait_seconds": 3.248453140258789e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 34450, "total_steps": 72479, "tokens": 18061721600, "loss": 3.225507229566574, "l_mix": 1.9848381280899048, "l_a": 2.099908620119095, "l_c": 2.035654753446579, "z_loss": 31.272143840789795, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15191839635372162, "tokens_per_second": 1423789.3292890117, "eta_seconds": 14003.580404663086, "data_wait_seconds": 2.053380012512207e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 34460, "total_steps": 72479, "tokens": 18066964480, "loss": 4.4274842739105225, "l_mix": 2.743634968996048, "l_a": 2.830507308244705, "l_c": 2.782323658466339, "z_loss": 29.035062551498413, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1740170568227768, "tokens_per_second": 1424352.896605056, "eta_seconds": 13994.35878531933, "data_wait_seconds": 1.8358230590820312e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 34470, "total_steps": 72479, "tokens": 18072207360, "loss": 4.580447018146515, "l_mix": 2.839464783668518, "l_a": 2.9257557690143585, "l_c": 2.877518057823181, "z_loss": 28.617467880249023, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15826308727264404, "tokens_per_second": 1290585.524978699, "eta_seconds": 15440.791955518722, "data_wait_seconds": 1.5676021575927734e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 34480, "total_steps": 72479, "tokens": 18077450240, "loss": 4.490115225315094, "l_mix": 2.7816174030303955, "l_a": 2.874510943889618, "l_c": 2.8204814195632935, "z_loss": 29.493131160736084, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16565947234630585, "tokens_per_second": 1444316.952398667, "eta_seconds": 13793.66189596653, "data_wait_seconds": 1.6421079635620117e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 34490, "total_steps": 72479, "tokens": 18082693120, "loss": 4.426998972892761, "l_mix": 2.742907702922821, "l_a": 2.829882472753525, "l_c": 2.7837547063827515, "z_loss": 29.790040016174316, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19885039329528809, "tokens_per_second": 1349148.2360499762, "eta_seconds": 14762.778692364693, "data_wait_seconds": 2.54213809967041e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 34500, "total_steps": 72479, "tokens": 18087936000, "loss": 3.2632409632205963, "l_mix": 2.0077259093523026, "l_a": 2.122056692838669, "l_c": 2.0629934817552567, "z_loss": 30.95159649848938, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19881325960159302, "tokens_per_second": 1419788.0564559612, "eta_seconds": 14024.58195183277, "data_wait_seconds": 3.260374069213867e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.25400448, "gate_a2b/L1_absmax": 0.70415932, "gate_a2b/L1_rms": 0.26807296, "gate_a2b/L3_absmean": 0.26203823, "gate_a2b/L3_absmax": 0.62675315, "gate_a2b/L3_rms": 0.2680113, "gate_a2b/L5_absmean": 0.4105176, "gate_a2b/L5_absmax": 0.65750861, "gate_a2b/L5_rms": 0.41638881, "gate_c2b/L1_absmean": 0.0175445, "gate_c2b/L1_absmax": 0.5785113, "gate_c2b/L1_rms": 0.06661612, "gate_c2b/L3_absmean": 0.21648227, "gate_c2b/L3_absmax": 0.48140517, "gate_c2b/L3_rms": 0.2208128, "gate_c2b/L5_absmean": 0.59478486, "gate_c2b/L5_absmax": 0.84020746, "gate_c2b/L5_rms": 0.60819358, "gate_b2a/L7_absmean": 0.0604476, "gate_b2a/L7_absmax": 0.32933781, "gate_b2a/L7_rms": 0.06588943, "gate_b2a/L11_absmean": 0.09581715, "gate_b2a/L11_absmax": 0.58201939, "gate_b2a/L11_rms": 0.10307599, "gate_b2a/L14_absmean": 0.24534573, "gate_b2a/L14_absmax": 0.73549962, "gate_b2a/L14_rms": 0.25187933, "gate_b2c/L3_absmean": 0.05067573, "gate_b2c/L3_absmax": 0.64084888, "gate_b2c/L3_rms": 0.06351096, "gate_b2c/L5_absmean": 0.05725125, "gate_b2c/L5_absmax": 0.71847332, "gate_b2c/L5_rms": 0.06916084, "gate_b2c/L6_absmean": 0.08347987, "gate_b2c/L6_absmax": 0.49030653, "gate_b2c/L6_rms": 0.09204348}, "streams": {"cos_a2b/round0": 0.384122, "cos_c2b/round0": 0.267541, "cos_b2a/round0": 0.125283, "cos_b2c/round0": 0.102143, "rms_a/round0": 3.100601, "rms_b/round0": 5.744342, "rms_c/round0": 2.711519, "cos_a2b/round1": 0.422325, "cos_c2b/round1": 0.30197, "cos_b2a/round1": 0.231458, "cos_b2c/round1": 0.124027, "rms_a/round1": 5.78715, "rms_b/round1": 12.988184, "rms_c/round1": 5.407261, "cos_a2b/round2": 0.52851, "cos_c2b/round2": 0.512005, "cos_b2a/round2": 0.075418, "cos_b2c/round2": 0.158615, "rms_a/round2": 14.166169, "rms_b/round2": 31.812536, "rms_c/round2": 9.759565}, "alpha_mean": 0.4026126807529735, "alpha_hist": [0.03607, 0.16068, 0.19365, 0.15433, 0.12561, 0.11421, 0.10118, 0.08012, 0.03229, 0.00186], "alpha_by_type": {"special": {"share": 0.00117, "mean": 0.60884, "std": 0.11697}, "whitespace": {"share": 0.03564, "mean": 0.61905, "std": 0.18084}, "digit": {"share": 0.00813, "mean": 0.36235, "std": 0.16224}, "word_start": {"share": 0.57507, "mean": 0.40782, "std": 0.19817}, "word_cont": {"share": 0.25624, "mean": 0.26468, "std": 0.13844}, "punct": {"share": 0.12373, "mean": 0.60242, "std": 0.18076}}, "alpha_p05": 0.11086, "alpha_p50": 0.37068, "alpha_p95": 0.7773}
{"step": 34510, "total_steps": 72479, "tokens": 18093178880, "loss": 3.041456490755081, "l_mix": 1.8819871991872787, "l_a": 1.9526106864213943, "l_c": 1.912286788225174, "z_loss": 35.15805530548096, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.23788511753082275, "tokens_per_second": 1423154.0874196969, "eta_seconds": 13987.727153348922, "data_wait_seconds": 3.248453140258789e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 34520, "total_steps": 72479, "tokens": 18098421760, "loss": 4.742224454879761, "l_mix": 2.940642535686493, "l_a": 3.0239729583263397, "l_c": 2.9812998473644257, "z_loss": 28.92917513847351, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14685533940792084, "tokens_per_second": 1445620.0218109477, "eta_seconds": 13766.721470189093, "data_wait_seconds": 1.6748905181884766e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 34530, "total_steps": 72479, "tokens": 18103664640, "loss": 4.739106357097626, "l_mix": 2.9395741522312164, "l_a": 3.0217941105365753, "l_c": 2.976646363735199, "z_loss": 29.841816663742065, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18111516535282135, "tokens_per_second": 1417502.8849662484, "eta_seconds": 14036.095109939575, "data_wait_seconds": 1.6033649444580078e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 34540, "total_steps": 72479, "tokens": 18108907520, "loss": 4.600062191486359, "l_mix": 2.851817339658737, "l_a": 2.9330779910087585, "l_c": 2.8944046795368195, "z_loss": 29.564037322998047, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18040554225444794, "tokens_per_second": 1330360.5859477657, "eta_seconds": 14951.557225990295, "data_wait_seconds": 2.1576881408691406e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 34550, "total_steps": 72479, "tokens": 18114150400, "loss": 4.696654736995697, "l_mix": 2.912271410226822, "l_a": 2.992329031229019, "l_c": 2.9556152522563934, "z_loss": 29.851704120635986, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14890077710151672, "tokens_per_second": 1454660.8093332273, "eta_seconds": 13670.348045682907, "data_wait_seconds": 2.2172927856445312e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 34560, "total_steps": 72479, "tokens": 18119393280, "loss": 4.496390998363495, "l_mix": 2.785795360803604, "l_a": 2.8776805102825165, "l_c": 2.8243047297000885, "z_loss": 30.41513705253601, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14851221442222595, "tokens_per_second": 1471618.0517321478, "eta_seconds": 13509.263934755325, "data_wait_seconds": 1.5676021575927734e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 34570, "total_steps": 72479, "tokens": 18124636160, "loss": 4.495481073856354, "l_mix": 2.785372406244278, "l_a": 2.8741079568862915, "l_c": 2.826253980398178, "z_loss": 29.4461932182312, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1669658124446869, "tokens_per_second": 1463202.4482212132, "eta_seconds": 13583.37926249504, "data_wait_seconds": 1.7583370208740234e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 34580, "total_steps": 72479, "tokens": 18129879040, "loss": 4.453340172767639, "l_mix": 2.7590565979480743, "l_a": 2.849384367465973, "l_c": 2.798227548599243, "z_loss": 30.499716758728027, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14011238515377045, "tokens_per_second": 1458674.4812619688, "eta_seconds": 13621.95004248619, "data_wait_seconds": 2.47955322265625e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 34590, "total_steps": 72479, "tokens": 18135121920, "loss": 4.576756536960602, "l_mix": 2.835948169231415, "l_a": 2.928693324327469, "l_c": 2.8740011751651764, "z_loss": 30.211971044540405, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1532684713602066, "tokens_per_second": 1371165.9236335375, "eta_seconds": 14487.48666343689, "data_wait_seconds": 2.962350845336914e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 34600, "total_steps": 72479, "tokens": 18140364800, "loss": 4.42876809835434, "l_mix": 2.743479609489441, "l_a": 2.8316222727298737, "l_c": 2.7860057950019836, "z_loss": 30.682700634002686, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1326446235179901, "tokens_per_second": 1362703.416068613, "eta_seconds": 14573.607813572884, "data_wait_seconds": 2.5779008865356445e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.25278383, "gate_a2b/L1_absmax": 0.70494056, "gate_a2b/L1_rms": 0.26693854, "gate_a2b/L3_absmean": 0.26169452, "gate_a2b/L3_absmax": 0.62418813, "gate_a2b/L3_rms": 0.26764694, "gate_a2b/L5_absmean": 0.40939054, "gate_a2b/L5_absmax": 0.65914708, "gate_a2b/L5_rms": 0.41522345, "gate_c2b/L1_absmean": 0.01670718, "gate_c2b/L1_absmax": 0.58020425, "gate_c2b/L1_rms": 0.06641598, "gate_c2b/L3_absmean": 0.21601097, "gate_c2b/L3_absmax": 0.48189658, "gate_c2b/L3_rms": 0.22023278, "gate_c2b/L5_absmean": 0.59465665, "gate_c2b/L5_absmax": 0.84042406, "gate_c2b/L5_rms": 0.60815787, "gate_b2a/L7_absmean": 0.06044311, "gate_b2a/L7_absmax": 0.32896644, "gate_b2a/L7_rms": 0.06586491, "gate_b2a/L11_absmean": 0.09529267, "gate_b2a/L11_absmax": 0.58350569, "gate_b2a/L11_rms": 0.10264347, "gate_b2a/L14_absmean": 0.24491879, "gate_b2a/L14_absmax": 0.7337566, "gate_b2a/L14_rms": 0.25140578, "gate_b2c/L3_absmean": 0.05069095, "gate_b2c/L3_absmax": 0.63865846, "gate_b2c/L3_rms": 0.06339082, "gate_b2c/L5_absmean": 0.05715353, "gate_b2c/L5_absmax": 0.71907586, "gate_b2c/L5_rms": 0.06904759, "gate_b2c/L6_absmean": 0.08300923, "gate_b2c/L6_absmax": 0.48739579, "gate_b2c/L6_rms": 0.09145812}, "streams": {"cos_a2b/round0": 0.392555, "cos_c2b/round0": 0.303274, "cos_b2a/round0": 0.13061, "cos_b2c/round0": 0.124891, "rms_a/round0": 2.999585, "rms_b/round0": 5.705809, "rms_c/round0": 2.675272, "cos_a2b/round1": 0.423274, "cos_c2b/round1": 0.285955, "cos_b2a/round1": 0.239918, "cos_b2c/round1": 0.11533, "rms_a/round1": 5.528885, "rms_b/round1": 12.639795, "rms_c/round1": 5.14731, "cos_a2b/round2": 0.541684, "cos_c2b/round2": 0.529987, "cos_b2a/round2": 0.086246, "cos_b2c/round2": 0.165964, "rms_a/round2": 13.914476, "rms_b/round2": 31.764763, "rms_c/round2": 9.23331}, "alpha_mean": 0.42059946750244137, "alpha_hist": [0.00934, 0.08022, 0.19533, 0.20758, 0.1777, 0.15007, 0.11371, 0.05927, 0.00677, 2e-05], "alpha_by_type": {"special": {"share": 0.00095, "mean": 0.57803, "std": 0.11139}, "whitespace": {"share": 0.02777, "mean": 0.54048, "std": 0.12382}, "digit": {"share": 0.06078, "mean": 0.50344, "std": 0.10013}, "word_start": {"share": 0.5188, "mean": 0.42371, "std": 0.16934}, "word_cont": {"share": 0.27951, "mean": 0.32047, "std": 0.12264}, "punct": {"share": 0.1122, "mean": 0.57977, "std": 0.15471}}, "alpha_p05": 0.16886, "alpha_p50": 0.4031, "alpha_p95": 0.73029}
{"step": 34610, "total_steps": 72479, "tokens": 18145607680, "loss": 3.287453204393387, "l_mix": 2.0228385478258133, "l_a": 2.140332967042923, "l_c": 2.0750489830970764, "z_loss": 31.284828901290894, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15980082750320435, "tokens_per_second": 1401493.9905299777, "eta_seconds": 14166.498326897621, "data_wait_seconds": 1.7136335372924805e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 34620, "total_steps": 72479, "tokens": 18150850560, "loss": 3.2505063712596893, "l_mix": 2.0123900175094604, "l_a": 2.0829129070043564, "l_c": 2.0441415160894394, "z_loss": 32.87913489341736, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2570643126964569, "tokens_per_second": 1444928.601012451, "eta_seconds": 13737.024360990525, "data_wait_seconds": 3.153085708618164e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 34630, "total_steps": 72479, "tokens": 18156093440, "loss": 2.9931246638298035, "l_mix": 1.8519944697618484, "l_a": 1.9224747866392136, "l_c": 1.8812924772500992, "z_loss": 34.81214141845703, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14028166234493256, "tokens_per_second": 1407883.07302732, "eta_seconds": 14094.761768341064, "data_wait_seconds": 2.372264862060547e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08236}}
{"step": 34640, "total_steps": 72479, "tokens": 18161336320, "loss": 4.503980398178101, "l_mix": 2.7907086610794067, "l_a": 2.881846606731415, "l_c": 2.8290587663650513, "z_loss": 29.057892560958862, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14577487111091614, "tokens_per_second": 1384834.8244880543, "eta_seconds": 14325.559468317031, "data_wait_seconds": 1.6182661056518555e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 34650, "total_steps": 72479, "tokens": 18166579200, "loss": 4.472449779510498, "l_mix": 2.7711936235427856, "l_a": 2.859921246767044, "l_c": 2.810932368040085, "z_loss": 29.766939640045166, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1427944153547287, "tokens_per_second": 1420643.8333384693, "eta_seconds": 13960.776294922829, "data_wait_seconds": 2.6553869247436523e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 34660, "total_steps": 72479, "tokens": 18171822080, "loss": 4.528977572917938, "l_mix": 2.8059056103229523, "l_a": 2.896381199359894, "l_c": 2.84719181060791, "z_loss": 30.24588632583618, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11837583780288696, "tokens_per_second": 1432723.0320607317, "eta_seconds": 13839.414477396012, "data_wait_seconds": 1.659989356994629e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 34670, "total_steps": 72479, "tokens": 18177064960, "loss": 4.541079103946686, "l_mix": 2.8139704763889313, "l_a": 2.9038250744342804, "l_c": 2.85320320725441, "z_loss": 29.55038022994995, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1458505541086197, "tokens_per_second": 1439014.4128641163, "eta_seconds": 13775.265080595016, "data_wait_seconds": 1.6987323760986328e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 34680, "total_steps": 72479, "tokens": 18182307840, "loss": 4.629907608032227, "l_mix": 2.869631201028824, "l_a": 2.9606765806674957, "l_c": 2.9069113433361053, "z_loss": 30.22990846633911, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15386873483657837, "tokens_per_second": 1445062.2931259791, "eta_seconds": 13713.98465399742, "data_wait_seconds": 2.3096799850463867e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20764, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 34690, "total_steps": 72479, "tokens": 18187550720, "loss": 4.470266878604889, "l_mix": 2.7693254947662354, "l_a": 2.857915073633194, "l_c": 2.8118889033794403, "z_loss": 30.50295329093933, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.144939124584198, "tokens_per_second": 1435045.5934105283, "eta_seconds": 13806.055586647988, "data_wait_seconds": 2.5957822799682617e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 34700, "total_steps": 72479, "tokens": 18192793600, "loss": 4.433102488517761, "l_mix": 2.7459670305252075, "l_a": 2.8370687663555145, "l_c": 2.7867157459259033, "z_loss": 30.13807463645935, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1694049835205078, "tokens_per_second": 1427617.8189697175, "eta_seconds": 13874.21485555172, "data_wait_seconds": 2.0176172256469727e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.25241706, "gate_a2b/L1_absmax": 0.70518255, "gate_a2b/L1_rms": 0.26636863, "gate_a2b/L3_absmean": 0.26052624, "gate_a2b/L3_absmax": 0.62420338, "gate_a2b/L3_rms": 0.26654536, "gate_a2b/L5_absmean": 0.40780297, "gate_a2b/L5_absmax": 0.6587202, "gate_a2b/L5_rms": 0.41357431, "gate_c2b/L1_absmean": 0.01692579, "gate_c2b/L1_absmax": 0.58147842, "gate_c2b/L1_rms": 0.06661247, "gate_c2b/L3_absmean": 0.21560346, "gate_c2b/L3_absmax": 0.48057756, "gate_c2b/L3_rms": 0.21990776, "gate_c2b/L5_absmean": 0.59485763, "gate_c2b/L5_absmax": 0.83448553, "gate_c2b/L5_rms": 0.60831946, "gate_b2a/L7_absmean": 0.06015162, "gate_b2a/L7_absmax": 0.32852527, "gate_b2a/L7_rms": 0.0655075, "gate_b2a/L11_absmean": 0.09525192, "gate_b2a/L11_absmax": 0.58174753, "gate_b2a/L11_rms": 0.10255028, "gate_b2a/L14_absmean": 0.2437638, "gate_b2a/L14_absmax": 0.7354244, "gate_b2a/L14_rms": 0.25033006, "gate_b2c/L3_absmean": 0.05022362, "gate_b2c/L3_absmax": 0.63737029, "gate_b2c/L3_rms": 0.06295149, "gate_b2c/L5_absmean": 0.05656831, "gate_b2c/L5_absmax": 0.72156823, "gate_b2c/L5_rms": 0.06858727, "gate_b2c/L6_absmean": 0.08212546, "gate_b2c/L6_absmax": 0.48563498, "gate_b2c/L6_rms": 0.0905815}, "streams": {"cos_a2b/round0": 0.385684, "cos_c2b/round0": 0.309238, "cos_b2a/round0": 0.136029, "cos_b2c/round0": 0.128242, "rms_a/round0": 3.015709, "rms_b/round0": 5.780669, "rms_c/round0": 2.708426, "cos_a2b/round1": 0.431358, "cos_c2b/round1": 0.29111, "cos_b2a/round1": 0.245056, "cos_b2c/round1": 0.11093, "rms_a/round1": 5.577353, "rms_b/round1": 12.671252, "rms_c/round1": 5.129856, "cos_a2b/round2": 0.5461, "cos_c2b/round2": 0.531426, "cos_b2a/round2": 0.101879, "cos_b2c/round2": 0.167169, "rms_a/round2": 14.037841, "rms_b/round2": 31.94816, "rms_c/round2": 9.297976}, "alpha_mean": 0.43142892556112145, "alpha_hist": [0.00829, 0.0761, 0.17537, 0.20317, 0.19319, 0.14893, 0.11591, 0.06914, 0.00989, 3e-05], "alpha_by_type": {"special": {"share": 0.00072, "mean": 0.5598, "std": 0.09879}, "whitespace": {"share": 0.01965, "mean": 0.52039, "std": 0.12159}, "digit": {"share": 0.02231, "mean": 0.46996, "std": 0.11822}, "word_start": {"share": 0.59372, "mean": 0.44979, "std": 0.16856}, "word_cont": {"share": 0.25981, "mean": 0.32158, "std": 0.12631}, "punct": {"share": 0.10379, "mean": 0.57535, "std": 0.15874}}, "alpha_p05": 0.16996, "alpha_p50": 0.42155, "alpha_p95": 0.71503}
{"step": 34710, "total_steps": 72479, "tokens": 18198036480, "loss": 4.588103890419006, "l_mix": 2.8438633382320404, "l_a": 2.9346730709075928, "l_c": 2.879461854696274, "z_loss": 28.749770164489746, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1443413496017456, "tokens_per_second": 1335193.5105739953, "eta_seconds": 14830.684327912331, "data_wait_seconds": 1.6570091247558594e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 34720, "total_steps": 72479, "tokens": 18203279360, "loss": 3.2524972558021545, "l_mix": 1.9997650384902954, "l_a": 2.118804484605789, "l_c": 2.056969478726387, "z_loss": 30.62121844291687, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.22368188202381134, "tokens_per_second": 1428013.2159972154, "eta_seconds": 13863.030376911163, "data_wait_seconds": 1.761317253112793e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 34730, "total_steps": 72479, "tokens": 18208522240, "loss": 3.221731513738632, "l_mix": 1.9829138070344925, "l_a": 2.095108062028885, "l_c": 2.034284234046936, "z_loss": 31.72506070137024, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17236679792404175, "tokens_per_second": 1372658.5265268916, "eta_seconds": 14418.260135006905, "data_wait_seconds": 2.1785497665405273e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 34740, "total_steps": 72479, "tokens": 18213765120, "loss": 3.003753960132599, "l_mix": 1.8592074066400528, "l_a": 1.9292238056659698, "l_c": 1.8859313130378723, "z_loss": 34.732253074645996, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.27857980132102966, "tokens_per_second": 1339801.2262627995, "eta_seconds": 14767.940530395508, "data_wait_seconds": 1.6689300537109375e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 34750, "total_steps": 72479, "tokens": 18219008000, "loss": 4.484352111816406, "l_mix": 2.778450757265091, "l_a": 2.867485612630844, "l_c": 2.818851977586746, "z_loss": 29.62905478477478, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12648963928222656, "tokens_per_second": 1473528.4357840219, "eta_seconds": 13424.146743035315, "data_wait_seconds": 1.9162893295288086e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 34760, "total_steps": 72479, "tokens": 18224250880, "loss": 4.506413102149963, "l_mix": 2.792152017354965, "l_a": 2.885120928287506, "l_c": 2.8290824592113495, "z_loss": 29.926326513290405, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16937434673309326, "tokens_per_second": 1467739.9023879855, "eta_seconds": 13473.517371726037, "data_wait_seconds": 1.52587890625e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 34770, "total_steps": 72479, "tokens": 18229493760, "loss": 4.4362998604774475, "l_mix": 2.7484519481658936, "l_a": 2.8393747210502625, "l_c": 2.7867849469184875, "z_loss": 29.570575952529907, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13990071415901184, "tokens_per_second": 1471973.5620225465, "eta_seconds": 13431.203319191933, "data_wait_seconds": 1.537799835205078e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 34780, "total_steps": 72479, "tokens": 18234736640, "loss": 4.479284763336182, "l_mix": 2.775297224521637, "l_a": 2.8667048811912537, "l_c": 2.813253700733185, "z_loss": 30.213459253311157, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14664670825004578, "tokens_per_second": 1441610.3853798783, "eta_seconds": 13710.454303359986, "data_wait_seconds": 1.659989356994629e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 34790, "total_steps": 72479, "tokens": 18239979520, "loss": 4.5253055691719055, "l_mix": 2.8030313551425934, "l_a": 2.8923806250095367, "l_c": 2.8485333919525146, "z_loss": 30.419232845306396, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.21329376101493835, "tokens_per_second": 1465899.697625221, "eta_seconds": 13479.701553940773, "data_wait_seconds": 3.007054328918457e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 34800, "total_steps": 72479, "tokens": 18245222400, "loss": 4.715498447418213, "l_mix": 2.9245679676532745, "l_a": 3.005220949649811, "l_c": 2.9645473957061768, "z_loss": 30.334978818893433, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14030152559280396, "tokens_per_second": 1335850.9040632278, "eta_seconds": 14788.063167762757, "data_wait_seconds": 2.759695053100586e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.25101936, "gate_a2b/L1_absmax": 0.70744497, "gate_a2b/L1_rms": 0.26502433, "gate_a2b/L3_absmean": 0.25918975, "gate_a2b/L3_absmax": 0.62660104, "gate_a2b/L3_rms": 0.26527935, "gate_a2b/L5_absmean": 0.40664932, "gate_a2b/L5_absmax": 0.66152185, "gate_a2b/L5_rms": 0.412453, "gate_c2b/L1_absmean": 0.01792355, "gate_c2b/L1_absmax": 0.58304673, "gate_c2b/L1_rms": 0.06702591, "gate_c2b/L3_absmean": 0.21567379, "gate_c2b/L3_absmax": 0.47800386, "gate_c2b/L3_rms": 0.21996018, "gate_c2b/L5_absmean": 0.59370828, "gate_c2b/L5_absmax": 0.84290588, "gate_c2b/L5_rms": 0.60707647, "gate_b2a/L7_absmean": 0.05960034, "gate_b2a/L7_absmax": 0.32957509, "gate_b2a/L7_rms": 0.06517126, "gate_b2a/L11_absmean": 0.09496374, "gate_b2a/L11_absmax": 0.5851438, "gate_b2a/L11_rms": 0.10237181, "gate_b2a/L14_absmean": 0.24302861, "gate_b2a/L14_absmax": 0.73454303, "gate_b2a/L14_rms": 0.24962963, "gate_b2c/L3_absmean": 0.05055217, "gate_b2c/L3_absmax": 0.63544285, "gate_b2c/L3_rms": 0.06305078, "gate_b2c/L5_absmean": 0.05649891, "gate_b2c/L5_absmax": 0.71319604, "gate_b2c/L5_rms": 0.06829037, "gate_b2c/L6_absmean": 0.0817322, "gate_b2c/L6_absmax": 0.4881365, "gate_b2c/L6_rms": 0.09022766}, "streams": {"cos_a2b/round0": 0.387378, "cos_c2b/round0": 0.305522, "cos_b2a/round0": 0.137391, "cos_b2c/round0": 0.130471, "rms_a/round0": 3.047647, "rms_b/round0": 5.842633, "rms_c/round0": 2.688527, "cos_a2b/round1": 0.42704, "cos_c2b/round1": 0.298654, "cos_b2a/round1": 0.244096, "cos_b2c/round1": 0.121151, "rms_a/round1": 5.593318, "rms_b/round1": 12.668276, "rms_c/round1": 5.221505, "cos_a2b/round2": 0.569154, "cos_c2b/round2": 0.515591, "cos_b2a/round2": 0.107323, "cos_b2c/round2": 0.173425, "rms_a/round2": 14.153185, "rms_b/round2": 31.167381, "rms_c/round2": 9.276331}, "alpha_mean": 0.45524826654047956, "alpha_hist": [0.00999, 0.05302, 0.14404, 0.18204, 0.19136, 0.20027, 0.14514, 0.06627, 0.0078, 6e-05], "alpha_by_type": {"special": {"share": 0.00081, "mean": 0.54257, "std": 0.16724}, "whitespace": {"share": 0.05908, "mean": 0.55345, "std": 0.11114}, "digit": {"share": 0.03519, "mean": 0.4634, "std": 0.12911}, "word_start": {"share": 0.55524, "mean": 0.47496, "std": 0.15585}, "word_cont": {"share": 0.24716, "mean": 0.33332, "std": 0.137}, "punct": {"share": 0.10252, "mean": 0.58238, "std": 0.15787}}, "alpha_p05": 0.17498, "alpha_p50": 0.45036, "alpha_p95": 0.70962}
{"step": 34810, "total_steps": 72479, "tokens": 18250465280, "loss": 4.7578359842300415, "l_mix": 2.950646996498108, "l_a": 3.0313269793987274, "l_c": 2.9926360845565796, "z_loss": 29.098082542419434, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15933652222156525, "tokens_per_second": 1450181.3601449402, "eta_seconds": 13618.575727677344, "data_wait_seconds": 1.84476375579834e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 34820, "total_steps": 72479, "tokens": 18255708160, "loss": 4.788706302642822, "l_mix": 2.9702321887016296, "l_a": 3.0516310036182404, "l_c": 3.009948968887329, "z_loss": 29.03292489051819, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1359284222126007, "tokens_per_second": 1454530.6268405092, "eta_seconds": 13574.249608540535, "data_wait_seconds": 2.0116567611694336e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 34830, "total_steps": 72479, "tokens": 18260951040, "loss": 4.870248854160309, "l_mix": 3.020884692668915, "l_a": 3.101118952035904, "l_c": 3.0634281635284424, "z_loss": 29.28842282295227, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.144583597779274, "tokens_per_second": 1449906.6541321508, "eta_seconds": 13613.923941755294, "data_wait_seconds": 3.2395124435424805e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 34840, "total_steps": 72479, "tokens": 18266193920, "loss": 4.786327958106995, "l_mix": 2.968621104955673, "l_a": 3.0498115718364716, "l_c": 3.009211093187332, "z_loss": 29.437729358673096, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11626586318016052, "tokens_per_second": 1470548.2235089291, "eta_seconds": 13419.264813303946, "data_wait_seconds": 1.6391277313232422e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 34850, "total_steps": 72479, "tokens": 18271436800, "loss": 3.196324735879898, "l_mix": 1.9791156202554703, "l_a": 2.0488406121730804, "l_c": 2.0085227340459824, "z_loss": 35.754260540008545, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19748911261558533, "tokens_per_second": 1402035.394389852, "eta_seconds": 14071.28039059639, "data_wait_seconds": 1.990795135498047e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 34860, "total_steps": 72479, "tokens": 18276679680, "loss": 4.5559558272361755, "l_mix": 2.8229030072689056, "l_a": 2.9127404987812042, "l_c": 2.8641018867492676, "z_loss": 30.623474597930908, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13928724825382233, "tokens_per_second": 1438567.0686055126, "eta_seconds": 13710.30291352272, "data_wait_seconds": 1.755356788635254e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 34870, "total_steps": 72479, "tokens": 18281922560, "loss": 4.576856732368469, "l_mix": 2.8368882536888123, "l_a": 2.923651695251465, "l_c": 2.876242995262146, "z_loss": 30.3658185005188, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13015951216220856, "tokens_per_second": 1448619.6102496888, "eta_seconds": 13611.542500519754, "data_wait_seconds": 1.850724220275879e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 34880, "total_steps": 72479, "tokens": 18287165440, "loss": 4.4465292096138, "l_mix": 2.754689007997513, "l_a": 2.8455671668052673, "l_c": 2.793899714946747, "z_loss": 30.349835634231567, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16188940405845642, "tokens_per_second": 1460349.930112469, "eta_seconds": 13498.617081785202, "data_wait_seconds": 1.5676021575927734e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 34890, "total_steps": 72479, "tokens": 18292408320, "loss": 4.5328468680381775, "l_mix": 2.809269607067108, "l_a": 2.8945700228214264, "l_c": 2.8506870567798615, "z_loss": 30.774691104888916, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16014371812343597, "tokens_per_second": 1370994.0964658402, "eta_seconds": 14374.578039979935, "data_wait_seconds": 2.2739171981811523e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 34900, "total_steps": 72479, "tokens": 18297651200, "loss": 4.570420682430267, "l_mix": 2.831853151321411, "l_a": 2.925531566143036, "l_c": 2.8696933686733246, "z_loss": 30.24700117111206, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15178799629211426, "tokens_per_second": 1466221.9477515768, "eta_seconds": 13437.405422973632, "data_wait_seconds": 2.5600194931030273e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.25031838, "gate_a2b/L1_absmax": 0.70543903, "gate_a2b/L1_rms": 0.26423547, "gate_a2b/L3_absmean": 0.25847545, "gate_a2b/L3_absmax": 0.62682766, "gate_a2b/L3_rms": 0.26453426, "gate_a2b/L5_absmean": 0.40451488, "gate_a2b/L5_absmax": 0.66534305, "gate_a2b/L5_rms": 0.41039237, "gate_c2b/L1_absmean": 0.01756909, "gate_c2b/L1_absmax": 0.58324945, "gate_c2b/L1_rms": 0.06714337, "gate_c2b/L3_absmean": 0.21549678, "gate_c2b/L3_absmax": 0.47368017, "gate_c2b/L3_rms": 0.21976358, "gate_c2b/L5_absmean": 0.59371632, "gate_c2b/L5_absmax": 0.83525407, "gate_c2b/L5_rms": 0.60711718, "gate_b2a/L7_absmean": 0.0596142, "gate_b2a/L7_absmax": 0.32796469, "gate_b2a/L7_rms": 0.06504532, "gate_b2a/L11_absmean": 0.0945235, "gate_b2a/L11_absmax": 0.58471984, "gate_b2a/L11_rms": 0.10197272, "gate_b2a/L14_absmean": 0.24277937, "gate_b2a/L14_absmax": 0.73605496, "gate_b2a/L14_rms": 0.24934736, "gate_b2c/L3_absmean": 0.04997997, "gate_b2c/L3_absmax": 0.63260937, "gate_b2c/L3_rms": 0.06241043, "gate_b2c/L5_absmean": 0.05577832, "gate_b2c/L5_absmax": 0.71493822, "gate_b2c/L5_rms": 0.06772485, "gate_b2c/L6_absmean": 0.08136426, "gate_b2c/L6_absmax": 0.4889605, "gate_b2c/L6_rms": 0.08983634}, "streams": {"cos_a2b/round0": 0.38371, "cos_c2b/round0": 0.302465, "cos_b2a/round0": 0.135929, "cos_b2c/round0": 0.123197, "rms_a/round0": 3.069061, "rms_b/round0": 5.888237, "rms_c/round0": 2.749488, "cos_a2b/round1": 0.422951, "cos_c2b/round1": 0.293067, "cos_b2a/round1": 0.245862, "cos_b2c/round1": 0.113213, "rms_a/round1": 5.703419, "rms_b/round1": 12.980003, "rms_c/round1": 5.28301, "cos_a2b/round2": 0.542882, "cos_c2b/round2": 0.528424, "cos_b2a/round2": 0.103337, "cos_b2c/round2": 0.167698, "rms_a/round2": 14.404593, "rms_b/round2": 32.440727, "rms_c/round2": 9.513096}, "alpha_mean": 0.4345231216341574, "alpha_hist": [0.00711, 0.06451, 0.16702, 0.21158, 0.20227, 0.166, 0.10883, 0.06169, 0.01064, 0.00035], "alpha_by_type": {"special": {"share": 0.00111, "mean": 0.56125, "std": 0.09183}, "whitespace": {"share": 0.02367, "mean": 0.5365, "std": 0.11224}, "digit": {"share": 0.03456, "mean": 0.46309, "std": 0.11656}, "word_start": {"share": 0.5331, "mean": 0.45013, "std": 0.16527}, "word_cont": {"share": 0.25174, "mean": 0.33215, "std": 0.12596}, "punct": {"share": 0.15582, "mean": 0.5238, "std": 0.16733}}, "alpha_p05": 0.17498, "alpha_p50": 0.43302, "alpha_p95": 0.7454}
{"step": 34910, "total_steps": 72479, "tokens": 18302894080, "loss": 3.060509145259857, "l_mix": 1.894107162952423, "l_a": 1.9611623883247375, "l_c": 1.926844209432602, "z_loss": 35.28893423080444, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.26882442831993103, "tokens_per_second": 1462427.6811236434, "eta_seconds": 13468.683700561523, "data_wait_seconds": 2.4974346160888672e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 34920, "total_steps": 72479, "tokens": 18308136960, "loss": 4.701332747936249, "l_mix": 2.915799468755722, "l_a": 2.99902206659317, "l_c": 2.9527553021907806, "z_loss": 29.356056213378906, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1390528827905655, "tokens_per_second": 1391683.1990123268, "eta_seconds": 14149.580167365073, "data_wait_seconds": 2.720952033996582e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 34930, "total_steps": 72479, "tokens": 18313379840, "loss": 4.7557127475738525, "l_mix": 2.9495081305503845, "l_a": 3.0331977009773254, "l_c": 2.987484484910965, "z_loss": 29.559249877929688, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1348884105682373, "tokens_per_second": 1422132.8453026847, "eta_seconds": 13842.933293485641, "data_wait_seconds": 2.6404857635498047e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 34940, "total_steps": 72479, "tokens": 18318622720, "loss": 4.859826147556305, "l_mix": 3.0153911113739014, "l_a": 3.0897217094898224, "l_c": 3.0583949387073517, "z_loss": 28.874547958374023, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2147446572780609, "tokens_per_second": 1417572.8801720717, "eta_seconds": 13883.763937139513, "data_wait_seconds": 1.7821788787841797e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 34950, "total_steps": 72479, "tokens": 18323865600, "loss": 4.763198912143707, "l_mix": 2.953292578458786, "l_a": 3.038389414548874, "l_c": 2.994631767272949, "z_loss": 30.445971727371216, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16463515162467957, "tokens_per_second": 1449829.0324514327, "eta_seconds": 13571.258342599867, "data_wait_seconds": 3.55839729309082e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 34960, "total_steps": 72479, "tokens": 18329108480, "loss": 3.0719969272613525, "l_mix": 1.9013782143592834, "l_a": 1.970702439546585, "l_c": 1.9313598424196243, "z_loss": 35.559035778045654, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16550108790397644, "tokens_per_second": 1465985.2066737288, "eta_seconds": 13418.117305994034, "data_wait_seconds": 4.318356513977051e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 34970, "total_steps": 72479, "tokens": 18334351360, "loss": 4.4711437821388245, "l_mix": 2.7704178392887115, "l_a": 2.8609765768051147, "l_c": 2.808109372854233, "z_loss": 29.85265588760376, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13578535616397858, "tokens_per_second": 1380114.3352397203, "eta_seconds": 14249.195222353936, "data_wait_seconds": 1.96993350982666e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 34980, "total_steps": 72479, "tokens": 18339594240, "loss": 4.4257179498672485, "l_mix": 2.741772621870041, "l_a": 2.8329328298568726, "l_c": 2.780217945575714, "z_loss": 30.527406930923462, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14028272032737732, "tokens_per_second": 1466935.4678310591, "eta_seconds": 13402.277157473563, "data_wait_seconds": 2.6285648345947266e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 34990, "total_steps": 72479, "tokens": 18344837120, "loss": 4.587827444076538, "l_mix": 2.843270719051361, "l_a": 2.9333406686782837, "l_c": 2.8818479776382446, "z_loss": 29.80751061439514, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11237439513206482, "tokens_per_second": 1294905.1031570325, "eta_seconds": 15178.743819975853, "data_wait_seconds": 1.9878149032592773e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 35000, "total_steps": 72479, "tokens": 18350080000, "loss": 4.507297217845917, "l_mix": 2.7928492426872253, "l_a": 2.8838014006614685, "l_c": 2.8310248851776123, "z_loss": 29.5430850982666, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1425820142030716, "tokens_per_second": 1398529.3754535315, "eta_seconds": 14050.323358869553, "data_wait_seconds": 1.811981201171875e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.24950838, "gate_a2b/L1_absmax": 0.70196342, "gate_a2b/L1_rms": 0.26339978, "gate_a2b/L3_absmean": 0.2567195, "gate_a2b/L3_absmax": 0.62390906, "gate_a2b/L3_rms": 0.26290482, "gate_a2b/L5_absmean": 0.40264568, "gate_a2b/L5_absmax": 0.66854095, "gate_a2b/L5_rms": 0.40852734, "gate_c2b/L1_absmean": 0.01838605, "gate_c2b/L1_absmax": 0.58743471, "gate_c2b/L1_rms": 0.06756057, "gate_c2b/L3_absmean": 0.21500354, "gate_c2b/L3_absmax": 0.4844586, "gate_c2b/L3_rms": 0.21929631, "gate_c2b/L5_absmean": 0.59415448, "gate_c2b/L5_absmax": 0.8257938, "gate_c2b/L5_rms": 0.60753793, "gate_b2a/L7_absmean": 0.05948592, "gate_b2a/L7_absmax": 0.32838258, "gate_b2a/L7_rms": 0.06504133, "gate_b2a/L11_absmean": 0.09515398, "gate_b2a/L11_absmax": 0.58579361, "gate_b2a/L11_rms": 0.10263209, "gate_b2a/L14_absmean": 0.24255753, "gate_b2a/L14_absmax": 0.74695283, "gate_b2a/L14_rms": 0.24928363, "gate_b2c/L3_absmean": 0.04944359, "gate_b2c/L3_absmax": 0.63417041, "gate_b2c/L3_rms": 0.0619891, "gate_b2c/L5_absmean": 0.05522239, "gate_b2c/L5_absmax": 0.71283543, "gate_b2c/L5_rms": 0.06720065, "gate_b2c/L6_absmean": 0.08086961, "gate_b2c/L6_absmax": 0.49425215, "gate_b2c/L6_rms": 0.08946969}, "streams": {"cos_a2b/round0": 0.387707, "cos_c2b/round0": 0.298816, "cos_b2a/round0": 0.139156, "cos_b2c/round0": 0.128865, "rms_a/round0": 3.142547, "rms_b/round0": 6.054294, "rms_c/round0": 2.755405, "cos_a2b/round1": 0.429056, "cos_c2b/round1": 0.294743, "cos_b2a/round1": 0.244046, "cos_b2c/round1": 0.122126, "rms_a/round1": 5.859905, "rms_b/round1": 13.231724, "rms_c/round1": 5.339136, "cos_a2b/round2": 0.560378, "cos_c2b/round2": 0.515953, "cos_b2a/round2": 0.105165, "cos_b2c/round2": 0.177247, "rms_a/round2": 14.850375, "rms_b/round2": 32.842312, "rms_c/round2": 9.639655}, "alpha_mean": 0.4477294444990605, "alpha_hist": [0.00848, 0.06253, 0.16098, 0.19032, 0.18646, 0.16997, 0.13393, 0.07639, 0.01091, 3e-05], "alpha_by_type": {"special": {"share": 0.00134, "mean": 0.58098, "std": 0.10262}, "whitespace": {"share": 0.02541, "mean": 0.53589, "std": 0.11642}, "digit": {"share": 0.0262, "mean": 0.44098, "std": 0.13428}, "word_start": {"share": 0.58499, "mean": 0.46927, "std": 0.1646}, "word_cont": {"share": 0.26025, "mean": 0.33378, "std": 0.13627}, "punct": {"share": 0.10181, "mean": 0.59323, "std": 0.16155}}, "alpha_p05": 0.17781, "alpha_p50": 0.43446, "alpha_p95": 0.73905}
{"step": 35010, "total_steps": 72479, "tokens": 18355322880, "loss": 4.505956709384918, "l_mix": 2.7925444543361664, "l_a": 2.882099002599716, "l_c": 2.8292749524116516, "z_loss": 30.362923622131348, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19667257368564606, "tokens_per_second": 1412186.9841354447, "eta_seconds": 13910.726619553565, "data_wait_seconds": 1.7702579498291016e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 35020, "total_steps": 72479, "tokens": 18360565760, "loss": 3.1820488274097443, "l_mix": 1.9680891185998917, "l_a": 2.0450054705142975, "l_c": 2.0015268474817276, "z_loss": 37.34762907028198, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14823167026042938, "tokens_per_second": 1167562.35763124, "eta_seconds": 16820.77540752888, "data_wait_seconds": 4.026293754577637e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 35030, "total_steps": 72479, "tokens": 18365808640, "loss": 3.0146613717079163, "l_mix": 1.8669523447752, "l_a": 1.9298870712518692, "l_c": 1.8958094418048859, "z_loss": 34.98369264602661, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16067753732204437, "tokens_per_second": 1444755.1609715512, "eta_seconds": 13589.888337063789, "data_wait_seconds": 1.7881393432617188e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08236}}
{"step": 35040, "total_steps": 72479, "tokens": 18371051520, "loss": 4.823422729969025, "l_mix": 2.9918483197689056, "l_a": 3.07022687792778, "l_c": 3.0350206792354584, "z_loss": 30.19291615486145, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15097424387931824, "tokens_per_second": 1322145.4529511791, "eta_seconds": 14846.18684592247, "data_wait_seconds": 1.704692840576172e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 35050, "total_steps": 72479, "tokens": 18376294400, "loss": 4.804489612579346, "l_mix": 2.9798144102096558, "l_a": 3.0590239465236664, "l_c": 3.0232268273830414, "z_loss": 29.56022834777832, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13885226845741272, "tokens_per_second": 1290807.793856125, "eta_seconds": 15202.554280662536, "data_wait_seconds": 4.1604042053222656e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 35060, "total_steps": 72479, "tokens": 18381537280, "loss": 4.602520406246185, "l_mix": 2.853881388902664, "l_a": 2.9369365572929382, "l_c": 2.891860395669937, "z_loss": 30.341628074645996, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1343212127685547, "tokens_per_second": 1230687.1222082335, "eta_seconds": 15940.958768463135, "data_wait_seconds": 1.6570091247558594e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 35070, "total_steps": 72479, "tokens": 18386780160, "loss": 4.823982834815979, "l_mix": 2.9909910559654236, "l_a": 3.073665529489517, "l_c": 3.0363068878650665, "z_loss": 30.738773345947266, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17932114005088806, "tokens_per_second": 1182766.1846862067, "eta_seconds": 16582.38969454765, "data_wait_seconds": 2.2351741790771484e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 35080, "total_steps": 72479, "tokens": 18392023040, "loss": 4.468134582042694, "l_mix": 2.767823964357376, "l_a": 2.863006502389908, "l_c": 2.804695248603821, "z_loss": 30.116751432418823, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11293840408325195, "tokens_per_second": 1231787.4977591115, "eta_seconds": 15918.20581688881, "data_wait_seconds": 2.4437904357910156e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 35090, "total_steps": 72479, "tokens": 18397265920, "loss": 4.450674653053284, "l_mix": 2.757514715194702, "l_a": 2.843673199415207, "l_c": 2.800193041563034, "z_loss": 30.55388855934143, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12208756059408188, "tokens_per_second": 1387077.2167136054, "eta_seconds": 14132.30914313793, "data_wait_seconds": 1.8537044525146484e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 35100, "total_steps": 72479, "tokens": 18402508800, "loss": 4.585000514984131, "l_mix": 2.8411213755607605, "l_a": 2.93269282579422, "l_c": 2.880237340927124, "z_loss": 30.293559789657593, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13082362711429596, "tokens_per_second": 1381936.8098649764, "eta_seconds": 14181.083398389816, "data_wait_seconds": 1.996755599975586e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.24880247, "gate_a2b/L1_absmax": 0.70036274, "gate_a2b/L1_rms": 0.26272634, "gate_a2b/L3_absmean": 0.25592843, "gate_a2b/L3_absmax": 0.61287403, "gate_a2b/L3_rms": 0.26209334, "gate_a2b/L5_absmean": 0.40100023, "gate_a2b/L5_absmax": 0.66867471, "gate_a2b/L5_rms": 0.4068616, "gate_c2b/L1_absmean": 0.01794337, "gate_c2b/L1_absmax": 0.58479619, "gate_c2b/L1_rms": 0.06743604, "gate_c2b/L3_absmean": 0.21465409, "gate_c2b/L3_absmax": 0.48049474, "gate_c2b/L3_rms": 0.21902655, "gate_c2b/L5_absmean": 0.59341604, "gate_c2b/L5_absmax": 0.83214384, "gate_c2b/L5_rms": 0.60677594, "gate_b2a/L7_absmean": 0.05911119, "gate_b2a/L7_absmax": 0.33759096, "gate_b2a/L7_rms": 0.06484124, "gate_b2a/L11_absmean": 0.09458927, "gate_b2a/L11_absmax": 0.59414136, "gate_b2a/L11_rms": 0.10231688, "gate_b2a/L14_absmean": 0.24141151, "gate_b2a/L14_absmax": 0.74130118, "gate_b2a/L14_rms": 0.24820881, "gate_b2c/L3_absmean": 0.04988562, "gate_b2c/L3_absmax": 0.64524758, "gate_b2c/L3_rms": 0.06271362, "gate_b2c/L5_absmean": 0.05531842, "gate_b2c/L5_absmax": 0.71672601, "gate_b2c/L5_rms": 0.06742717, "gate_b2c/L6_absmean": 0.07971378, "gate_b2c/L6_absmax": 0.47744066, "gate_b2c/L6_rms": 0.08822812}, "streams": {"cos_a2b/round0": 0.391322, "cos_c2b/round0": 0.303568, "cos_b2a/round0": 0.130931, "cos_b2c/round0": 0.130036, "rms_a/round0": 3.187704, "rms_b/round0": 6.025362, "rms_c/round0": 2.751043, "cos_a2b/round1": 0.423639, "cos_c2b/round1": 0.289741, "cos_b2a/round1": 0.241735, "cos_b2c/round1": 0.117565, "rms_a/round1": 5.817224, "rms_b/round1": 13.237524, "rms_c/round1": 5.297903, "cos_a2b/round2": 0.565701, "cos_c2b/round2": 0.514657, "cos_b2a/round2": 0.100902, "cos_b2c/round2": 0.170491, "rms_a/round2": 14.882475, "rms_b/round2": 33.094891, "rms_c/round2": 9.403282}, "alpha_mean": 0.44760596841709344, "alpha_hist": [0.00673, 0.05417, 0.15646, 0.20761, 0.19351, 0.17389, 0.12389, 0.07097, 0.01189, 0.00089], "alpha_by_type": {"special": {"share": 0.00082, "mean": 0.59207, "std": 0.11126}, "whitespace": {"share": 0.03224, "mean": 0.53919, "std": 0.12517}, "digit": {"share": 0.05524, "mean": 0.48673, "std": 0.09761}, "word_start": {"share": 0.54976, "mean": 0.46169, "std": 0.16097}, "word_cont": {"share": 0.25449, "mean": 0.33084, "std": 0.12715}, "punct": {"share": 0.10745, "mean": 0.6034, "std": 0.15687}}, "alpha_p05": 0.20562, "alpha_p50": 0.4552, "alpha_p95": 0.74651}
{"step": 35110, "total_steps": 72479, "tokens": 18407751680, "loss": 4.492719233036041, "l_mix": 2.783711701631546, "l_a": 2.871248960494995, "l_c": 2.825442463159561, "z_loss": 30.04254937171936, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11774806678295135, "tokens_per_second": 1448675.9153568498, "eta_seconds": 13524.155447268486, "data_wait_seconds": 1.5795230865478516e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 35120, "total_steps": 72479, "tokens": 18412994560, "loss": 4.5172770619392395, "l_mix": 2.7995503544807434, "l_a": 2.8848613500595093, "l_c": 2.8408939838409424, "z_loss": 30.537622928619385, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1279607117176056, "tokens_per_second": 1457889.905715498, "eta_seconds": 13435.085403370857, "data_wait_seconds": 2.1755695343017578e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 35130, "total_steps": 72479, "tokens": 18418237440, "loss": 3.11957511305809, "l_mix": 1.9312735348939896, "l_a": 2.0016229152679443, "l_c": 1.9593821167945862, "z_loss": 35.97118854522705, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17131559550762177, "tokens_per_second": 1390772.4869228292, "eta_seconds": 14079.680678272247, "data_wait_seconds": 2.893805503845215e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 35140, "total_steps": 72479, "tokens": 18423480320, "loss": 3.44612917304039, "l_mix": 2.134472042322159, "l_a": 2.2127323746681213, "l_c": 2.1594578623771667, "z_loss": 37.64437913894653, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.405131995677948, "tokens_per_second": 1445952.9071151111, "eta_seconds": 13538.74634206295, "data_wait_seconds": 1.7970800399780273e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08236}}
{"step": 35150, "total_steps": 72479, "tokens": 18428723200, "loss": 4.315708041191101, "l_mix": 2.674667865037918, "l_a": 2.7573322057724, "l_c": 2.712801516056061, "z_loss": 30.660993099212646, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14522434771060944, "tokens_per_second": 1366984.9191591446, "eta_seconds": 14317.017311382295, "data_wait_seconds": 2.2649765014648438e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 35160, "total_steps": 72479, "tokens": 18433966080, "loss": 4.546639382839203, "l_mix": 2.817749947309494, "l_a": 2.9070850610733032, "l_c": 2.855879455804825, "z_loss": 29.892219305038452, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16234995424747467, "tokens_per_second": 1454581.8118553052, "eta_seconds": 13451.222689938544, "data_wait_seconds": 2.4586915969848633e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 35170, "total_steps": 72479, "tokens": 18439208960, "loss": 4.471477925777435, "l_mix": 2.769595116376877, "l_a": 2.8626155853271484, "l_c": 2.8103269040584564, "z_loss": 30.961233377456665, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12664654850959778, "tokens_per_second": 1358611.4702368383, "eta_seconds": 14397.538531446457, "data_wait_seconds": 1.6301870346069336e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 35180, "total_steps": 72479, "tokens": 18444451840, "loss": 4.539017140865326, "l_mix": 2.812622606754303, "l_a": 2.9010069966316223, "l_c": 2.8536414206027985, "z_loss": 31.16643238067627, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1418023705482483, "tokens_per_second": 1467729.616195768, "eta_seconds": 13323.58351035118, "data_wait_seconds": 1.9669532775878906e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 35190, "total_steps": 72479, "tokens": 18449694720, "loss": 4.43420147895813, "l_mix": 2.7470873296260834, "l_a": 2.834658533334732, "l_c": 2.7890551388263702, "z_loss": 30.874749422073364, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12274406105279922, "tokens_per_second": 1474506.3996535249, "eta_seconds": 13258.793069052696, "data_wait_seconds": 3.102421760559082e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 35200, "total_steps": 72479, "tokens": 18454937600, "loss": 4.4772138595581055, "l_mix": 2.773974746465683, "l_a": 2.864787995815277, "l_c": 2.8126755356788635, "z_loss": 30.367090463638306, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1251092255115509, "tokens_per_second": 1446497.0510927087, "eta_seconds": 13511.906116390228, "data_wait_seconds": 1.5169382095336914e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.2474788, "gate_a2b/L1_absmax": 0.70468706, "gate_a2b/L1_rms": 0.26141867, "gate_a2b/L3_absmean": 0.25431761, "gate_a2b/L3_absmax": 0.61284262, "gate_a2b/L3_rms": 0.26042783, "gate_a2b/L5_absmean": 0.39894176, "gate_a2b/L5_absmax": 0.66644448, "gate_a2b/L5_rms": 0.4048216, "gate_c2b/L1_absmean": 0.01740717, "gate_c2b/L1_absmax": 0.58767951, "gate_c2b/L1_rms": 0.06739042, "gate_c2b/L3_absmean": 0.21503861, "gate_c2b/L3_absmax": 0.47358915, "gate_c2b/L3_rms": 0.21936904, "gate_c2b/L5_absmean": 0.59407026, "gate_c2b/L5_absmax": 0.83110243, "gate_c2b/L5_rms": 0.60745704, "gate_b2a/L7_absmean": 0.05916883, "gate_b2a/L7_absmax": 0.33623219, "gate_b2a/L7_rms": 0.06478959, "gate_b2a/L11_absmean": 0.09392704, "gate_b2a/L11_absmax": 0.59037161, "gate_b2a/L11_rms": 0.10161897, "gate_b2a/L14_absmean": 0.24082023, "gate_b2a/L14_absmax": 0.73918796, "gate_b2a/L14_rms": 0.24766269, "gate_b2c/L3_absmean": 0.0488265, "gate_b2c/L3_absmax": 0.64229476, "gate_b2c/L3_rms": 0.06160352, "gate_b2c/L5_absmean": 0.05464093, "gate_b2c/L5_absmax": 0.70405084, "gate_b2c/L5_rms": 0.06646682, "gate_b2c/L6_absmean": 0.07956046, "gate_b2c/L6_absmax": 0.47973692, "gate_b2c/L6_rms": 0.08798657}, "streams": {"cos_a2b/round0": 0.392427, "cos_c2b/round0": 0.307116, "cos_b2a/round0": 0.136618, "cos_b2c/round0": 0.130847, "rms_a/round0": 3.195591, "rms_b/round0": 6.127984, "rms_c/round0": 2.787597, "cos_a2b/round1": 0.428662, "cos_c2b/round1": 0.294182, "cos_b2a/round1": 0.241488, "cos_b2c/round1": 0.11772, "rms_a/round1": 5.958971, "rms_b/round1": 13.472919, "rms_c/round1": 5.387401, "cos_a2b/round2": 0.558874, "cos_c2b/round2": 0.512615, "cos_b2a/round2": 0.099855, "cos_b2c/round2": 0.168769, "rms_a/round2": 15.113155, "rms_b/round2": 33.70269, "rms_c/round2": 9.638983}, "alpha_mean": 0.44208174648120746, "alpha_hist": [0.01193, 0.07396, 0.16902, 0.18906, 0.17046, 0.15938, 0.1339, 0.07959, 0.01245, 0.00026], "alpha_by_type": {"special": {"share": 0.00139, "mean": 0.57213, "std": 0.09578}, "whitespace": {"share": 0.03119, "mean": 0.56489, "std": 0.12603}, "digit": {"share": 0.03754, "mean": 0.50186, "std": 0.12681}, "word_start": {"share": 0.56702, "mean": 0.45978, "std": 0.17262}, "word_cont": {"share": 0.25725, "mean": 0.31658, "std": 0.13296}, "punct": {"share": 0.10562, "mean": 0.59354, "std": 0.15778}}, "alpha_p05": 0.16318, "alpha_p50": 0.42465, "alpha_p95": 0.73488}
{"step": 35210, "total_steps": 72479, "tokens": 18460180480, "loss": 4.637537360191345, "l_mix": 2.8743403553962708, "l_a": 2.95985946059227, "l_c": 2.9174633622169495, "z_loss": 30.687273263931274, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12549719214439392, "tokens_per_second": 1455825.0197282396, "eta_seconds": 13421.72940237522, "data_wait_seconds": 1.7583370208740234e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 35220, "total_steps": 72479, "tokens": 18465423360, "loss": 4.484372854232788, "l_mix": 2.778572916984558, "l_a": 2.8686060905456543, "l_c": 2.817393720149994, "z_loss": 31.149344205856323, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12425907701253891, "tokens_per_second": 1388199.2541377086, "eta_seconds": 14071.788710284232, "data_wait_seconds": 1.671910285949707e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 35230, "total_steps": 72479, "tokens": 18470666240, "loss": 4.568901538848877, "l_mix": 2.8315859138965607, "l_a": 2.9183561503887177, "l_c": 2.872695952653885, "z_loss": 30.346562385559082, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14661945402622223, "tokens_per_second": 1327969.0486552084, "eta_seconds": 14706.068437194825, "data_wait_seconds": 3.287196159362793e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 35240, "total_steps": 72479, "tokens": 18475909120, "loss": 2.879498451948166, "l_mix": 1.7815430909395218, "l_a": 1.8496577888727188, "l_c": 1.810193344950676, "z_loss": 34.925687313079834, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.25536662340164185, "tokens_per_second": 1335128.90114407, "eta_seconds": 14623.27780880928, "data_wait_seconds": 1.7076730728149414e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 35250, "total_steps": 72479, "tokens": 18481152000, "loss": 2.9350189864635468, "l_mix": 1.8161894977092743, "l_a": 1.8872003108263016, "l_c": 1.8422312140464783, "z_loss": 36.43489456176758, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16804009675979614, "tokens_per_second": 1415630.2958365653, "eta_seconds": 13788.005250668524, "data_wait_seconds": 2.5540590286254883e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 35260, "total_steps": 72479, "tokens": 18486394880, "loss": 4.4458688497543335, "l_mix": 2.7544109523296356, "l_a": 2.842912197113037, "l_c": 2.795280873775482, "z_loss": 30.39688229560852, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11154818534851074, "tokens_per_second": 1320488.276182937, "eta_seconds": 14777.469383072852, "data_wait_seconds": 1.531839370727539e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 35270, "total_steps": 72479, "tokens": 18491637760, "loss": 4.577546834945679, "l_mix": 2.837080180644989, "l_a": 2.9256567656993866, "l_c": 2.8758984208106995, "z_loss": 30.552361249923706, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11664576083421707, "tokens_per_second": 1448593.7495401965, "eta_seconds": 13467.01392173767, "data_wait_seconds": 1.7523765563964844e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 35280, "total_steps": 72479, "tokens": 18496880640, "loss": 4.492999255657196, "l_mix": 2.784369170665741, "l_a": 2.8740756809711456, "l_c": 2.8213579654693604, "z_loss": 31.24243927001953, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16356052458286285, "tokens_per_second": 1409105.671139873, "eta_seconds": 13840.686125564574, "data_wait_seconds": 1.7881393432617188e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 35290, "total_steps": 72479, "tokens": 18502123520, "loss": 4.505919337272644, "l_mix": 2.791764110326767, "l_a": 2.8814265429973602, "l_c": 2.832424283027649, "z_loss": 31.081771850585938, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14540085196495056, "tokens_per_second": 1413172.3723365855, "eta_seconds": 13797.146628165245, "data_wait_seconds": 1.5914440155029297e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 35300, "total_steps": 72479, "tokens": 18507366400, "loss": 4.4626041650772095, "l_mix": 2.7648514211177826, "l_a": 2.85588875412941, "l_c": 2.803286552429199, "z_loss": 30.19803524017334, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1632107049226761, "tokens_per_second": 1456326.9488643915, "eta_seconds": 13384.702911114693, "data_wait_seconds": 1.728534698486328e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.24694824, "gate_a2b/L1_absmax": 0.70508832, "gate_a2b/L1_rms": 0.26083186, "gate_a2b/L3_absmean": 0.25309679, "gate_a2b/L3_absmax": 0.61695838, "gate_a2b/L3_rms": 0.2592268, "gate_a2b/L5_absmean": 0.39770472, "gate_a2b/L5_absmax": 0.66748846, "gate_a2b/L5_rms": 0.40358394, "gate_c2b/L1_absmean": 0.01713448, "gate_c2b/L1_absmax": 0.58872634, "gate_c2b/L1_rms": 0.0673341, "gate_c2b/L3_absmean": 0.21432772, "gate_c2b/L3_absmax": 0.47736192, "gate_c2b/L3_rms": 0.21869543, "gate_c2b/L5_absmean": 0.59310973, "gate_c2b/L5_absmax": 0.82329929, "gate_c2b/L5_rms": 0.60650641, "gate_b2a/L7_absmean": 0.05897646, "gate_b2a/L7_absmax": 0.33589077, "gate_b2a/L7_rms": 0.06463276, "gate_b2a/L11_absmean": 0.09357256, "gate_b2a/L11_absmax": 0.58884734, "gate_b2a/L11_rms": 0.10125066, "gate_b2a/L14_absmean": 0.2396096, "gate_b2a/L14_absmax": 0.74127787, "gate_b2a/L14_rms": 0.24647628, "gate_b2c/L3_absmean": 0.04851943, "gate_b2c/L3_absmax": 0.64014238, "gate_b2c/L3_rms": 0.06130007, "gate_b2c/L5_absmean": 0.05418147, "gate_b2c/L5_absmax": 0.69754601, "gate_b2c/L5_rms": 0.06594966, "gate_b2c/L6_absmean": 0.07916772, "gate_b2c/L6_absmax": 0.47920403, "gate_b2c/L6_rms": 0.0875391}, "streams": {"cos_a2b/round0": 0.39, "cos_c2b/round0": 0.302565, "cos_b2a/round0": 0.138407, "cos_b2c/round0": 0.128954, "rms_a/round0": 3.222191, "rms_b/round0": 6.130753, "rms_c/round0": 2.809014, "cos_a2b/round1": 0.427542, "cos_c2b/round1": 0.297707, "cos_b2a/round1": 0.250505, "cos_b2c/round1": 0.116748, "rms_a/round1": 5.899241, "rms_b/round1": 13.419662, "rms_c/round1": 5.393972, "cos_a2b/round2": 0.561517, "cos_c2b/round2": 0.517287, "cos_b2a/round2": 0.102312, "cos_b2c/round2": 0.170973, "rms_a/round2": 15.103558, "rms_b/round2": 33.500336, "rms_c/round2": 9.583344}, "alpha_mean": 0.4467122617231709, "alpha_hist": [0.00795, 0.06259, 0.16652, 0.19405, 0.17937, 0.16643, 0.13426, 0.07802, 0.0107, 0.00012], "alpha_by_type": {"special": {"share": 0.00101, "mean": 0.55473, "std": 0.10663}, "whitespace": {"share": 0.02782, "mean": 0.53474, "std": 0.12155}, "digit": {"share": 0.03943, "mean": 0.47325, "std": 0.12102}, "word_start": {"share": 0.56184, "mean": 0.47306, "std": 0.16618}, "word_cont": {"share": 0.25381, "mean": 0.32028, "std": 0.13071}, "punct": {"share": 0.11609, "mean": 0.56458, "std": 0.16766}}, "alpha_p05": 0.18477, "alpha_p50": 0.43963, "alpha_p95": 0.73829}
{"step": 35310, "total_steps": 72479, "tokens": 18512609280, "loss": 4.709694504737854, "l_mix": 2.920854091644287, "l_a": 2.9996866285800934, "l_c": 2.963114559650421, "z_loss": 30.524367332458496, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1326037049293518, "tokens_per_second": 1415642.5076010316, "eta_seconds": 13765.665107798575, "data_wait_seconds": 1.71661376953125e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 35320, "total_steps": 72479, "tokens": 18517852160, "loss": 4.84409236907959, "l_mix": 3.004932016134262, "l_a": 3.085502475500107, "l_c": 3.0450315475463867, "z_loss": 29.8589084148407, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.10902640968561172, "tokens_per_second": 1401431.1115002872, "eta_seconds": 13901.516551280021, "data_wait_seconds": 1.761317253112793e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 35330, "total_steps": 72479, "tokens": 18523095040, "loss": 4.835992932319641, "l_mix": 3.000325471162796, "l_a": 3.077456384897232, "l_c": 3.0414351522922516, "z_loss": 29.64799189567566, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11177150905132294, "tokens_per_second": 1472379.4220392716, "eta_seconds": 13228.095027995108, "data_wait_seconds": 4.062056541442871e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 35340, "total_steps": 72479, "tokens": 18528337920, "loss": 4.6938318610191345, "l_mix": 2.9103867411613464, "l_a": 2.9931635558605194, "l_c": 2.951653391122818, "z_loss": 30.88164734840393, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1370508372783661, "tokens_per_second": 1367816.3224056393, "eta_seconds": 14235.487406492233, "data_wait_seconds": 2.822279930114746e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 35350, "total_steps": 72479, "tokens": 18533580800, "loss": 4.711626768112183, "l_mix": 2.922126829624176, "l_a": 3.0018802285194397, "l_c": 2.963119328022003, "z_loss": 30.29661989212036, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14380836486816406, "tokens_per_second": 1440246.611331392, "eta_seconds": 13515.941644191742, "data_wait_seconds": 2.4318695068359375e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 35360, "total_steps": 72479, "tokens": 18538823680, "loss": 3.2482781410217285, "l_mix": 2.0113856196403503, "l_a": 2.0828527212142944, "l_c": 2.040122225880623, "z_loss": 35.543320655822754, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16270208358764648, "tokens_per_second": 1411730.0594929752, "eta_seconds": 13785.246082377435, "data_wait_seconds": 2.6911497116088867e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 35370, "total_steps": 72479, "tokens": 18544066560, "loss": 4.601679265499115, "l_mix": 2.851725846529007, "l_a": 2.9400311708450317, "l_c": 2.8931469321250916, "z_loss": 30.447657585144043, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12111368775367737, "tokens_per_second": 1398503.137664738, "eta_seconds": 13911.876826024056, "data_wait_seconds": 2.181529998779297e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 35380, "total_steps": 72479, "tokens": 18549309440, "loss": 4.539932012557983, "l_mix": 2.813295215368271, "l_a": 2.9008132815361023, "l_c": 2.854642152786255, "z_loss": 30.617098093032837, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13074259459972382, "tokens_per_second": 1469493.4027206658, "eta_seconds": 13236.235342049598, "data_wait_seconds": 1.5676021575927734e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 35390, "total_steps": 72479, "tokens": 18554552320, "loss": 4.454289555549622, "l_mix": 2.7592912316322327, "l_a": 2.851852834224701, "l_c": 2.798141658306122, "z_loss": 30.317302942276, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1366376280784607, "tokens_per_second": 1279414.2936771072, "eta_seconds": 15198.609026098251, "data_wait_seconds": 2.092123031616211e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 35400, "total_steps": 72479, "tokens": 18559795200, "loss": 4.490216851234436, "l_mix": 2.7807894945144653, "l_a": 2.8740968704223633, "l_c": 2.8239942491054535, "z_loss": 31.030542850494385, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15979500114917755, "tokens_per_second": 1464779.3276007934, "eta_seconds": 13271.67470600605, "data_wait_seconds": 2.524256706237793e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.24663983, "gate_a2b/L1_absmax": 0.70763248, "gate_a2b/L1_rms": 0.26040661, "gate_a2b/L3_absmean": 0.2523655, "gate_a2b/L3_absmax": 0.6166622, "gate_a2b/L3_rms": 0.25850436, "gate_a2b/L5_absmean": 0.39619628, "gate_a2b/L5_absmax": 0.66896868, "gate_a2b/L5_rms": 0.40206268, "gate_c2b/L1_absmean": 0.01722456, "gate_c2b/L1_absmax": 0.59170687, "gate_c2b/L1_rms": 0.06731607, "gate_c2b/L3_absmean": 0.21409504, "gate_c2b/L3_absmax": 0.47245258, "gate_c2b/L3_rms": 0.21839041, "gate_c2b/L5_absmean": 0.59268445, "gate_c2b/L5_absmax": 0.8308599, "gate_c2b/L5_rms": 0.60601616, "gate_b2a/L7_absmean": 0.05842662, "gate_b2a/L7_absmax": 0.33471182, "gate_b2a/L7_rms": 0.06411865, "gate_b2a/L11_absmean": 0.09307287, "gate_b2a/L11_absmax": 0.58866018, "gate_b2a/L11_rms": 0.10080843, "gate_b2a/L14_absmean": 0.23910233, "gate_b2a/L14_absmax": 0.74155122, "gate_b2a/L14_rms": 0.24599811, "gate_b2c/L3_absmean": 0.04835584, "gate_b2c/L3_absmax": 0.637196, "gate_b2c/L3_rms": 0.06104441, "gate_b2c/L5_absmean": 0.05375087, "gate_b2c/L5_absmax": 0.69570059, "gate_b2c/L5_rms": 0.0654793, "gate_b2c/L6_absmean": 0.07872178, "gate_b2c/L6_absmax": 0.48021385, "gate_b2c/L6_rms": 0.08714563}, "streams": {"cos_a2b/round0": 0.384907, "cos_c2b/round0": 0.300826, "cos_b2a/round0": 0.131575, "cos_b2c/round0": 0.125147, "rms_a/round0": 3.245485, "rms_b/round0": 6.244268, "rms_c/round0": 2.836425, "cos_a2b/round1": 0.419548, "cos_c2b/round1": 0.292797, "cos_b2a/round1": 0.243408, "cos_b2c/round1": 0.112765, "rms_a/round1": 6.010959, "rms_b/round1": 13.802193, "rms_c/round1": 5.471416, "cos_a2b/round2": 0.53936, "cos_c2b/round2": 0.528021, "cos_b2a/round2": 0.096448, "cos_b2c/round2": 0.162597, "rms_a/round2": 15.197316, "rms_b/round2": 34.396591, "rms_c/round2": 9.866155}, "alpha_mean": 0.4096631023570225, "alpha_hist": [0.01427, 0.09285, 0.20415, 0.20761, 0.17157, 0.13921, 0.10965, 0.05408, 0.00642, 0.0002], "alpha_by_type": {"special": {"share": 0.00107, "mean": 0.54158, "std": 0.12296}, "whitespace": {"share": 0.02969, "mean": 0.52016, "std": 0.1345}, "digit": {"share": 0.03911, "mean": 0.45515, "std": 0.13769}, "word_start": {"share": 0.54324, "mean": 0.42502, "std": 0.16627}, "word_cont": {"share": 0.27858, "mean": 0.30339, "std": 0.12847}, "punct": {"share": 0.10831, "mean": 0.55796, "std": 0.16979}}, "alpha_p05": 0.16886, "alpha_p50": 0.39794, "alpha_p95": 0.72565}
{"step": 35410, "total_steps": 72479, "tokens": 18565038080, "loss": 4.647841930389404, "l_mix": 2.879785805940628, "l_a": 2.971681147813797, "l_c": 2.921838939189911, "z_loss": 31.259162187576294, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17427660524845123, "tokens_per_second": 1480460.6543984297, "eta_seconds": 13127.557165575028, "data_wait_seconds": 1.6182661056518555e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 35420, "total_steps": 72479, "tokens": 18570280960, "loss": 3.0203266739845276, "l_mix": 1.8682539910078049, "l_a": 1.9415350258350372, "l_c": 1.8987071216106415, "z_loss": 35.80480337142944, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.21884785592556, "tokens_per_second": 1444304.8100831509, "eta_seconds": 13452.554375195503, "data_wait_seconds": 1.823902130126953e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 35430, "total_steps": 72479, "tokens": 18575523840, "loss": 4.762265563011169, "l_mix": 2.9543046951293945, "l_a": 3.03221994638443, "l_c": 2.9943162202835083, "z_loss": 30.12299132347107, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14001508057117462, "tokens_per_second": 1384168.4251118354, "eta_seconds": 14033.224396395684, "data_wait_seconds": 2.0682811737060547e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20768, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 35440, "total_steps": 72479, "tokens": 18580766720, "loss": 4.6479551792144775, "l_mix": 2.8823354244232178, "l_a": 2.964242458343506, "l_c": 2.921156257390976, "z_loss": 30.59734559059143, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13730697333812714, "tokens_per_second": 1459547.857930083, "eta_seconds": 13304.87597682476, "data_wait_seconds": 2.2470951080322266e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 35450, "total_steps": 72479, "tokens": 18586009600, "loss": 4.89900004863739, "l_mix": 3.0385851860046387, "l_a": 3.114638179540634, "l_c": 3.0867442786693573, "z_loss": 29.25586771965027, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.28768467903137207, "tokens_per_second": 1462907.1186556139, "eta_seconds": 13270.740229797364, "data_wait_seconds": 1.9848346710205078e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 35460, "total_steps": 72479, "tokens": 18591252480, "loss": 4.691227674484253, "l_mix": 2.909098505973816, "l_a": 2.9928746819496155, "l_c": 2.9475558400154114, "z_loss": 31.17530393600464, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19654209911823273, "tokens_per_second": 1341102.531694266, "eta_seconds": 14472.135435819626, "data_wait_seconds": 5.614757537841797e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 35470, "total_steps": 72479, "tokens": 18596495360, "loss": 4.678998231887817, "l_mix": 2.901224732398987, "l_a": 2.9834359884262085, "l_c": 2.942475289106369, "z_loss": 31.156229734420776, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13910681009292603, "tokens_per_second": 1390312.9628729038, "eta_seconds": 13956.120032072067, "data_wait_seconds": 2.3186206817626953e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 35480, "total_steps": 72479, "tokens": 18601738240, "loss": 4.58574914932251, "l_mix": 2.843037962913513, "l_a": 2.9261023104190826, "l_c": 2.882934868335724, "z_loss": 30.80084466934204, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15496093034744263, "tokens_per_second": 1393827.062911643, "eta_seconds": 13917.172530341148, "data_wait_seconds": 2.0682811737060547e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 35490, "total_steps": 72479, "tokens": 18606981120, "loss": 4.502532005310059, "l_mix": 2.7901516258716583, "l_a": 2.876797527074814, "l_c": 2.8311366736888885, "z_loss": 31.338240146636963, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14364340901374817, "tokens_per_second": 1368601.8884091538, "eta_seconds": 14169.853918981551, "data_wait_seconds": 2.8342008590698242e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 35500, "total_steps": 72479, "tokens": 18612224000, "loss": 4.535360395908356, "l_mix": 2.809761732816696, "l_a": 2.90019953250885, "l_c": 2.8517956733703613, "z_loss": 31.526350736618042, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12771768867969513, "tokens_per_second": 1461328.6330873866, "eta_seconds": 13267.136161589622, "data_wait_seconds": 1.722574234008789e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.2448235, "gate_a2b/L1_absmax": 0.70632702, "gate_a2b/L1_rms": 0.25870457, "gate_a2b/L3_absmean": 0.2510798, "gate_a2b/L3_absmax": 0.61950016, "gate_a2b/L3_rms": 0.25731844, "gate_a2b/L5_absmean": 0.39456549, "gate_a2b/L5_absmax": 0.6695919, "gate_a2b/L5_rms": 0.40043235, "gate_c2b/L1_absmean": 0.01706155, "gate_c2b/L1_absmax": 0.59237671, "gate_c2b/L1_rms": 0.0671788, "gate_c2b/L3_absmean": 0.21396719, "gate_c2b/L3_absmax": 0.47276098, "gate_c2b/L3_rms": 0.21822758, "gate_c2b/L5_absmean": 0.59238493, "gate_c2b/L5_absmax": 0.8302083, "gate_c2b/L5_rms": 0.60576802, "gate_b2a/L7_absmean": 0.05850569, "gate_b2a/L7_absmax": 0.33340865, "gate_b2a/L7_rms": 0.06412998, "gate_b2a/L11_absmean": 0.09303503, "gate_b2a/L11_absmax": 0.5901702, "gate_b2a/L11_rms": 0.10076858, "gate_b2a/L14_absmean": 0.23803651, "gate_b2a/L14_absmax": 0.74122477, "gate_b2a/L14_rms": 0.24493197, "gate_b2c/L3_absmean": 0.0477955, "gate_b2c/L3_absmax": 0.63550729, "gate_b2c/L3_rms": 0.06047802, "gate_b2c/L5_absmean": 0.05337957, "gate_b2c/L5_absmax": 0.69624805, "gate_b2c/L5_rms": 0.06519442, "gate_b2c/L6_absmean": 0.07777524, "gate_b2c/L6_absmax": 0.47985393, "gate_b2c/L6_rms": 0.08626944}, "streams": {"cos_a2b/round0": 0.386778, "cos_c2b/round0": 0.30734, "cos_b2a/round0": 0.137838, "cos_b2c/round0": 0.128975, "rms_a/round0": 3.287703, "rms_b/round0": 6.278406, "rms_c/round0": 2.805766, "cos_a2b/round1": 0.428177, "cos_c2b/round1": 0.296716, "cos_b2a/round1": 0.241693, "cos_b2c/round1": 0.116123, "rms_a/round1": 6.118417, "rms_b/round1": 13.815392, "rms_c/round1": 5.470468, "cos_a2b/round2": 0.563994, "cos_c2b/round2": 0.51606, "cos_b2a/round2": 0.110251, "cos_b2c/round2": 0.173974, "rms_a/round2": 15.607628, "rms_b/round2": 34.668739, "rms_c/round2": 9.759865}, "alpha_mean": 0.4553352059279945, "alpha_hist": [0.00424, 0.05107, 0.1441, 0.18297, 0.21291, 0.19673, 0.1348, 0.06538, 0.00768, 0.00012], "alpha_by_type": {"special": {"share": 0.00092, "mean": 0.54089, "std": 0.12071}, "whitespace": {"share": 0.03172, "mean": 0.51107, "std": 0.11707}, "digit": {"share": 0.03851, "mean": 0.45496, "std": 0.10605}, "word_start": {"share": 0.55574, "mean": 0.47418, "std": 0.15694}, "word_cont": {"share": 0.25768, "mean": 0.34774, "std": 0.12635}, "punct": {"share": 0.11543, "mean": 0.5889, "std": 0.14435}}, "alpha_p05": 0.18713, "alpha_p50": 0.45835, "alpha_p95": 0.72194}
{"step": 35510, "total_steps": 72479, "tokens": 18617466880, "loss": 4.482431709766388, "l_mix": 2.7773056626319885, "l_a": 2.866279810667038, "l_c": 2.817473739385605, "z_loss": 30.462849378585815, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13495148718357086, "tokens_per_second": 1391027.4400520322, "eta_seconds": 13933.875431871415, "data_wait_seconds": 1.8388032913208008e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 35520, "total_steps": 72479, "tokens": 18622709760, "loss": 4.546441376209259, "l_mix": 2.8169038891792297, "l_a": 2.9086616039276123, "l_c": 2.856463134288788, "z_loss": 30.809975385665894, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1282995641231537, "tokens_per_second": 1448313.825904113, "eta_seconds": 13379.117043161392, "data_wait_seconds": 1.576542854309082e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 35530, "total_steps": 72479, "tokens": 18627952640, "loss": 3.0845512449741364, "l_mix": 1.9088440835475922, "l_a": 1.9768737852573395, "l_c": 1.9421500265598297, "z_loss": 35.54574108123779, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1687469184398651, "tokens_per_second": 1463168.470491451, "eta_seconds": 13239.70390470028, "data_wait_seconds": 2.384185791015625e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 35540, "total_steps": 72479, "tokens": 18633195520, "loss": 3.0204228460788727, "l_mix": 1.869944378733635, "l_a": 1.9395314157009125, "l_c": 1.895396575331688, "z_loss": 36.10592460632324, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17912182211875916, "tokens_per_second": 1454475.3087416943, "eta_seconds": 13315.230802202226, "data_wait_seconds": 2.390146255493164e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08236}}
{"step": 35550, "total_steps": 72479, "tokens": 18638438400, "loss": 4.667787283658981, "l_mix": 2.895176023244858, "l_a": 2.9737488627433777, "l_c": 2.934954822063446, "z_loss": 30.99636220932007, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12086337804794312, "tokens_per_second": 1388705.526403503, "eta_seconds": 13942.07136349678, "data_wait_seconds": 2.4080276489257812e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 35560, "total_steps": 72479, "tokens": 18643681280, "loss": 4.425355434417725, "l_mix": 2.7446579039096832, "l_a": 2.81903275847435, "l_c": 2.783292382955551, "z_loss": 31.843632221221924, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1675645262002945, "tokens_per_second": 1471703.2442863013, "eta_seconds": 13152.23619105816, "data_wait_seconds": 1.913309097290039e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 35570, "total_steps": 72479, "tokens": 18648924160, "loss": 4.763912558555603, "l_mix": 2.955351233482361, "l_a": 3.0334193110466003, "l_c": 2.995118200778961, "z_loss": 30.100717306137085, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13901165127754211, "tokens_per_second": 1404230.7547242222, "eta_seconds": 13780.460032582283, "data_wait_seconds": 1.8328428268432617e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 35580, "total_steps": 72479, "tokens": 18654167040, "loss": 4.542751967906952, "l_mix": 2.8179949522018433, "l_a": 2.8925026059150696, "l_c": 2.8566874265670776, "z_loss": 30.430850982666016, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1538451910018921, "tokens_per_second": 1412187.8003368664, "eta_seconds": 13699.100719738008, "data_wait_seconds": 3.263354301452637e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 35590, "total_steps": 72479, "tokens": 18659409920, "loss": 4.557882755994797, "l_mix": 2.8238745033740997, "l_a": 2.916616827249527, "l_c": 2.863410532474518, "z_loss": 30.899456024169922, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13519228994846344, "tokens_per_second": 1411538.402394211, "eta_seconds": 13701.68888015747, "data_wait_seconds": 1.71661376953125e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 35600, "total_steps": 72479, "tokens": 18664652800, "loss": 4.608313262462616, "l_mix": 2.856367737054825, "l_a": 2.9408218264579773, "l_c": 2.8989961743354797, "z_loss": 31.47979497909546, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13489003479480743, "tokens_per_second": 1368618.327847404, "eta_seconds": 14127.545100474357, "data_wait_seconds": 1.8298625946044922e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.24421783, "gate_a2b/L1_absmax": 0.70659286, "gate_a2b/L1_rms": 0.25807875, "gate_a2b/L3_absmean": 0.25023016, "gate_a2b/L3_absmax": 0.61724472, "gate_a2b/L3_rms": 0.25644061, "gate_a2b/L5_absmean": 0.39305109, "gate_a2b/L5_absmax": 0.67197502, "gate_a2b/L5_rms": 0.39895186, "gate_c2b/L1_absmean": 0.01752617, "gate_c2b/L1_absmax": 0.59395784, "gate_c2b/L1_rms": 0.06743714, "gate_c2b/L3_absmean": 0.21342556, "gate_c2b/L3_absmax": 0.47634634, "gate_c2b/L3_rms": 0.21778975, "gate_c2b/L5_absmean": 0.5921821, "gate_c2b/L5_absmax": 0.84149969, "gate_c2b/L5_rms": 0.60551125, "gate_b2a/L7_absmean": 0.0581864, "gate_b2a/L7_absmax": 0.33394006, "gate_b2a/L7_rms": 0.06380309, "gate_b2a/L11_absmean": 0.09264062, "gate_b2a/L11_absmax": 0.58892477, "gate_b2a/L11_rms": 0.10038822, "gate_b2a/L14_absmean": 0.23737165, "gate_b2a/L14_absmax": 0.74251431, "gate_b2a/L14_rms": 0.24431884, "gate_b2c/L3_absmean": 0.04750141, "gate_b2c/L3_absmax": 0.63515937, "gate_b2c/L3_rms": 0.06019871, "gate_b2c/L5_absmean": 0.05347082, "gate_b2c/L5_absmax": 0.6971522, "gate_b2c/L5_rms": 0.06520929, "gate_b2c/L6_absmean": 0.07752176, "gate_b2c/L6_absmax": 0.48023692, "gate_b2c/L6_rms": 0.08594513}, "streams": {"cos_a2b/round0": 0.387067, "cos_c2b/round0": 0.303144, "cos_b2a/round0": 0.13795, "cos_b2c/round0": 0.127997, "rms_a/round0": 3.328448, "rms_b/round0": 6.352086, "rms_c/round0": 2.850738, "cos_a2b/round1": 0.424889, "cos_c2b/round1": 0.297415, "cos_b2a/round1": 0.248814, "cos_b2c/round1": 0.121568, "rms_a/round1": 6.183, "rms_b/round1": 14.035388, "rms_c/round1": 5.568632, "cos_a2b/round2": 0.555933, "cos_c2b/round2": 0.52096, "cos_b2a/round2": 0.105981, "cos_b2c/round2": 0.172173, "rms_a/round2": 15.807284, "rms_b/round2": 35.190392, "rms_c/round2": 10.171921}, "alpha_mean": 0.43407237824735034, "alpha_hist": [0.00778, 0.0695, 0.18141, 0.20477, 0.17923, 0.15567, 0.12361, 0.06877, 0.0092, 5e-05], "alpha_by_type": {"special": {"share": 0.00166, "mean": 0.58906, "std": 0.12257}, "whitespace": {"share": 0.02617, "mean": 0.51434, "std": 0.11797}, "digit": {"share": 0.02876, "mean": 0.43017, "std": 0.10265}, "word_start": {"share": 0.57353, "mean": 0.45851, "std": 0.16468}, "word_cont": {"share": 0.27141, "mean": 0.31921, "std": 0.13156}, "punct": {"share": 0.09846, "mean": 0.58551, "std": 0.15923}}, "alpha_p05": 0.17953, "alpha_p50": 0.42584, "alpha_p95": 0.74205}
{"step": 35610, "total_steps": 72479, "tokens": 18669895680, "loss": 4.5084444880485535, "l_mix": 2.793685883283615, "l_a": 2.881831169128418, "l_c": 2.8340305387973785, "z_loss": 31.237672090530396, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11334134638309479, "tokens_per_second": 1408465.3286784654, "eta_seconds": 13724.139230418206, "data_wait_seconds": 3.165006637573242e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 35620, "total_steps": 72479, "tokens": 18675138560, "loss": 4.563492596149445, "l_mix": 2.827245742082596, "l_a": 2.919003516435623, "l_c": 2.8684857189655304, "z_loss": 31.16480016708374, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12548133730888367, "tokens_per_second": 1470249.3316935, "eta_seconds": 13143.84640443325, "data_wait_seconds": 1.6033649444580078e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 35630, "total_steps": 72479, "tokens": 18680381440, "loss": 4.522647261619568, "l_mix": 2.803207129240036, "l_a": 2.889298737049103, "l_c": 2.842168301343918, "z_loss": 30.64697790145874, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.10798460990190506, "tokens_per_second": 1415399.8610695777, "eta_seconds": 13649.491598367693, "data_wait_seconds": 1.7374753952026367e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 35640, "total_steps": 72479, "tokens": 18685624320, "loss": 3.0298416018486023, "l_mix": 1.8758556842803955, "l_a": 1.9425373077392578, "l_c": 1.9040823578834534, "z_loss": 36.20653676986694, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14940786361694336, "tokens_per_second": 1418088.0052317157, "eta_seconds": 13619.920315766334, "data_wait_seconds": 2.625584602355957e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 35650, "total_steps": 72479, "tokens": 18690867200, "loss": 3.057104378938675, "l_mix": 1.8920424580574036, "l_a": 1.9622011929750443, "l_c": 1.9213382303714752, "z_loss": 36.79159736633301, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18195651471614838, "tokens_per_second": 1433687.2890211944, "eta_seconds": 13468.07138478756, "data_wait_seconds": 1.7255544662475586e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08236}}
{"step": 35660, "total_steps": 72479, "tokens": 18696110080, "loss": 3.22546923160553, "l_mix": 1.9826084524393082, "l_a": 2.104214996099472, "l_c": 2.038654014468193, "z_loss": 32.88875675201416, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1693839430809021, "tokens_per_second": 1376924.6250703325, "eta_seconds": 14019.474646997453, "data_wait_seconds": 4.5806169509887695e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20045, "finemath": 0.08233}}
{"step": 35670, "total_steps": 72479, "tokens": 18701352960, "loss": 4.489696562290192, "l_mix": 2.7828536331653595, "l_a": 2.8664484918117523, "l_c": 2.8230278491973877, "z_loss": 30.644415140151978, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12316397577524185, "tokens_per_second": 1465352.7734869777, "eta_seconds": 13169.877821350097, "data_wait_seconds": 1.5854835510253906e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 35680, "total_steps": 72479, "tokens": 18706595840, "loss": 4.482221782207489, "l_mix": 2.7770379185676575, "l_a": 2.8680643141269684, "l_c": 2.815881848335266, "z_loss": 31.94603443145752, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1484740674495697, "tokens_per_second": 1474838.0834390044, "eta_seconds": 13081.621859812736, "data_wait_seconds": 1.8596649169921875e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 35690, "total_steps": 72479, "tokens": 18711838720, "loss": 4.5536521673202515, "l_mix": 2.821336567401886, "l_a": 2.911147862672806, "l_c": 2.863237112760544, "z_loss": 31.05623197555542, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12898807227611542, "tokens_per_second": 1398691.5379369918, "eta_seconds": 13790.053567099572, "data_wait_seconds": 2.0295381546020508e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 35700, "total_steps": 72479, "tokens": 18717081600, "loss": 4.461002349853516, "l_mix": 2.7634679675102234, "l_a": 2.85374853014946, "l_c": 2.8046987652778625, "z_loss": 31.549391508102417, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12062075734138489, "tokens_per_second": 1426992.676569333, "eta_seconds": 13512.885292696952, "data_wait_seconds": 1.7076730728149414e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.24424961, "gate_a2b/L1_absmax": 0.7051053, "gate_a2b/L1_rms": 0.25788161, "gate_a2b/L3_absmean": 0.24910764, "gate_a2b/L3_absmax": 0.6168341, "gate_a2b/L3_rms": 0.25529686, "gate_a2b/L5_absmean": 0.39159662, "gate_a2b/L5_absmax": 0.67224336, "gate_a2b/L5_rms": 0.39749917, "gate_c2b/L1_absmean": 0.01710568, "gate_c2b/L1_absmax": 0.59242821, "gate_c2b/L1_rms": 0.06743265, "gate_c2b/L3_absmean": 0.21315466, "gate_c2b/L3_absmax": 0.47462609, "gate_c2b/L3_rms": 0.21746434, "gate_c2b/L5_absmean": 0.59212464, "gate_c2b/L5_absmax": 0.82666218, "gate_c2b/L5_rms": 0.60543031, "gate_b2a/L7_absmean": 0.05811182, "gate_b2a/L7_absmax": 0.33204493, "gate_b2a/L7_rms": 0.06370573, "gate_b2a/L11_absmean": 0.09218372, "gate_b2a/L11_absmax": 0.58833432, "gate_b2a/L11_rms": 0.09996083, "gate_b2a/L14_absmean": 0.23679963, "gate_b2a/L14_absmax": 0.74198097, "gate_b2a/L14_rms": 0.24371548, "gate_b2c/L3_absmean": 0.04739425, "gate_b2c/L3_absmax": 0.63484704, "gate_b2c/L3_rms": 0.06003137, "gate_b2c/L5_absmean": 0.05303442, "gate_b2c/L5_absmax": 0.69513869, "gate_b2c/L5_rms": 0.0647928, "gate_b2c/L6_absmean": 0.07731096, "gate_b2c/L6_absmax": 0.47715846, "gate_b2c/L6_rms": 0.0856517}, "streams": {"cos_a2b/round0": 0.388889, "cos_c2b/round0": 0.304744, "cos_b2a/round0": 0.136424, "cos_b2c/round0": 0.125911, "rms_a/round0": 3.340358, "rms_b/round0": 6.354047, "rms_c/round0": 2.877202, "cos_a2b/round1": 0.421557, "cos_c2b/round1": 0.294158, "cos_b2a/round1": 0.24449, "cos_b2c/round1": 0.114337, "rms_a/round1": 6.190227, "rms_b/round1": 14.015198, "rms_c/round1": 5.535935, "cos_a2b/round2": 0.543112, "cos_c2b/round2": 0.533433, "cos_b2a/round2": 0.097228, "cos_b2c/round2": 0.16786, "rms_a/round2": 15.70592, "rms_b/round2": 35.35836, "rms_c/round2": 10.007613}, "alpha_mean": 0.41600806135407, "alpha_hist": [0.01279, 0.07919, 0.18719, 0.21294, 0.18861, 0.15991, 0.10634, 0.04842, 0.00456, 5e-05], "alpha_by_type": {"special": {"share": 0.00124, "mean": 0.53214, "std": 0.12129}, "whitespace": {"share": 0.02428, "mean": 0.50918, "std": 0.12268}, "digit": {"share": 0.04056, "mean": 0.42456, "std": 0.11693}, "word_start": {"share": 0.55539, "mean": 0.43473, "std": 0.16066}, "word_cont": {"share": 0.26514, "mean": 0.30934, "std": 0.1262}, "punct": {"share": 0.1134, "mean": 0.54943, "std": 0.15609}}, "alpha_p05": 0.17302, "alpha_p50": 0.41253, "alpha_p95": 0.70232}
{"step": 35710, "total_steps": 72479, "tokens": 18722324480, "loss": 4.579545855522156, "l_mix": 2.838592082262039, "l_a": 2.9246824979782104, "l_c": 2.8784965872764587, "z_loss": 31.10236930847168, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1125328466296196, "tokens_per_second": 1412665.530690878, "eta_seconds": 13646.220604372025, "data_wait_seconds": 1.6629695892333984e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 35720, "total_steps": 72479, "tokens": 18727567360, "loss": 4.513192236423492, "l_mix": 2.7966200411319733, "l_a": 2.8867558240890503, "l_c": 2.83515128493309, "z_loss": 30.678303956985474, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11043862253427505, "tokens_per_second": 1451185.5483456792, "eta_seconds": 13280.384864616393, "data_wait_seconds": 1.570582389831543e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 35730, "total_steps": 72479, "tokens": 18732810240, "loss": 4.555336952209473, "l_mix": 2.8230150043964386, "l_a": 2.9103176295757294, "l_c": 2.864088326692581, "z_loss": 31.325359344482422, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.10571454465389252, "tokens_per_second": 1388612.046452781, "eta_seconds": 13875.048658275604, "data_wait_seconds": 1.7374753952026367e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 35740, "total_steps": 72479, "tokens": 18738053120, "loss": 4.481799751520157, "l_mix": 2.776879757642746, "l_a": 2.864556223154068, "l_c": 2.818510353565216, "z_loss": 30.779460191726685, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1503930389881134, "tokens_per_second": 1339496.7331622199, "eta_seconds": 14379.890861344338, "data_wait_seconds": 2.0444393157958984e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 35750, "total_steps": 72479, "tokens": 18743296000, "loss": 4.51884263753891, "l_mix": 2.8007436394691467, "l_a": 2.8883329033851624, "l_c": 2.8386634290218353, "z_loss": 31.508391618728638, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1856323629617691, "tokens_per_second": 1481161.6654035277, "eta_seconds": 13000.994018268586, "data_wait_seconds": 1.8537044525146484e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 35760, "total_steps": 72479, "tokens": 18748538880, "loss": 2.884099006652832, "l_mix": 1.785593181848526, "l_a": 1.8515356183052063, "l_c": 1.8101502060890198, "z_loss": 37.02629518508911, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19864235818386078, "tokens_per_second": 1375025.140091007, "eta_seconds": 14000.712067508697, "data_wait_seconds": 2.4199485778808594e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 35770, "total_steps": 72479, "tokens": 18753781760, "loss": 3.244924485683441, "l_mix": 1.995431512594223, "l_a": 2.114105463027954, "l_c": 2.050870880484581, "z_loss": 32.52407169342041, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18106292188167572, "tokens_per_second": 1396487.4650870713, "eta_seconds": 13781.783706021308, "data_wait_seconds": 3.269314765930176e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20045, "finemath": 0.08233}}
{"step": 35780, "total_steps": 72479, "tokens": 18759024640, "loss": 4.448049187660217, "l_mix": 2.756555885076523, "l_a": 2.8434529900550842, "l_c": 2.7948578596115112, "z_loss": 31.33526372909546, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12967005372047424, "tokens_per_second": 1460326.3642514823, "eta_seconds": 13175.71591050625, "data_wait_seconds": 1.7911195755004883e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 35790, "total_steps": 72479, "tokens": 18764267520, "loss": 4.5120251178741455, "l_mix": 2.796020030975342, "l_a": 2.884333908557892, "l_c": 2.835682839155197, "z_loss": 31.21387267112732, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12016001343727112, "tokens_per_second": 1270258.212373374, "eta_seconds": 15143.064807319643, "data_wait_seconds": 1.963973045349121e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 35800, "total_steps": 72479, "tokens": 18769510400, "loss": 4.5444135665893555, "l_mix": 2.8155605494976044, "l_a": 2.9073421359062195, "l_c": 2.8555010855197906, "z_loss": 30.817201375961304, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12498215585947037, "tokens_per_second": 1423075.8041309272, "eta_seconds": 13513.236256408692, "data_wait_seconds": 1.531839370727539e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.24295805, "gate_a2b/L1_absmax": 0.70586133, "gate_a2b/L1_rms": 0.25666553, "gate_a2b/L3_absmean": 0.24858759, "gate_a2b/L3_absmax": 0.61782199, "gate_a2b/L3_rms": 0.25471461, "gate_a2b/L5_absmean": 0.39056143, "gate_a2b/L5_absmax": 0.67287618, "gate_a2b/L5_rms": 0.39645451, "gate_c2b/L1_absmean": 0.01741603, "gate_c2b/L1_absmax": 0.59306651, "gate_c2b/L1_rms": 0.06740671, "gate_c2b/L3_absmean": 0.21327741, "gate_c2b/L3_absmax": 0.47179666, "gate_c2b/L3_rms": 0.21758576, "gate_c2b/L5_absmean": 0.59121084, "gate_c2b/L5_absmax": 0.82596809, "gate_c2b/L5_rms": 0.60450339, "gate_b2a/L7_absmean": 0.0577517, "gate_b2a/L7_absmax": 0.3326112, "gate_b2a/L7_rms": 0.06339547, "gate_b2a/L11_absmean": 0.09230314, "gate_b2a/L11_absmax": 0.58652335, "gate_b2a/L11_rms": 0.10000943, "gate_b2a/L14_absmean": 0.23598383, "gate_b2a/L14_absmax": 0.74228525, "gate_b2a/L14_rms": 0.24297212, "gate_b2c/L3_absmean": 0.04734068, "gate_b2c/L3_absmax": 0.63444209, "gate_b2c/L3_rms": 0.05995142, "gate_b2c/L5_absmean": 0.05260604, "gate_b2c/L5_absmax": 0.69387579, "gate_b2c/L5_rms": 0.06438114, "gate_b2c/L6_absmean": 0.07666878, "gate_b2c/L6_absmax": 0.47542298, "gate_b2c/L6_rms": 0.0850354}, "streams": {"cos_a2b/round0": 0.385311, "cos_c2b/round0": 0.306837, "cos_b2a/round0": 0.140017, "cos_b2c/round0": 0.126748, "rms_a/round0": 3.368874, "rms_b/round0": 6.447415, "rms_c/round0": 2.851599, "cos_a2b/round1": 0.425636, "cos_c2b/round1": 0.295763, "cos_b2a/round1": 0.240969, "cos_b2c/round1": 0.115317, "rms_a/round1": 6.327369, "rms_b/round1": 14.203801, "rms_c/round1": 5.598825, "cos_a2b/round2": 0.557934, "cos_c2b/round2": 0.519958, "cos_b2a/round2": 0.104372, "cos_b2c/round2": 0.17166, "rms_a/round2": 16.045641, "rms_b/round2": 35.576321, "rms_c/round2": 10.145429}, "alpha_mean": 0.4503131612743658, "alpha_hist": [0.01018, 0.0609, 0.15076, 0.1971, 0.18333, 0.17256, 0.13641, 0.07599, 0.01263, 0.00014], "alpha_by_type": {"special": {"share": 0.00114, "mean": 0.55892, "std": 0.12437}, "whitespace": {"share": 0.02272, "mean": 0.53404, "std": 0.11841}, "digit": {"share": 0.0325, "mean": 0.43454, "std": 0.12729}, "word_start": {"share": 0.56656, "mean": 0.47132, "std": 0.16789}, "word_cont": {"share": 0.26215, "mean": 0.33902, "std": 0.13501}, "punct": {"share": 0.11493, "mean": 0.58743, "std": 0.15915}}, "alpha_p05": 0.17953, "alpha_p50": 0.44794, "alpha_p95": 0.74242}
{"step": 35810, "total_steps": 72479, "tokens": 18774753280, "loss": 4.561229825019836, "l_mix": 2.8277030885219574, "l_a": 2.9128182530403137, "l_c": 2.865603983402252, "z_loss": 30.701218843460083, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1818239688873291, "tokens_per_second": 1472578.5901337473, "eta_seconds": 13055.409606528283, "data_wait_seconds": 2.130866050720215e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 35820, "total_steps": 72479, "tokens": 18779996160, "loss": 4.560892939567566, "l_mix": 2.826101541519165, "l_a": 2.915835350751877, "l_c": 2.8668022453784943, "z_loss": 30.481653213500977, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18134507536888123, "tokens_per_second": 1440036.6662279153, "eta_seconds": 13346.794732904433, "data_wait_seconds": 2.7388334274291992e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08236}}
{"step": 35830, "total_steps": 72479, "tokens": 18785239040, "loss": 4.549480885267258, "l_mix": 2.8218170404434204, "l_a": 2.8981626629829407, "l_c": 2.8607165813446045, "z_loss": 30.830230951309204, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1300421953201294, "tokens_per_second": 1404638.0642746382, "eta_seconds": 13679.417780780794, "data_wait_seconds": 2.4527311325073242e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20768, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 35840, "total_steps": 72479, "tokens": 18790481920, "loss": 4.733772933483124, "l_mix": 2.9362545013427734, "l_a": 3.014398604631424, "l_c": 2.9773293137550354, "z_loss": 31.049857139587402, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11290495842695236, "tokens_per_second": 1464540.1259784226, "eta_seconds": 13116.327570176125, "data_wait_seconds": 4.690885543823242e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 35850, "total_steps": 72479, "tokens": 18795724800, "loss": 4.725194334983826, "l_mix": 2.930039405822754, "l_a": 3.0116535425186157, "l_c": 2.972195953130722, "z_loss": 30.879104375839233, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15288308262825012, "tokens_per_second": 1467706.9870803773, "eta_seconds": 13084.454404759406, "data_wait_seconds": 1.7017126083374023e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 35860, "total_steps": 72479, "tokens": 18800967680, "loss": 4.694043755531311, "l_mix": 2.9117866456508636, "l_a": 2.9904129207134247, "l_c": 2.950443983078003, "z_loss": 31.141052961349487, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11459414660930634, "tokens_per_second": 1443592.3758069081, "eta_seconds": 13299.39295451641, "data_wait_seconds": 2.1755695343017578e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 35870, "total_steps": 72479, "tokens": 18806210560, "loss": 4.7842937707901, "l_mix": 2.967772424221039, "l_a": 3.0465889275074005, "l_c": 3.0084820687770844, "z_loss": 30.32502818107605, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12902936339378357, "tokens_per_second": 1356037.3490003014, "eta_seconds": 14154.22621371746, "data_wait_seconds": 1.6927719116210938e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 35880, "total_steps": 72479, "tokens": 18811453440, "loss": 3.2263223826885223, "l_mix": 1.985054761171341, "l_a": 2.1008826196193695, "l_c": 2.0366760790348053, "z_loss": 32.84971237182617, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1433410793542862, "tokens_per_second": 1425399.043049448, "eta_seconds": 13461.785740327836, "data_wait_seconds": 2.911686897277832e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20045, "finemath": 0.08233}}
{"step": 35890, "total_steps": 72479, "tokens": 18816696320, "loss": 4.530888080596924, "l_mix": 2.807286560535431, "l_a": 2.895999103784561, "l_c": 2.849338948726654, "z_loss": 31.586532831192017, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.10387556254863739, "tokens_per_second": 1438544.7651569434, "eta_seconds": 13335.124562430381, "data_wait_seconds": 1.609325408935547e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 35900, "total_steps": 72479, "tokens": 18821939200, "loss": 4.553876101970673, "l_mix": 2.822258174419403, "l_a": 2.91103333234787, "l_c": 2.861026257276535, "z_loss": 30.684418201446533, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12585733830928802, "tokens_per_second": 1401928.0454186732, "eta_seconds": 13679.682644677163, "data_wait_seconds": 2.944469451904297e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.24208598, "gate_a2b/L1_absmax": 0.70939445, "gate_a2b/L1_rms": 0.25581351, "gate_a2b/L3_absmean": 0.24790366, "gate_a2b/L3_absmax": 0.61943787, "gate_a2b/L3_rms": 0.25411016, "gate_a2b/L5_absmean": 0.38920876, "gate_a2b/L5_absmax": 0.67376792, "gate_a2b/L5_rms": 0.39505589, "gate_c2b/L1_absmean": 0.01763465, "gate_c2b/L1_absmax": 0.5948593, "gate_c2b/L1_rms": 0.06733079, "gate_c2b/L3_absmean": 0.21287046, "gate_c2b/L3_absmax": 0.47668812, "gate_c2b/L3_rms": 0.217199, "gate_c2b/L5_absmean": 0.59142846, "gate_c2b/L5_absmax": 0.82984954, "gate_c2b/L5_rms": 0.60488886, "gate_b2a/L7_absmean": 0.0575169, "gate_b2a/L7_absmax": 0.3345145, "gate_b2a/L7_rms": 0.06320086, "gate_b2a/L11_absmean": 0.09167428, "gate_b2a/L11_absmax": 0.58622587, "gate_b2a/L11_rms": 0.09942282, "gate_b2a/L14_absmean": 0.23537034, "gate_b2a/L14_absmax": 0.74332762, "gate_b2a/L14_rms": 0.24237107, "gate_b2c/L3_absmean": 0.0474203, "gate_b2c/L3_absmax": 0.6334433, "gate_b2c/L3_rms": 0.05988526, "gate_b2c/L5_absmean": 0.05241758, "gate_b2c/L5_absmax": 0.69560045, "gate_b2c/L5_rms": 0.06428723, "gate_b2c/L6_absmean": 0.07627752, "gate_b2c/L6_absmax": 0.47331691, "gate_b2c/L6_rms": 0.08464251}, "streams": {"cos_a2b/round0": 0.384694, "cos_c2b/round0": 0.300326, "cos_b2a/round0": 0.138236, "cos_b2c/round0": 0.12683, "rms_a/round0": 3.390248, "rms_b/round0": 6.521619, "rms_c/round0": 2.905702, "cos_a2b/round1": 0.419381, "cos_c2b/round1": 0.296914, "cos_b2a/round1": 0.246386, "cos_b2c/round1": 0.118583, "rms_a/round1": 6.249228, "rms_b/round1": 14.322307, "rms_c/round1": 5.638868, "cos_a2b/round2": 0.557991, "cos_c2b/round2": 0.513932, "cos_b2a/round2": 0.100012, "cos_b2c/round2": 0.17303, "rms_a/round2": 16.054707, "rms_b/round2": 35.640003, "rms_c/round2": 9.998678}, "alpha_mean": 0.43506100093366484, "alpha_hist": [0.01195, 0.06963, 0.17537, 0.2009, 0.18068, 0.15744, 0.12209, 0.07133, 0.0105, 0.00012], "alpha_by_type": {"special": {"share": 0.00099, "mean": 0.56493, "std": 0.08869}, "whitespace": {"share": 0.02789, "mean": 0.52281, "std": 0.11996}, "digit": {"share": 0.03069, "mean": 0.43868, "std": 0.1117}, "word_start": {"share": 0.56052, "mean": 0.46288, "std": 0.16566}, "word_cont": {"share": 0.27689, "mean": 0.31544, "std": 0.13093}, "punct": {"share": 0.10303, "mean": 0.5791, "std": 0.16959}}, "alpha_p05": 0.15229, "alpha_p50": 0.4268, "alpha_p95": 0.7454}
{"step": 35910, "total_steps": 72479, "tokens": 18827182080, "loss": 4.543526709079742, "l_mix": 2.815354883670807, "l_a": 2.9030610024929047, "l_c": 2.8575113713741302, "z_loss": 31.503758668899536, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11250170320272446, "tokens_per_second": 1438955.5606616088, "eta_seconds": 13324.030565047266, "data_wait_seconds": 2.6613473892211914e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 35920, "total_steps": 72479, "tokens": 18832424960, "loss": 4.541493773460388, "l_mix": 2.813826620578766, "l_a": 2.907522052526474, "l_c": 2.8513681292533875, "z_loss": 31.910316944122314, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.146269753575325, "tokens_per_second": 1458701.1869629181, "eta_seconds": 13140.076366090774, "data_wait_seconds": 1.710653305053711e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 35930, "total_steps": 72479, "tokens": 18837667840, "loss": 4.576335847377777, "l_mix": 2.8361744582653046, "l_a": 2.9221377670764923, "l_c": 2.8783999383449554, "z_loss": 31.129183530807495, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1635012924671173, "tokens_per_second": 1444325.3952227554, "eta_seconds": 13267.233391714097, "data_wait_seconds": 1.621246337890625e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 35940, "total_steps": 72479, "tokens": 18842910720, "loss": 3.2773053646087646, "l_mix": 2.0150794237852097, "l_a": 2.135859489440918, "l_c": 2.071560174226761, "z_loss": 33.040701389312744, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2064184993505478, "tokens_per_second": 1403768.5684105875, "eta_seconds": 13646.807360625267, "data_wait_seconds": 3.567337989807129e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20045, "finemath": 0.08233}}
{"step": 35950, "total_steps": 72479, "tokens": 18848153600, "loss": 3.2540847659111023, "l_mix": 2.0016070008277893, "l_a": 2.1203107237815857, "l_c": 2.054614931344986, "z_loss": 32.87036752700806, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16378764808177948, "tokens_per_second": 1407640.1955773425, "eta_seconds": 13605.548074126242, "data_wait_seconds": 1.7762184143066406e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20045, "finemath": 0.08234}}
{"step": 35960, "total_steps": 72479, "tokens": 18853396480, "loss": 3.2442615628242493, "l_mix": 1.9958962500095367, "l_a": 2.115101933479309, "l_c": 2.0461156219244003, "z_loss": 32.23244905471802, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1660035103559494, "tokens_per_second": 1397977.966442272, "eta_seconds": 13695.83350496292, "data_wait_seconds": 2.0951032638549805e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 35970, "total_steps": 72479, "tokens": 18858639360, "loss": 4.7152493596076965, "l_mix": 2.924561530351639, "l_a": 3.003300756216049, "l_c": 2.9656587839126587, "z_loss": 30.837851762771606, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11727066338062286, "tokens_per_second": 1382272.2880104517, "eta_seconds": 13847.65560159683, "data_wait_seconds": 2.5987625122070312e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 35980, "total_steps": 72479, "tokens": 18863882240, "loss": 4.72906631231308, "l_mix": 2.9340686202049255, "l_a": 3.0104770958423615, "l_c": 2.9728482365608215, "z_loss": 30.633903980255127, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13800260424613953, "tokens_per_second": 1361127.3057460072, "eta_seconds": 14058.92573840618, "data_wait_seconds": 1.817941665649414e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 35990, "total_steps": 72479, "tokens": 18869125120, "loss": 3.211561232805252, "l_mix": 1.9760625660419464, "l_a": 2.092217892408371, "l_c": 2.0261109471321106, "z_loss": 33.59210681915283, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1969268023967743, "tokens_per_second": 1468411.6578446042, "eta_seconds": 13028.189152407645, "data_wait_seconds": 2.8342008590698242e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 36000, "total_steps": 72479, "tokens": 18874368000, "loss": 4.521285831928253, "l_mix": 2.8010511696338654, "l_a": 2.890320986509323, "l_c": 2.843794524669647, "z_loss": 31.278327465057373, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1454390287399292, "tokens_per_second": 1426182.4188991364, "eta_seconds": 13410.277464199065, "data_wait_seconds": 1.8268823623657227e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.24149995, "gate_a2b/L1_absmax": 0.70791996, "gate_a2b/L1_rms": 0.25513095, "gate_a2b/L3_absmean": 0.24658234, "gate_a2b/L3_absmax": 0.62056863, "gate_a2b/L3_rms": 0.25273958, "gate_a2b/L5_absmean": 0.38730678, "gate_a2b/L5_absmax": 0.67686129, "gate_a2b/L5_rms": 0.39320594, "gate_c2b/L1_absmean": 0.01757075, "gate_c2b/L1_absmax": 0.59408277, "gate_c2b/L1_rms": 0.06729577, "gate_c2b/L3_absmean": 0.21273223, "gate_c2b/L3_absmax": 0.47472364, "gate_c2b/L3_rms": 0.21708179, "gate_c2b/L5_absmean": 0.59085834, "gate_c2b/L5_absmax": 0.82922471, "gate_c2b/L5_rms": 0.60416824, "gate_b2a/L7_absmean": 0.05732475, "gate_b2a/L7_absmax": 0.33446324, "gate_b2a/L7_rms": 0.06304639, "gate_b2a/L11_absmean": 0.09166667, "gate_b2a/L11_absmax": 0.58809209, "gate_b2a/L11_rms": 0.09948176, "gate_b2a/L14_absmean": 0.23475438, "gate_b2a/L14_absmax": 0.74145502, "gate_b2a/L14_rms": 0.24173699, "gate_b2c/L3_absmean": 0.04682133, "gate_b2c/L3_absmax": 0.63129944, "gate_b2c/L3_rms": 0.05924729, "gate_b2c/L5_absmean": 0.05213311, "gate_b2c/L5_absmax": 0.69208169, "gate_b2c/L5_rms": 0.06392518, "gate_b2c/L6_absmean": 0.07554813, "gate_b2c/L6_absmax": 0.47333661, "gate_b2c/L6_rms": 0.08391541}, "streams": {"cos_a2b/round0": 0.384168, "cos_c2b/round0": 0.308289, "cos_b2a/round0": 0.136114, "cos_b2c/round0": 0.125663, "rms_a/round0": 3.417952, "rms_b/round0": 6.539993, "rms_c/round0": 2.932718, "cos_a2b/round1": 0.422167, "cos_c2b/round1": 0.291356, "cos_b2a/round1": 0.243745, "cos_b2c/round1": 0.11492, "rms_a/round1": 6.378995, "rms_b/round1": 14.532616, "rms_c/round1": 5.672024, "cos_a2b/round2": 0.547796, "cos_c2b/round2": 0.525646, "cos_b2a/round2": 0.090851, "cos_b2c/round2": 0.17022, "rms_a/round2": 16.214441, "rms_b/round2": 36.277275, "rms_c/round2": 10.309706}, "alpha_mean": 0.4308775631208661, "alpha_hist": [0.01047, 0.08156, 0.17781, 0.19572, 0.17499, 0.16129, 0.12042, 0.06602, 0.01146, 0.00026], "alpha_by_type": {"special": {"share": 0.00148, "mean": 0.56269, "std": 0.1138}, "whitespace": {"share": 0.02888, "mean": 0.53685, "std": 0.13531}, "digit": {"share": 0.03516, "mean": 0.49125, "std": 0.12176}, "word_start": {"share": 0.55988, "mean": 0.44133, "std": 0.17202}, "word_cont": {"share": 0.26968, "mean": 0.32863, "std": 0.13425}, "punct": {"share": 0.10492, "mean": 0.58664, "std": 0.15995}}, "alpha_p05": 0.1561, "alpha_p50": 0.41111, "alpha_p95": 0.72662}
{"step": 36010, "total_steps": 72479, "tokens": 18879610880, "loss": 4.450808227062225, "l_mix": 2.756760358810425, "l_a": 2.8494924902915955, "l_c": 2.797333687543869, "z_loss": 31.30738377571106, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14921364188194275, "tokens_per_second": 1368821.5110185542, "eta_seconds": 13968.409261608125, "data_wait_seconds": 1.996755599975586e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 36020, "total_steps": 72479, "tokens": 18884853760, "loss": 4.697848975658417, "l_mix": 2.912857949733734, "l_a": 2.997032403945923, "l_c": 2.9529371857643127, "z_loss": 30.51234769821167, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1080872043967247, "tokens_per_second": 1453523.8230643303, "eta_seconds": 13150.810388302803, "data_wait_seconds": 2.0951032638549805e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 36030, "total_steps": 72479, "tokens": 18890096640, "loss": 4.5986058712005615, "l_mix": 2.849532186985016, "l_a": 2.938987076282501, "l_c": 2.8912582993507385, "z_loss": 31.636255264282227, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14517392218112946, "tokens_per_second": 1259555.6132897006, "eta_seconds": 15171.837678599357, "data_wait_seconds": 1.9311904907226562e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 36040, "total_steps": 72479, "tokens": 18895339520, "loss": 4.491121292114258, "l_mix": 2.782592862844467, "l_a": 2.8704685270786285, "l_c": 2.8246256709098816, "z_loss": 30.73538589477539, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16233836114406586, "tokens_per_second": 1369545.8789293142, "eta_seconds": 13949.536650013923, "data_wait_seconds": 1.5676021575927734e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 36050, "total_steps": 72479, "tokens": 18900582400, "loss": 3.21005117893219, "l_mix": 1.9753513783216476, "l_a": 2.0915454626083374, "l_c": 2.0241202116012573, "z_loss": 31.82050609588623, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18024423718452454, "tokens_per_second": 1389469.8855977315, "eta_seconds": 13745.736953330039, "data_wait_seconds": 1.862645149230957e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 36060, "total_steps": 72479, "tokens": 18905825280, "loss": 3.2034003138542175, "l_mix": 1.9697225093841553, "l_a": 2.0862787067890167, "l_c": 2.0259803235530853, "z_loss": 33.28455400466919, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13563884794712067, "tokens_per_second": 1460106.9376270692, "eta_seconds": 13077.154953479767, "data_wait_seconds": 1.5616416931152344e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 36070, "total_steps": 72479, "tokens": 18911068160, "loss": 3.1830046474933624, "l_mix": 1.9580230712890625, "l_a": 2.074533611536026, "l_c": 2.0087383091449738, "z_loss": 33.352052211761475, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15492157638072968, "tokens_per_second": 1407957.3498770948, "eta_seconds": 13557.798319435118, "data_wait_seconds": 1.9788742065429688e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 36080, "total_steps": 72479, "tokens": 18916311040, "loss": 3.225231260061264, "l_mix": 1.9830626845359802, "l_a": 2.1061651408672333, "l_c": 2.034396529197693, "z_loss": 33.253268241882324, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1738273948431015, "tokens_per_second": 1423501.3085211285, "eta_seconds": 13406.07050921917, "data_wait_seconds": 2.053380012512207e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 36090, "total_steps": 72479, "tokens": 18921553920, "loss": 3.266829699277878, "l_mix": 2.009421393275261, "l_a": 2.126395344734192, "l_c": 2.064965605735779, "z_loss": 33.0465521812439, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15253441035747528, "tokens_per_second": 1439598.2990740058, "eta_seconds": 13252.5274892807, "data_wait_seconds": 2.321600914001465e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 36100, "total_steps": 72479, "tokens": 18926796800, "loss": 4.692843973636627, "l_mix": 2.9104581773281097, "l_a": 2.9898944199085236, "l_c": 2.951391339302063, "z_loss": 30.525546073913574, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14236615598201752, "tokens_per_second": 1477368.5888504528, "eta_seconds": 12910.165611982346, "data_wait_seconds": 2.1785497665405273e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.24063304, "gate_a2b/L1_absmax": 0.70805317, "gate_a2b/L1_rms": 0.25419301, "gate_a2b/L3_absmean": 0.24548547, "gate_a2b/L3_absmax": 0.62019372, "gate_a2b/L3_rms": 0.25172299, "gate_a2b/L5_absmean": 0.38593212, "gate_a2b/L5_absmax": 0.67797649, "gate_a2b/L5_rms": 0.39175892, "gate_c2b/L1_absmean": 0.01762902, "gate_c2b/L1_absmax": 0.59596115, "gate_c2b/L1_rms": 0.06731734, "gate_c2b/L3_absmean": 0.21242988, "gate_c2b/L3_absmax": 0.4746803, "gate_c2b/L3_rms": 0.21672842, "gate_c2b/L5_absmean": 0.5902943, "gate_c2b/L5_absmax": 0.83030283, "gate_c2b/L5_rms": 0.60358828, "gate_b2a/L7_absmean": 0.05767847, "gate_b2a/L7_absmax": 0.33271307, "gate_b2a/L7_rms": 0.0632544, "gate_b2a/L11_absmean": 0.09125733, "gate_b2a/L11_absmax": 0.58767462, "gate_b2a/L11_rms": 0.09909334, "gate_b2a/L14_absmean": 0.23431991, "gate_b2a/L14_absmax": 0.74146599, "gate_b2a/L14_rms": 0.24136578, "gate_b2c/L3_absmean": 0.04681613, "gate_b2c/L3_absmax": 0.63179958, "gate_b2c/L3_rms": 0.05935614, "gate_b2c/L5_absmean": 0.05189253, "gate_b2c/L5_absmax": 0.68694729, "gate_b2c/L5_rms": 0.06356154, "gate_b2c/L6_absmean": 0.07505164, "gate_b2c/L6_absmax": 0.4702633, "gate_b2c/L6_rms": 0.08340697}, "streams": {"cos_a2b/round0": 0.381128, "cos_c2b/round0": 0.304751, "cos_b2a/round0": 0.139561, "cos_b2c/round0": 0.124867, "rms_a/round0": 3.450522, "rms_b/round0": 6.61045, "rms_c/round0": 2.903708, "cos_a2b/round1": 0.423237, "cos_c2b/round1": 0.296957, "cos_b2a/round1": 0.245816, "cos_b2c/round1": 0.118593, "rms_a/round1": 6.403567, "rms_b/round1": 14.314887, "rms_c/round1": 5.722491, "cos_a2b/round2": 0.564075, "cos_c2b/round2": 0.519487, "cos_b2a/round2": 0.099663, "cos_b2c/round2": 0.173966, "rms_a/round2": 16.394075, "rms_b/round2": 35.784466, "rms_c/round2": 10.335164}, "alpha_mean": 0.46239749748555425, "alpha_hist": [0.00308, 0.03868, 0.13536, 0.18929, 0.21068, 0.21202, 0.13893, 0.0648, 0.00682, 0.00034], "alpha_by_type": {"special": {"share": 0.00075, "mean": 0.59699, "std": 0.13798}, "whitespace": {"share": 0.06833, "mean": 0.53485, "std": 0.10042}, "digit": {"share": 0.03447, "mean": 0.49357, "std": 0.12415}, "word_start": {"share": 0.51961, "mean": 0.48378, "std": 0.15163}, "word_cont": {"share": 0.25859, "mean": 0.35022, "std": 0.12532}, "punct": {"share": 0.11826, "mean": 0.56193, "std": 0.15007}}, "alpha_p05": 0.21867, "alpha_p50": 0.45774, "alpha_p95": 0.72584}
{"step": 36110, "total_steps": 72479, "tokens": 18932039680, "loss": 4.55487722158432, "l_mix": 2.821946293115616, "l_a": 2.911825180053711, "l_c": 2.864611119031906, "z_loss": 30.76277732849121, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1769072711467743, "tokens_per_second": 1463238.1800863175, "eta_seconds": 13031.255288100243, "data_wait_seconds": 1.7195940017700195e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 36120, "total_steps": 72479, "tokens": 18937282560, "loss": 4.474702775478363, "l_mix": 2.7718635201454163, "l_a": 2.86546128988266, "l_c": 2.8106696903705597, "z_loss": 31.329275131225586, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.103416807949543, "tokens_per_second": 1478348.3786588986, "eta_seconds": 12894.51638543606, "data_wait_seconds": 1.8984079360961914e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 36130, "total_steps": 72479, "tokens": 18942525440, "loss": 4.59756463766098, "l_mix": 2.850199729204178, "l_a": 2.933073341846466, "l_c": 2.8914759755134583, "z_loss": 31.850757837295532, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.10661936551332474, "tokens_per_second": 1378764.5269094496, "eta_seconds": 13822.044402837753, "data_wait_seconds": 2.4825334548950195e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 36140, "total_steps": 72479, "tokens": 18947768320, "loss": 4.502948820590973, "l_mix": 2.790032595396042, "l_a": 2.8792069256305695, "l_c": 2.8305134773254395, "z_loss": 31.433818578720093, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11467646807432175, "tokens_per_second": 1153260.848877017, "eta_seconds": 16520.2015229702, "data_wait_seconds": 1.7464160919189453e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 36150, "total_steps": 72479, "tokens": 18953011200, "loss": 4.351587533950806, "l_mix": 2.6963415443897247, "l_a": 2.7838984429836273, "l_c": 2.733588069677353, "z_loss": 32.370487451553345, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13673372566699982, "tokens_per_second": 1435894.4584671757, "eta_seconds": 13264.804136323928, "data_wait_seconds": 2.187490463256836e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 36160, "total_steps": 72479, "tokens": 18958254080, "loss": 3.274312287569046, "l_mix": 2.0135387778282166, "l_a": 2.1338937878608704, "l_c": 2.068684369325638, "z_loss": 33.88472890853882, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18874183297157288, "tokens_per_second": 1383664.8455329775, "eta_seconds": 13761.725560545921, "data_wait_seconds": 1.862645149230957e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 36170, "total_steps": 72479, "tokens": 18963496960, "loss": 3.2377461194992065, "l_mix": 1.9911849647760391, "l_a": 2.1088100969791412, "l_c": 2.0463936030864716, "z_loss": 32.893550395965576, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14572900533676147, "tokens_per_second": 1298584.1953475156, "eta_seconds": 14659.329029417037, "data_wait_seconds": 1.9252300262451172e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 36180, "total_steps": 72479, "tokens": 18968739840, "loss": 3.2088052332401276, "l_mix": 1.9733818620443344, "l_a": 2.092004880309105, "l_c": 2.0260729044675827, "z_loss": 32.304479122161865, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16384167969226837, "tokens_per_second": 1476231.5178822512, "eta_seconds": 12891.697461724281, "data_wait_seconds": 2.232193946838379e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 36190, "total_steps": 72479, "tokens": 18973982720, "loss": 3.225212126970291, "l_mix": 1.984473094344139, "l_a": 2.1020030975341797, "l_c": 2.033793717622757, "z_loss": 32.94335985183716, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16611139476299286, "tokens_per_second": 1403744.9115117036, "eta_seconds": 13553.664256215096, "data_wait_seconds": 1.9669532775878906e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 36200, "total_steps": 72479, "tokens": 18979225600, "loss": 3.2112121284008026, "l_mix": 1.9765792042016983, "l_a": 2.086522489786148, "l_c": 2.028920352458954, "z_loss": 33.32164144515991, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20857656002044678, "tokens_per_second": 1395202.1134468145, "eta_seconds": 13632.895312213897, "data_wait_seconds": 1.823902130126953e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.2400708, "gate_a2b/L1_absmax": 0.70790046, "gate_a2b/L1_rms": 0.25356746, "gate_a2b/L3_absmean": 0.24453104, "gate_a2b/L3_absmax": 0.61978763, "gate_a2b/L3_rms": 0.25073513, "gate_a2b/L5_absmean": 0.38453081, "gate_a2b/L5_absmax": 0.6774404, "gate_a2b/L5_rms": 0.39038509, "gate_c2b/L1_absmean": 0.01742713, "gate_c2b/L1_absmax": 0.59547812, "gate_c2b/L1_rms": 0.06724499, "gate_c2b/L3_absmean": 0.21229473, "gate_c2b/L3_absmax": 0.47673061, "gate_c2b/L3_rms": 0.21654083, "gate_c2b/L5_absmean": 0.59057134, "gate_c2b/L5_absmax": 0.82820791, "gate_c2b/L5_rms": 0.60383123, "gate_b2a/L7_absmean": 0.05724471, "gate_b2a/L7_absmax": 0.3321563, "gate_b2a/L7_rms": 0.06282865, "gate_b2a/L11_absmean": 0.09133735, "gate_b2a/L11_absmax": 0.58807814, "gate_b2a/L11_rms": 0.0991518, "gate_b2a/L14_absmean": 0.23378181, "gate_b2a/L14_absmax": 0.74182522, "gate_b2a/L14_rms": 0.24083981, "gate_b2c/L3_absmean": 0.04642376, "gate_b2c/L3_absmax": 0.62933904, "gate_b2c/L3_rms": 0.05893446, "gate_b2c/L5_absmean": 0.05161544, "gate_b2c/L5_absmax": 0.68822068, "gate_b2c/L5_rms": 0.0633517, "gate_b2c/L6_absmean": 0.0749305, "gate_b2c/L6_absmax": 0.46921083, "gate_b2c/L6_rms": 0.08322035}, "streams": {"cos_a2b/round0": 0.381677, "cos_c2b/round0": 0.27472, "cos_b2a/round0": 0.126996, "cos_b2c/round0": 0.103869, "rms_a/round0": 3.62078, "rms_b/round0": 6.715355, "rms_c/round0": 2.974569, "cos_a2b/round1": 0.419865, "cos_c2b/round1": 0.293138, "cos_b2a/round1": 0.234074, "cos_b2c/round1": 0.120902, "rms_a/round1": 6.819828, "rms_b/round1": 15.369499, "rms_c/round1": 5.975592, "cos_a2b/round2": 0.524977, "cos_c2b/round2": 0.50918, "cos_b2a/round2": 0.080632, "cos_b2c/round2": 0.158459, "rms_a/round2": 16.935295, "rms_b/round2": 37.970036, "rms_c/round2": 11.065731}, "alpha_mean": 0.4193728218487749, "alpha_hist": [0.03056, 0.1432, 0.18716, 0.16039, 0.1261, 0.11006, 0.10747, 0.08884, 0.04401, 0.00221], "alpha_by_type": {"special": {"share": 0.00108, "mean": 0.58739, "std": 0.10933}, "whitespace": {"share": 0.04329, "mean": 0.58604, "std": 0.1675}, "digit": {"share": 0.00916, "mean": 0.37058, "std": 0.17288}, "word_start": {"share": 0.55698, "mean": 0.42463, "std": 0.2046}, "word_cont": {"share": 0.26028, "mean": 0.28011, "std": 0.14243}, "punct": {"share": 0.12921, "mean": 0.62345, "std": 0.18077}}, "alpha_p05": 0.1023, "alpha_p50": 0.37731, "alpha_p95": 0.80069}
{"step": 36210, "total_steps": 72479, "tokens": 18984468480, "loss": 3.241487503051758, "l_mix": 1.9939980953931808, "l_a": 2.1121115684509277, "l_c": 2.0461863428354263, "z_loss": 33.154300689697266, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15403394401073456, "tokens_per_second": 1468016.1160643059, "eta_seconds": 12953.128554868697, "data_wait_seconds": 2.8133392333984375e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 36220, "total_steps": 72479, "tokens": 18989711360, "loss": 4.592345416545868, "l_mix": 2.8457589745521545, "l_a": 2.9311799108982086, "l_c": 2.890774667263031, "z_loss": 31.224159479141235, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19757366180419922, "tokens_per_second": 1405270.7996903197, "eta_seconds": 13527.75464785099, "data_wait_seconds": 1.8924474716186523e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 36230, "total_steps": 72479, "tokens": 18994954240, "loss": 4.596856117248535, "l_mix": 2.849333703517914, "l_a": 2.9357755184173584, "l_c": 2.889299124479294, "z_loss": 32.12085008621216, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17222128808498383, "tokens_per_second": 1376748.4210459096, "eta_seconds": 13804.203746652602, "data_wait_seconds": 3.30507755279541e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 36240, "total_steps": 72479, "tokens": 19000197120, "loss": 4.475624263286591, "l_mix": 2.7728081941604614, "l_a": 2.8628930747509003, "l_c": 2.813160240650177, "z_loss": 31.2693088054657, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1330713927745819, "tokens_per_second": 1472834.4325571866, "eta_seconds": 12900.073770689964, "data_wait_seconds": 2.0265579223632812e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08234}}
{"step": 36250, "total_steps": 72479, "tokens": 19005440000, "loss": 4.483342945575714, "l_mix": 2.777615547180176, "l_a": 2.8677492439746857, "l_c": 2.8180083334445953, "z_loss": 31.186642169952393, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12118837982416153, "tokens_per_second": 1438870.0687581557, "eta_seconds": 13200.934792113305, "data_wait_seconds": 3.3855438232421875e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08234}}
{"step": 36260, "total_steps": 72479, "tokens": 19010682880, "loss": 4.553776741027832, "l_mix": 2.82249054312706, "l_a": 2.9106483161449432, "l_c": 2.860305517911911, "z_loss": 31.909590005874634, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17431634664535522, "tokens_per_second": 1408703.4373999056, "eta_seconds": 13479.903979682922, "data_wait_seconds": 1.9788742065429688e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 36270, "total_steps": 72479, "tokens": 19015925760, "loss": 3.2466919124126434, "l_mix": 1.9956977814435959, "l_a": 2.121220052242279, "l_c": 2.048760339617729, "z_loss": 33.815213680267334, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2083413004875183, "tokens_per_second": 1272164.884450584, "eta_seconds": 14922.550075101852, "data_wait_seconds": 3.5196542739868164e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 36280, "total_steps": 72479, "tokens": 19021168640, "loss": 3.2355010211467743, "l_mix": 1.9904108345508575, "l_a": 2.109586924314499, "l_c": 2.0407134145498276, "z_loss": 32.59254169464111, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13007500767707825, "tokens_per_second": 1135178.4229962518, "eta_seconds": 16718.694548392297, "data_wait_seconds": 1.7851591110229492e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 36290, "total_steps": 72479, "tokens": 19026411520, "loss": 3.2330203652381897, "l_mix": 1.989459365606308, "l_a": 2.102330595254898, "l_c": 2.042872503399849, "z_loss": 33.06184148788452, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2037767469882965, "tokens_per_second": 1464908.716335851, "eta_seconds": 12951.973198342324, "data_wait_seconds": 2.1249055862426758e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 36300, "total_steps": 72479, "tokens": 19031654400, "loss": 3.2176381945610046, "l_mix": 1.9789411425590515, "l_a": 2.095328986644745, "l_c": 2.03366120159626, "z_loss": 33.451212882995605, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16614064574241638, "tokens_per_second": 1318824.8870071694, "eta_seconds": 14382.664248204232, "data_wait_seconds": 1.7017126083374023e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.23791774, "gate_a2b/L1_absmax": 0.70784783, "gate_a2b/L1_rms": 0.25138968, "gate_a2b/L3_absmean": 0.24355529, "gate_a2b/L3_absmax": 0.6232754, "gate_a2b/L3_rms": 0.24972573, "gate_a2b/L5_absmean": 0.38257033, "gate_a2b/L5_absmax": 0.68000883, "gate_a2b/L5_rms": 0.38848144, "gate_c2b/L1_absmean": 0.01752524, "gate_c2b/L1_absmax": 0.5978018, "gate_c2b/L1_rms": 0.06783147, "gate_c2b/L3_absmean": 0.21182482, "gate_c2b/L3_absmax": 0.47619802, "gate_c2b/L3_rms": 0.21617031, "gate_c2b/L5_absmean": 0.590424, "gate_c2b/L5_absmax": 0.82677537, "gate_c2b/L5_rms": 0.60371667, "gate_b2a/L7_absmean": 0.05723376, "gate_b2a/L7_absmax": 0.33299565, "gate_b2a/L7_rms": 0.06285623, "gate_b2a/L11_absmean": 0.09116005, "gate_b2a/L11_absmax": 0.59308404, "gate_b2a/L11_rms": 0.09918004, "gate_b2a/L14_absmean": 0.23353621, "gate_b2a/L14_absmax": 0.74424076, "gate_b2a/L14_rms": 0.24069628, "gate_b2c/L3_absmean": 0.04586966, "gate_b2c/L3_absmax": 0.62868226, "gate_b2c/L3_rms": 0.05837449, "gate_b2c/L5_absmean": 0.05102564, "gate_b2c/L5_absmax": 0.68255436, "gate_b2c/L5_rms": 0.06269763, "gate_b2c/L6_absmean": 0.07426579, "gate_b2c/L6_absmax": 0.4700872, "gate_b2c/L6_rms": 0.08262375}, "streams": {"cos_a2b/round0": 0.377747, "cos_c2b/round0": 0.271407, "cos_b2a/round0": 0.126207, "cos_b2c/round0": 0.10291, "rms_a/round0": 3.655883, "rms_b/round0": 6.811709, "rms_c/round0": 3.001566, "cos_a2b/round1": 0.421429, "cos_c2b/round1": 0.295606, "cos_b2a/round1": 0.232109, "cos_b2c/round1": 0.123426, "rms_a/round1": 6.95822, "rms_b/round1": 15.650011, "rms_c/round1": 6.023427, "cos_a2b/round2": 0.520876, "cos_c2b/round2": 0.509246, "cos_b2a/round2": 0.077511, "cos_b2c/round2": 0.155559, "rms_a/round2": 17.190821, "rms_b/round2": 38.578461, "rms_c/round2": 11.18105}, "alpha_mean": 0.42147329179645254, "alpha_hist": [0.03902, 0.15666, 0.18257, 0.14725, 0.11378, 0.10101, 0.09785, 0.09598, 0.06075, 0.00513], "alpha_by_type": {"special": {"share": 0.00111, "mean": 0.60314, "std": 0.11284}, "whitespace": {"share": 0.0394, "mean": 0.61606, "std": 0.18433}, "digit": {"share": 0.01135, "mean": 0.34607, "std": 0.19002}, "word_start": {"share": 0.55995, "mean": 0.43308, "std": 0.2173}, "word_cont": {"share": 0.25781, "mean": 0.26655, "std": 0.14528}, "punct": {"share": 0.13037, "mean": 0.62419, "std": 0.19805}}, "alpha_p05": 0.1188, "alpha_p50": 0.38839, "alpha_p95": 0.82133}
{"step": 36310, "total_steps": 72479, "tokens": 19036897280, "loss": 3.2276230454444885, "l_mix": 1.9856336116790771, "l_a": 2.1052325665950775, "l_c": 2.0347321331501007, "z_loss": 33.21606206893921, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17912515997886658, "tokens_per_second": 1100281.8250535375, "eta_seconds": 17234.650468826294, "data_wait_seconds": 2.5272369384765625e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 36320, "total_steps": 72479, "tokens": 19042140160, "loss": 3.241576671600342, "l_mix": 1.9942401349544525, "l_a": 2.1118001341819763, "l_c": 2.045988142490387, "z_loss": 33.18136405944824, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15408039093017578, "tokens_per_second": 1449752.5660164452, "eta_seconds": 13076.527840948105, "data_wait_seconds": 1.659989356994629e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 36330, "total_steps": 72479, "tokens": 19047383040, "loss": 4.567868888378143, "l_mix": 2.829955667257309, "l_a": 2.920584738254547, "l_c": 2.8724589347839355, "z_loss": 32.05843257904053, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13334780931472778, "tokens_per_second": 1395673.8221973514, "eta_seconds": 13579.452885460852, "data_wait_seconds": 1.582503318786621e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 36340, "total_steps": 72479, "tokens": 19052625920, "loss": 4.504817187786102, "l_mix": 2.791807919740677, "l_a": 2.880187749862671, "l_c": 2.829842507839203, "z_loss": 31.672054767608643, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15185722708702087, "tokens_per_second": 1418605.3351072916, "eta_seconds": 13356.24755039215, "data_wait_seconds": 1.659989356994629e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 36350, "total_steps": 72479, "tokens": 19057868800, "loss": 4.571482181549072, "l_mix": 2.832325577735901, "l_a": 2.924360305070877, "l_c": 2.872827708721161, "z_loss": 30.82016921043396, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16675083339214325, "tokens_per_second": 1469712.9094580873, "eta_seconds": 12888.23213710785, "data_wait_seconds": 1.6063451766967773e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 36360, "total_steps": 72479, "tokens": 19063111680, "loss": 3.2138445377349854, "l_mix": 1.9759768843650818, "l_a": 2.094886064529419, "l_c": 2.0313393473625183, "z_loss": 31.675106287002563, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2629672586917877, "tokens_per_second": 1465469.2744537685, "eta_seconds": 12921.97564432621, "data_wait_seconds": 2.0831823348999023e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 36370, "total_steps": 72479, "tokens": 19068354560, "loss": 3.2128952145576477, "l_mix": 1.9766930043697357, "l_a": 2.0914146900177, "l_c": 2.029259070754051, "z_loss": 34.1112174987793, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20220579206943512, "tokens_per_second": 1400736.245397732, "eta_seconds": 13515.403384613992, "data_wait_seconds": 1.6391277313232422e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 36380, "total_steps": 72479, "tokens": 19073597440, "loss": 4.805436909198761, "l_mix": 2.980605721473694, "l_a": 3.0614453554153442, "l_c": 3.0213250517845154, "z_loss": 31.251539707183838, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12190262228250504, "tokens_per_second": 1458650.9694434095, "eta_seconds": 12975.189341711997, "data_wait_seconds": 2.384185791015625e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 36390, "total_steps": 72479, "tokens": 19078840320, "loss": 3.205172121524811, "l_mix": 1.97145576775074, "l_a": 2.0886649787425995, "l_c": 2.023722693324089, "z_loss": 33.10255479812622, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13184255361557007, "tokens_per_second": 1438565.9393003555, "eta_seconds": 13152.702364969253, "data_wait_seconds": 1.990795135498047e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 36400, "total_steps": 72479, "tokens": 19084083200, "loss": 3.198435515165329, "l_mix": 1.9678022116422653, "l_a": 2.0855543464422226, "l_c": 2.0165566504001617, "z_loss": 33.66125297546387, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17277665436267853, "tokens_per_second": 1329144.0587447658, "eta_seconds": 14231.554982733725, "data_wait_seconds": 2.0503997802734375e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.23745237, "gate_a2b/L1_absmax": 0.71485084, "gate_a2b/L1_rms": 0.2510505, "gate_a2b/L3_absmean": 0.2430595, "gate_a2b/L3_absmax": 0.62334251, "gate_a2b/L3_rms": 0.24922186, "gate_a2b/L5_absmean": 0.3811537, "gate_a2b/L5_absmax": 0.6801455, "gate_a2b/L5_rms": 0.38702446, "gate_c2b/L1_absmean": 0.01932201, "gate_c2b/L1_absmax": 0.60335267, "gate_c2b/L1_rms": 0.06768867, "gate_c2b/L3_absmean": 0.21209908, "gate_c2b/L3_absmax": 0.47877496, "gate_c2b/L3_rms": 0.21651472, "gate_c2b/L5_absmean": 0.59070826, "gate_c2b/L5_absmax": 0.82865858, "gate_c2b/L5_rms": 0.60390311, "gate_b2a/L7_absmean": 0.05734153, "gate_b2a/L7_absmax": 0.3324649, "gate_b2a/L7_rms": 0.06299946, "gate_b2a/L11_absmean": 0.09096475, "gate_b2a/L11_absmax": 0.58592141, "gate_b2a/L11_rms": 0.09881835, "gate_b2a/L14_absmean": 0.23314175, "gate_b2a/L14_absmax": 0.7423175, "gate_b2a/L14_rms": 0.24027711, "gate_b2c/L3_absmean": 0.04600804, "gate_b2c/L3_absmax": 0.6272698, "gate_b2c/L3_rms": 0.05843552, "gate_b2c/L5_absmean": 0.05044737, "gate_b2c/L5_absmax": 0.67818075, "gate_b2c/L5_rms": 0.06218315, "gate_b2c/L6_absmean": 0.0739585, "gate_b2c/L6_absmax": 0.47065696, "gate_b2c/L6_rms": 0.08233285}, "streams": {"cos_a2b/round0": 0.380833, "cos_c2b/round0": 0.272757, "cos_b2a/round0": 0.129687, "cos_b2c/round0": 0.102842, "rms_a/round0": 3.697843, "rms_b/round0": 6.823537, "rms_c/round0": 3.017745, "cos_a2b/round1": 0.425266, "cos_c2b/round1": 0.29755, "cos_b2a/round1": 0.233023, "cos_b2c/round1": 0.123048, "rms_a/round1": 6.982813, "rms_b/round1": 15.646638, "rms_c/round1": 6.093668, "cos_a2b/round2": 0.533617, "cos_c2b/round2": 0.502233, "cos_b2a/round2": 0.079613, "cos_b2c/round2": 0.161013, "rms_a/round2": 17.336834, "rms_b/round2": 38.512096, "rms_c/round2": 11.144081}, "alpha_mean": 0.44570381743308474, "alpha_hist": [0.02365, 0.12032, 0.17604, 0.15616, 0.13284, 0.11166, 0.10649, 0.10283, 0.06425, 0.00575], "alpha_by_type": {"special": {"share": 0.00114, "mean": 0.60029, "std": 0.12596}, "whitespace": {"share": 0.03821, "mean": 0.62953, "std": 0.16858}, "digit": {"share": 0.00983, "mean": 0.3991, "std": 0.17319}, "word_start": {"share": 0.57631, "mean": 0.45164, "std": 0.20993}, "word_cont": {"share": 0.24966, "mean": 0.29975, "std": 0.14817}, "punct": {"share": 0.12485, "mean": 0.65618, "std": 0.18172}}, "alpha_p05": 0.13916, "alpha_p50": 0.44927, "alpha_p95": 0.82699}
{"step": 36410, "total_steps": 72479, "tokens": 19089326080, "loss": 3.241027444601059, "l_mix": 1.9937792867422104, "l_a": 2.111327439546585, "l_c": 2.0461662113666534, "z_loss": 32.57352018356323, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16500075161457062, "tokens_per_second": 1428184.7927868012, "eta_seconds": 13240.964311838152, "data_wait_seconds": 2.6911497116088867e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 36420, "total_steps": 72479, "tokens": 19094568960, "loss": 3.2391136586666107, "l_mix": 1.9923601746559143, "l_a": 2.112548589706421, "l_c": 2.043296068906784, "z_loss": 31.745492219924927, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17935845255851746, "tokens_per_second": 1432153.4759618936, "eta_seconds": 13200.611044359206, "data_wait_seconds": 3.743171691894531e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 36430, "total_steps": 72479, "tokens": 19099811840, "loss": 3.233145594596863, "l_mix": 1.987983837723732, "l_a": 2.111657530069351, "l_c": 2.0388814955949783, "z_loss": 33.5806565284729, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1809178590774536, "tokens_per_second": 1411264.192185581, "eta_seconds": 13392.289138102531, "data_wait_seconds": 2.777576446533203e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 36440, "total_steps": 72479, "tokens": 19105054720, "loss": 4.269472122192383, "l_mix": 2.6443073749542236, "l_a": 2.7328540980815887, "l_c": 2.684361606836319, "z_loss": 32.14163017272949, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1751113384962082, "tokens_per_second": 1395425.1325224217, "eta_seconds": 13540.543875575066, "data_wait_seconds": 1.817941665649414e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 36450, "total_steps": 72479, "tokens": 19110297600, "loss": 4.411506295204163, "l_mix": 2.73234361410141, "l_a": 2.8226961493492126, "l_c": 2.7745128870010376, "z_loss": 32.385557413101196, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17093470692634583, "tokens_per_second": 1442665.9552161598, "eta_seconds": 13093.517791628838, "data_wait_seconds": 1.77919864654541e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 36460, "total_steps": 72479, "tokens": 19115540480, "loss": 4.47649621963501, "l_mix": 2.7734139263629913, "l_a": 2.8649829626083374, "l_c": 2.8119577765464783, "z_loss": 31.772420167922974, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12514351308345795, "tokens_per_second": 1359663.360775283, "eta_seconds": 13888.974298191071, "data_wait_seconds": 1.7344951629638672e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 36470, "total_steps": 72479, "tokens": 19120783360, "loss": 3.235239088535309, "l_mix": 1.990661233663559, "l_a": 2.108457177877426, "l_c": 2.0401356518268585, "z_loss": 33.04438638687134, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18114961683750153, "tokens_per_second": 1446180.3686491142, "eta_seconds": 13054.448118138313, "data_wait_seconds": 2.619624137878418e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 36480, "total_steps": 72479, "tokens": 19126026240, "loss": 3.244940459728241, "l_mix": 1.9955433905124664, "l_a": 2.1150496006011963, "l_c": 2.0496071577072144, "z_loss": 32.87259769439697, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1630362570285797, "tokens_per_second": 1478038.3604043154, "eta_seconds": 12769.522238135338, "data_wait_seconds": 1.958012580871582e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 36490, "total_steps": 72479, "tokens": 19131269120, "loss": 3.2597220838069916, "l_mix": 2.0054730027914047, "l_a": 2.124056428670883, "l_c": 2.056773528456688, "z_loss": 32.56309652328491, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16101104021072388, "tokens_per_second": 1402684.4861751245, "eta_seconds": 13451.778370666505, "data_wait_seconds": 2.2143125534057617e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 36500, "total_steps": 72479, "tokens": 19136512000, "loss": 4.667575776576996, "l_mix": 2.8938437700271606, "l_a": 2.9767025113105774, "l_c": 2.935736984014511, "z_loss": 31.659133434295654, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13879640400409698, "tokens_per_second": 1452940.684209556, "eta_seconds": 12982.882341313361, "data_wait_seconds": 2.0295381546020508e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.23610802, "gate_a2b/L1_absmax": 0.7109713, "gate_a2b/L1_rms": 0.24957055, "gate_a2b/L3_absmean": 0.24132256, "gate_a2b/L3_absmax": 0.61692941, "gate_a2b/L3_rms": 0.2475339, "gate_a2b/L5_absmean": 0.38010865, "gate_a2b/L5_absmax": 0.68137717, "gate_a2b/L5_rms": 0.38602069, "gate_c2b/L1_absmean": 0.01683732, "gate_c2b/L1_absmax": 0.59944814, "gate_c2b/L1_rms": 0.06741129, "gate_c2b/L3_absmean": 0.21190646, "gate_c2b/L3_absmax": 0.47496253, "gate_c2b/L3_rms": 0.21622358, "gate_c2b/L5_absmean": 0.58964932, "gate_c2b/L5_absmax": 0.81996047, "gate_c2b/L5_rms": 0.60281879, "gate_b2a/L7_absmean": 0.05736915, "gate_b2a/L7_absmax": 0.33389315, "gate_b2a/L7_rms": 0.06289797, "gate_b2a/L11_absmean": 0.09066574, "gate_b2a/L11_absmax": 0.58438331, "gate_b2a/L11_rms": 0.09846412, "gate_b2a/L14_absmean": 0.23205474, "gate_b2a/L14_absmax": 0.75341022, "gate_b2a/L14_rms": 0.23931944, "gate_b2c/L3_absmean": 0.04555136, "gate_b2c/L3_absmax": 0.62810296, "gate_b2c/L3_rms": 0.05805272, "gate_b2c/L5_absmean": 0.05036504, "gate_b2c/L5_absmax": 0.67331618, "gate_b2c/L5_rms": 0.06187623, "gate_b2c/L6_absmean": 0.07339089, "gate_b2c/L6_absmax": 0.4774141, "gate_b2c/L6_rms": 0.08184419}, "streams": {"cos_a2b/round0": 0.382264, "cos_c2b/round0": 0.305661, "cos_b2a/round0": 0.136908, "cos_b2c/round0": 0.120443, "rms_a/round0": 3.584973, "rms_b/round0": 6.797229, "rms_c/round0": 2.969377, "cos_a2b/round1": 0.413806, "cos_c2b/round1": 0.286283, "cos_b2a/round1": 0.234424, "cos_b2c/round1": 0.115818, "rms_a/round1": 6.683308, "rms_b/round1": 14.987779, "rms_c/round1": 5.83781, "cos_a2b/round2": 0.532287, "cos_c2b/round2": 0.538603, "cos_b2a/round2": 0.095168, "cos_b2c/round2": 0.176231, "rms_a/round2": 16.878185, "rms_b/round2": 37.774815, "rms_c/round2": 10.582742}, "alpha_mean": 0.444187278208517, "alpha_hist": [0.00681, 0.05682, 0.15582, 0.22157, 0.17619, 0.16342, 0.15823, 0.05293, 0.00807, 0.00012], "alpha_by_type": {"special": {"share": 0.00058, "mean": 0.54305, "std": 0.17474}, "whitespace": {"share": 0.13289, "mean": 0.59023, "std": 0.10394}, "digit": {"share": 0.03818, "mean": 0.48388, "std": 0.15915}, "word_start": {"share": 0.47447, "mean": 0.44181, "std": 0.15512}, "word_cont": {"share": 0.27148, "mean": 0.33114, "std": 0.1191}, "punct": {"share": 0.0824, "mean": 0.57574, "std": 0.1682}}, "alpha_p05": 0.19745, "alpha_p50": 0.4442, "alpha_p95": 0.72971}
{"step": 36510, "total_steps": 72479, "tokens": 19141754880, "loss": 4.766777873039246, "l_mix": 2.957006722688675, "l_a": 3.0374418795108795, "l_c": 2.9951282143592834, "z_loss": 31.639392137527466, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16542059183120728, "tokens_per_second": 1444243.6273026543, "eta_seconds": 13057.433465862274, "data_wait_seconds": 2.092123031616211e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 36520, "total_steps": 72479, "tokens": 19146997760, "loss": 3.2435469329357147, "l_mix": 1.9956566393375397, "l_a": 2.1135465800762177, "l_c": 2.0460874140262604, "z_loss": 32.985671520233154, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19976000487804413, "tokens_per_second": 1231558.0503796565, "eta_seconds": 15308.14742040634, "data_wait_seconds": 2.0891427993774414e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 36530, "total_steps": 72479, "tokens": 19152240640, "loss": 3.242097169160843, "l_mix": 1.9948499500751495, "l_a": 2.11042982339859, "l_c": 2.0470606684684753, "z_loss": 33.33630895614624, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17484599351882935, "tokens_per_second": 1248884.9492395257, "eta_seconds": 15091.56573908329, "data_wait_seconds": 2.3871660232543945e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 36540, "total_steps": 72479, "tokens": 19157483520, "loss": 3.2575968503952026, "l_mix": 2.0058208405971527, "l_a": 2.1174354255199432, "l_c": 2.05515119433403, "z_loss": 33.684364318847656, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.24783015251159668, "tokens_per_second": 1303317.8218334187, "eta_seconds": 14457.246050310134, "data_wait_seconds": 2.22623348236084e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 36550, "total_steps": 72479, "tokens": 19162726400, "loss": 4.435966908931732, "l_mix": 2.748326987028122, "l_a": 2.83522492647171, "l_c": 2.790241152048111, "z_loss": 31.444243669509888, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16625867784023285, "tokens_per_second": 1450858.386141315, "eta_seconds": 12983.447407364845, "data_wait_seconds": 1.7255544662475586e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 36560, "total_steps": 72479, "tokens": 19167969280, "loss": 4.433735191822052, "l_mix": 2.7476642727851868, "l_a": 2.8321157693862915, "l_c": 2.7881202399730682, "z_loss": 31.652747869491577, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13775400817394257, "tokens_per_second": 1369422.127131596, "eta_seconds": 13751.713440942765, "data_wait_seconds": 2.086162567138672e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 36570, "total_steps": 72479, "tokens": 19173212160, "loss": 4.470374524593353, "l_mix": 2.7688297033309937, "l_a": 2.8609228134155273, "l_c": 2.8108929097652435, "z_loss": 32.64636516571045, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16011500358581543, "tokens_per_second": 1473982.5760559258, "eta_seconds": 12772.646093535423, "data_wait_seconds": 1.582503318786621e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 36580, "total_steps": 72479, "tokens": 19178455040, "loss": 3.2053189277648926, "l_mix": 1.9704322665929794, "l_a": 2.093877375125885, "l_c": 2.0224111825227737, "z_loss": 34.528746128082275, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19756650924682617, "tokens_per_second": 1440266.6092958893, "eta_seconds": 13068.00754146576, "data_wait_seconds": 2.059340476989746e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 36590, "total_steps": 72479, "tokens": 19183697920, "loss": 3.1853219866752625, "l_mix": 1.9596534967422485, "l_a": 2.0774355232715607, "l_c": 2.008125811815262, "z_loss": 33.046000957489014, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17144298553466797, "tokens_per_second": 1359197.445077013, "eta_seconds": 13843.589906787873, "data_wait_seconds": 1.9043684005737305e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 36600, "total_steps": 72479, "tokens": 19188940800, "loss": 3.2549712657928467, "l_mix": 2.0027677565813065, "l_a": 2.118191510438919, "l_c": 2.055819869041443, "z_loss": 34.73804330825806, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2295069545507431, "tokens_per_second": 1336508.0516933238, "eta_seconds": 14074.684494543077, "data_wait_seconds": 1.901388168334961e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.23600936, "gate_a2b/L1_absmax": 0.71023649, "gate_a2b/L1_rms": 0.24959919, "gate_a2b/L3_absmean": 0.24038696, "gate_a2b/L3_absmax": 0.61514044, "gate_a2b/L3_rms": 0.24651194, "gate_a2b/L5_absmean": 0.37831837, "gate_a2b/L5_absmax": 0.68132752, "gate_a2b/L5_rms": 0.38424352, "gate_c2b/L1_absmean": 0.01716799, "gate_c2b/L1_absmax": 0.60121781, "gate_c2b/L1_rms": 0.06763966, "gate_c2b/L3_absmean": 0.21225853, "gate_c2b/L3_absmax": 0.47336739, "gate_c2b/L3_rms": 0.21657878, "gate_c2b/L5_absmean": 0.58952647, "gate_c2b/L5_absmax": 0.83645213, "gate_c2b/L5_rms": 0.60282052, "gate_b2a/L7_absmean": 0.05709931, "gate_b2a/L7_absmax": 0.33212197, "gate_b2a/L7_rms": 0.06264525, "gate_b2a/L11_absmean": 0.09010216, "gate_b2a/L11_absmax": 0.5848369, "gate_b2a/L11_rms": 0.09799072, "gate_b2a/L14_absmean": 0.23161221, "gate_b2a/L14_absmax": 0.75203371, "gate_b2a/L14_rms": 0.23888448, "gate_b2c/L3_absmean": 0.04524119, "gate_b2c/L3_absmax": 0.6264053, "gate_b2c/L3_rms": 0.05769178, "gate_b2c/L5_absmean": 0.05014433, "gate_b2c/L5_absmax": 0.67446661, "gate_b2c/L5_rms": 0.06170078, "gate_b2c/L6_absmean": 0.07311113, "gate_b2c/L6_absmax": 0.4718003, "gate_b2c/L6_rms": 0.08149688}, "streams": {"cos_a2b/round0": 0.381286, "cos_c2b/round0": 0.271138, "cos_b2a/round0": 0.130896, "cos_b2c/round0": 0.100133, "rms_a/round0": 3.731824, "rms_b/round0": 6.939244, "rms_c/round0": 3.070011, "cos_a2b/round1": 0.422099, "cos_c2b/round1": 0.298433, "cos_b2a/round1": 0.237596, "cos_b2c/round1": 0.125146, "rms_a/round1": 7.065562, "rms_b/round1": 15.94922, "rms_c/round1": 6.114055, "cos_a2b/round2": 0.523622, "cos_c2b/round2": 0.508067, "cos_b2a/round2": 0.083571, "cos_b2c/round2": 0.161699, "rms_a/round2": 17.490391, "rms_b/round2": 39.195633, "rms_c/round2": 11.228285}, "alpha_mean": 0.41104369976868327, "alpha_hist": [0.04036, 0.16785, 0.18866, 0.14458, 0.11525, 0.09984, 0.09518, 0.0914, 0.05211, 0.00478], "alpha_by_type": {"special": {"share": 0.00099, "mean": 0.61944, "std": 0.1117}, "whitespace": {"share": 0.03305, "mean": 0.63905, "std": 0.16517}, "digit": {"share": 0.00832, "mean": 0.40413, "std": 0.1928}, "word_start": {"share": 0.57364, "mean": 0.41482, "std": 0.21487}, "word_cont": {"share": 0.26146, "mean": 0.2678, "std": 0.14371}, "punct": {"share": 0.12254, "mean": 0.63626, "std": 0.18526}}, "alpha_p05": 0.11398, "alpha_p50": 0.34953, "alpha_p95": 0.80928}
{"step": 36610, "total_steps": 72479, "tokens": 19194183680, "loss": 3.22556272149086, "l_mix": 1.996270090341568, "l_a": 2.0657087713479996, "l_c": 2.0319331288337708, "z_loss": 37.015681743621826, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.3895864188671112, "tokens_per_second": 1462797.7387554683, "eta_seconds": 12855.971658802033, "data_wait_seconds": 2.1070241928100586e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 36620, "total_steps": 72479, "tokens": 19199426560, "loss": 4.777622103691101, "l_mix": 2.9628936052322388, "l_a": 3.045935332775116, "l_c": 3.003159373998642, "z_loss": 32.02545189857483, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15819333493709564, "tokens_per_second": 1431913.3419325363, "eta_seconds": 13129.595794272424, "data_wait_seconds": 2.8967857360839844e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 36630, "total_steps": 72479, "tokens": 19204669440, "loss": 4.659050822257996, "l_mix": 2.8888787031173706, "l_a": 2.9685718715190887, "l_c": 2.9320015609264374, "z_loss": 31.617478370666504, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18085666000843048, "tokens_per_second": 1400403.8747230666, "eta_seconds": 13421.271428370475, "data_wait_seconds": 4.094839096069336e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 36640, "total_steps": 72479, "tokens": 19209912320, "loss": 4.596340000629425, "l_mix": 2.849871277809143, "l_a": 2.931721419095993, "l_c": 2.8898406624794006, "z_loss": 31.53015398979187, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15379218757152557, "tokens_per_second": 1468615.5406545082, "eta_seconds": 12794.333923244476, "data_wait_seconds": 1.96993350982666e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 36650, "total_steps": 72479, "tokens": 19215155200, "loss": 4.5396007895469666, "l_mix": 2.8133608400821686, "l_a": 2.902047723531723, "l_c": 2.852085143327713, "z_loss": 31.30781626701355, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14892177283763885, "tokens_per_second": 1321008.2517700964, "eta_seconds": 14219.98290081024, "data_wait_seconds": 2.562999725341797e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 36660, "total_steps": 72479, "tokens": 19220398080, "loss": 4.521219551563263, "l_mix": 2.8019883036613464, "l_a": 2.888247400522232, "l_c": 2.842523366212845, "z_loss": 32.1604380607605, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11529961228370667, "tokens_per_second": 1378786.3119204252, "eta_seconds": 13620.291780996324, "data_wait_seconds": 1.7762184143066406e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 36670, "total_steps": 72479, "tokens": 19225640960, "loss": 4.32734215259552, "l_mix": 2.6801821887493134, "l_a": 2.7692238092422485, "l_c": 2.7213095128536224, "z_loss": 32.52975034713745, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12846893072128296, "tokens_per_second": 1472674.3480334012, "eta_seconds": 12748.391399002076, "data_wait_seconds": 2.2739171981811523e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 36680, "total_steps": 72479, "tokens": 19230883840, "loss": 4.39177131652832, "l_mix": 2.7204293310642242, "l_a": 2.81204617023468, "l_c": 2.759093552827835, "z_loss": 31.78891658782959, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14835721254348755, "tokens_per_second": 1426511.7787187712, "eta_seconds": 13157.259822177886, "data_wait_seconds": 2.4259090423583984e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 36690, "total_steps": 72479, "tokens": 19236126720, "loss": 3.2703436613082886, "l_mix": 2.0126991868019104, "l_a": 2.13063907623291, "l_c": 2.0615090876817703, "z_loss": 32.27708625793457, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17069591581821442, "tokens_per_second": 1458755.4722168397, "eta_seconds": 12862.843423295022, "data_wait_seconds": 1.582503318786621e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 36700, "total_steps": 72479, "tokens": 19241369600, "loss": 3.209544986486435, "l_mix": 1.9745742231607437, "l_a": 2.09142142534256, "l_c": 2.025147795677185, "z_loss": 33.28937530517578, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16989614069461823, "tokens_per_second": 1438869.3155706718, "eta_seconds": 13036.972954392433, "data_wait_seconds": 1.946091651916504e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.23514056, "gate_a2b/L1_absmax": 0.71355557, "gate_a2b/L1_rms": 0.24876909, "gate_a2b/L3_absmean": 0.2399791, "gate_a2b/L3_absmax": 0.6121158, "gate_a2b/L3_rms": 0.2461165, "gate_a2b/L5_absmean": 0.37752503, "gate_a2b/L5_absmax": 0.68217862, "gate_a2b/L5_rms": 0.38338909, "gate_c2b/L1_absmean": 0.01769268, "gate_c2b/L1_absmax": 0.60199672, "gate_c2b/L1_rms": 0.06761453, "gate_c2b/L3_absmean": 0.21182886, "gate_c2b/L3_absmax": 0.47676733, "gate_c2b/L3_rms": 0.21617115, "gate_c2b/L5_absmean": 0.58962297, "gate_c2b/L5_absmax": 0.82737947, "gate_c2b/L5_rms": 0.60283947, "gate_b2a/L7_absmean": 0.05712293, "gate_b2a/L7_absmax": 0.33027223, "gate_b2a/L7_rms": 0.06256735, "gate_b2a/L11_absmean": 0.09065706, "gate_b2a/L11_absmax": 0.58579463, "gate_b2a/L11_rms": 0.09852237, "gate_b2a/L14_absmean": 0.23162729, "gate_b2a/L14_absmax": 0.75031441, "gate_b2a/L14_rms": 0.23886745, "gate_b2c/L3_absmean": 0.04509441, "gate_b2c/L3_absmax": 0.62637299, "gate_b2c/L3_rms": 0.05754765, "gate_b2c/L5_absmean": 0.04998207, "gate_b2c/L5_absmax": 0.67650241, "gate_b2c/L5_rms": 0.06155667, "gate_b2c/L6_absmean": 0.07314818, "gate_b2c/L6_absmax": 0.46621066, "gate_b2c/L6_rms": 0.08138634}, "streams": {"cos_a2b/round0": 0.382296, "cos_c2b/round0": 0.276124, "cos_b2a/round0": 0.128465, "cos_b2c/round0": 0.102288, "rms_a/round0": 3.756945, "rms_b/round0": 6.959893, "rms_c/round0": 3.058054, "cos_a2b/round1": 0.421087, "cos_c2b/round1": 0.291822, "cos_b2a/round1": 0.234407, "cos_b2c/round1": 0.122285, "rms_a/round1": 7.091731, "rms_b/round1": 16.024689, "rms_c/round1": 6.141716, "cos_a2b/round2": 0.524581, "cos_c2b/round2": 0.510585, "cos_b2a/round2": 0.078749, "cos_b2c/round2": 0.160966, "rms_a/round2": 17.7054, "rms_b/round2": 39.633537, "rms_c/round2": 11.283056}, "alpha_mean": 0.41209159813305973, "alpha_hist": [0.03648, 0.15872, 0.18861, 0.15166, 0.11848, 0.10977, 0.09988, 0.0892, 0.04335, 0.00383], "alpha_by_type": {"special": {"share": 0.00101, "mean": 0.59306, "std": 0.12366}, "whitespace": {"share": 0.05594, "mean": 0.59638, "std": 0.15605}, "digit": {"share": 0.01521, "mean": 0.37021, "std": 0.1615}, "word_start": {"share": 0.53346, "mean": 0.41346, "std": 0.21174}, "word_cont": {"share": 0.26431, "mean": 0.27136, "std": 0.13856}, "punct": {"share": 0.13007, "mean": 0.61673, "std": 0.18566}}, "alpha_p05": 0.1128, "alpha_p50": 0.37159, "alpha_p95": 0.79085}
{"step": 36710, "total_steps": 72479, "tokens": 19246612480, "loss": 3.230698734521866, "l_mix": 1.9870330691337585, "l_a": 2.1059499084949493, "l_c": 2.0396019369363785, "z_loss": 33.48406219482422, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14325356483459473, "tokens_per_second": 1367707.939508516, "eta_seconds": 13711.448862934112, "data_wait_seconds": 2.4527311325073242e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 36720, "total_steps": 72479, "tokens": 19251855360, "loss": 3.0871399641036987, "l_mix": 1.9109591394662857, "l_a": 1.981787770986557, "l_c": 1.9388148784637451, "z_loss": 38.29926252365112, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13695713877677917, "tokens_per_second": 1463055.4499150652, "eta_seconds": 12814.288476276397, "data_wait_seconds": 3.260374069213867e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 36730, "total_steps": 72479, "tokens": 19257098240, "loss": 4.444569528102875, "l_mix": 2.75397264957428, "l_a": 2.839676707983017, "l_c": 2.795646011829376, "z_loss": 31.841450929641724, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19584660232067108, "tokens_per_second": 1342164.0090912187, "eta_seconds": 13964.591201257706, "data_wait_seconds": 1.895427703857422e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 36740, "total_steps": 72479, "tokens": 19262341120, "loss": 4.457920134067535, "l_mix": 2.7620745301246643, "l_a": 2.853723108768463, "l_c": 2.7990950644016266, "z_loss": 31.581880807876587, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.145882248878479, "tokens_per_second": 1448693.8575647932, "eta_seconds": 12934.084543919562, "data_wait_seconds": 1.659989356994629e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 36750, "total_steps": 72479, "tokens": 19267584000, "loss": 4.415659606456757, "l_mix": 2.7353398203849792, "l_a": 2.8284098505973816, "l_c": 2.7726560533046722, "z_loss": 31.622300386428833, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13423685729503632, "tokens_per_second": 1362514.7925173386, "eta_seconds": 13748.317489743233, "data_wait_seconds": 1.5497207641601562e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 36760, "total_steps": 72479, "tokens": 19272826880, "loss": 4.40896600484848, "l_mix": 2.731410026550293, "l_a": 2.8187490701675415, "l_c": 2.7731039822101593, "z_loss": 32.64961910247803, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1495809704065323, "tokens_per_second": 1470931.1595122688, "eta_seconds": 12731.420468521117, "data_wait_seconds": 1.52587890625e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 36770, "total_steps": 72479, "tokens": 19278069760, "loss": 4.422087788581848, "l_mix": 2.7389695942401886, "l_a": 2.8288018703460693, "l_c": 2.781591445207596, "z_loss": 33.19223880767822, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16379690170288086, "tokens_per_second": 1315665.3858272089, "eta_seconds": 14229.910122799873, "data_wait_seconds": 1.749396324157715e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 36780, "total_steps": 72479, "tokens": 19283312640, "loss": 4.478141009807587, "l_mix": 2.775114566087723, "l_a": 2.861464262008667, "l_c": 2.8152900636196136, "z_loss": 31.73186683654785, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11845187842845917, "tokens_per_second": 1407317.0597099306, "eta_seconds": 13299.460262250901, "data_wait_seconds": 2.2292137145996094e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 36790, "total_steps": 72479, "tokens": 19288555520, "loss": 4.403891861438751, "l_mix": 2.728124678134918, "l_a": 2.818492352962494, "l_c": 2.767398029565811, "z_loss": 31.886779069900513, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14627373218536377, "tokens_per_second": 1444275.9729020176, "eta_seconds": 12955.497967886924, "data_wait_seconds": 3.17692756652832e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 36800, "total_steps": 72479, "tokens": 19293798400, "loss": 3.2788428962230682, "l_mix": 2.0179939568042755, "l_a": 2.133269339799881, "l_c": 2.069560468196869, "z_loss": 32.561644077301025, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17821340262889862, "tokens_per_second": 1345790.6424488113, "eta_seconds": 13899.689121007917, "data_wait_seconds": 1.7076730728149414e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.23465991, "gate_a2b/L1_absmax": 0.70994174, "gate_a2b/L1_rms": 0.24817531, "gate_a2b/L3_absmean": 0.23915949, "gate_a2b/L3_absmax": 0.6132952, "gate_a2b/L3_rms": 0.24525796, "gate_a2b/L5_absmean": 0.37586749, "gate_a2b/L5_absmax": 0.68364936, "gate_a2b/L5_rms": 0.38172305, "gate_c2b/L1_absmean": 0.01758862, "gate_c2b/L1_absmax": 0.60035425, "gate_c2b/L1_rms": 0.06761795, "gate_c2b/L3_absmean": 0.21119857, "gate_c2b/L3_absmax": 0.47224122, "gate_c2b/L3_rms": 0.21548606, "gate_c2b/L5_absmean": 0.58960915, "gate_c2b/L5_absmax": 0.82475585, "gate_c2b/L5_rms": 0.60277766, "gate_b2a/L7_absmean": 0.05678267, "gate_b2a/L7_absmax": 0.33038491, "gate_b2a/L7_rms": 0.0623389, "gate_b2a/L11_absmean": 0.09010023, "gate_b2a/L11_absmax": 0.58618969, "gate_b2a/L11_rms": 0.09802815, "gate_b2a/L14_absmean": 0.23104309, "gate_b2a/L14_absmax": 0.75395137, "gate_b2a/L14_rms": 0.23833857, "gate_b2c/L3_absmean": 0.04496244, "gate_b2c/L3_absmax": 0.62539816, "gate_b2c/L3_rms": 0.05746409, "gate_b2c/L5_absmean": 0.04969309, "gate_b2c/L5_absmax": 0.67401248, "gate_b2c/L5_rms": 0.06135632, "gate_b2c/L6_absmean": 0.07233259, "gate_b2c/L6_absmax": 0.46506837, "gate_b2c/L6_rms": 0.08060022}, "streams": {"cos_a2b/round0": 0.379527, "cos_c2b/round0": 0.266566, "cos_b2a/round0": 0.125927, "cos_b2c/round0": 0.101521, "rms_a/round0": 3.812054, "rms_b/round0": 7.112627, "rms_c/round0": 3.109139, "cos_a2b/round1": 0.422126, "cos_c2b/round1": 0.296163, "cos_b2a/round1": 0.234399, "cos_b2c/round1": 0.119075, "rms_a/round1": 7.184651, "rms_b/round1": 16.191313, "rms_c/round1": 6.225801, "cos_a2b/round2": 0.524296, "cos_c2b/round2": 0.512415, "cos_b2a/round2": 0.080235, "cos_b2c/round2": 0.158136, "rms_a/round2": 17.806128, "rms_b/round2": 39.736919, "rms_c/round2": 11.509621}, "alpha_mean": 0.4282358297653204, "alpha_hist": [0.02097, 0.13661, 0.20059, 0.16116, 0.11992, 0.10263, 0.09956, 0.10057, 0.05382, 0.00417], "alpha_by_type": {"special": {"share": 0.00107, "mean": 0.65844, "std": 0.1145}, "whitespace": {"share": 0.03252, "mean": 0.65594, "std": 0.16457}, "digit": {"share": 0.00917, "mean": 0.4236, "std": 0.1662}, "word_start": {"share": 0.57417, "mean": 0.43103, "std": 0.21035}, "word_cont": {"share": 0.26617, "mean": 0.29855, "std": 0.14178}, "punct": {"share": 0.1169, "mean": 0.64474, "std": 0.18505}}, "alpha_p05": 0.1394, "alpha_p50": 0.39117, "alpha_p95": 0.79944}
{"step": 36810, "total_steps": 72479, "tokens": 19299041280, "loss": 3.228952258825302, "l_mix": 1.9857476651668549, "l_a": 2.1075300872325897, "l_c": 2.0364851653575897, "z_loss": 33.511372089385986, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16049084067344666, "tokens_per_second": 1348742.0249659996, "eta_seconds": 13865.38591208458, "data_wait_seconds": 1.7762184143066406e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 36820, "total_steps": 72479, "tokens": 19304284160, "loss": 3.2486201226711273, "l_mix": 1.9984466880559921, "l_a": 2.1152892112731934, "l_c": 2.0519552677869797, "z_loss": 33.25205993652344, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16760317981243134, "tokens_per_second": 1470988.720433289, "eta_seconds": 12709.537151646613, "data_wait_seconds": 1.5944242477416992e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 36830, "total_steps": 72479, "tokens": 19309527040, "loss": 3.2926072776317596, "l_mix": 2.0287811756134033, "l_a": 2.128668427467346, "l_c": 2.0840851962566376, "z_loss": 35.77695393562317, "lr": 0.0023, "wd": 0.01, "grad_norm": 4.099041938781738, "tokens_per_second": 1436465.3055503806, "eta_seconds": 13011.343079280852, "data_wait_seconds": 2.130866050720215e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 36840, "total_steps": 72479, "tokens": 19314769920, "loss": 4.408323168754578, "l_mix": 2.73038786649704, "l_a": 2.8200903832912445, "l_c": 2.7730270624160767, "z_loss": 31.64329481124878, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17954203486442566, "tokens_per_second": 1411886.867357938, "eta_seconds": 13234.134025883675, "data_wait_seconds": 1.895427703857422e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 36850, "total_steps": 72479, "tokens": 19320012800, "loss": 4.347930371761322, "l_mix": 2.6934867203235626, "l_a": 2.7819364070892334, "l_c": 2.7328756153583527, "z_loss": 32.00036644935608, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15693482756614685, "tokens_per_second": 1460098.2123460986, "eta_seconds": 12793.562100172043, "data_wait_seconds": 2.2917985916137695e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 36860, "total_steps": 72479, "tokens": 19325255680, "loss": 4.391146779060364, "l_mix": 2.720078468322754, "l_a": 2.809185206890106, "l_c": 2.7610424160957336, "z_loss": 32.9476113319397, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14264214038848877, "tokens_per_second": 1424692.580346888, "eta_seconds": 13107.820262145997, "data_wait_seconds": 1.5854835510253906e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 36870, "total_steps": 72479, "tokens": 19330498560, "loss": 4.360748052597046, "l_mix": 2.701614499092102, "l_a": 2.7848308980464935, "l_c": 2.74561408162117, "z_loss": 32.127636432647705, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.22198596596717834, "tokens_per_second": 1447747.9170011408, "eta_seconds": 12895.457263493538, "data_wait_seconds": 1.659989356994629e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 36880, "total_steps": 72479, "tokens": 19335741440, "loss": 4.438982248306274, "l_mix": 2.750384509563446, "l_a": 2.838223248720169, "l_c": 2.790436089038849, "z_loss": 32.51631832122803, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14018498361110687, "tokens_per_second": 1405211.7118856967, "eta_seconds": 13282.075828242303, "data_wait_seconds": 1.6391277313232422e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 36890, "total_steps": 72479, "tokens": 19340984320, "loss": 4.716488778591156, "l_mix": 2.9257953763008118, "l_a": 3.002019554376602, "l_c": 2.9669581949710846, "z_loss": 31.902257442474365, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15491709113121033, "tokens_per_second": 1145264.366589049, "eta_seconds": 16292.208311319353, "data_wait_seconds": 2.2470951080322266e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 36900, "total_steps": 72479, "tokens": 19346227200, "loss": 4.725748956203461, "l_mix": 2.93186217546463, "l_a": 3.005979508161545, "l_c": 2.973642975091934, "z_loss": 31.674174308776855, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13646072149276733, "tokens_per_second": 1420714.6898161676, "eta_seconds": 13129.759891772272, "data_wait_seconds": 1.6927719116210938e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.23389225, "gate_a2b/L1_absmax": 0.71479553, "gate_a2b/L1_rms": 0.2473256, "gate_a2b/L3_absmean": 0.23828807, "gate_a2b/L3_absmax": 0.6154052, "gate_a2b/L3_rms": 0.24450839, "gate_a2b/L5_absmean": 0.37421289, "gate_a2b/L5_absmax": 0.68691558, "gate_a2b/L5_rms": 0.38009071, "gate_c2b/L1_absmean": 0.01653712, "gate_c2b/L1_absmax": 0.60537267, "gate_c2b/L1_rms": 0.06739718, "gate_c2b/L3_absmean": 0.21179996, "gate_c2b/L3_absmax": 0.47242823, "gate_c2b/L3_rms": 0.21604992, "gate_c2b/L5_absmean": 0.58894438, "gate_c2b/L5_absmax": 0.82252449, "gate_c2b/L5_rms": 0.60210204, "gate_b2a/L7_absmean": 0.05674321, "gate_b2a/L7_absmax": 0.33663526, "gate_b2a/L7_rms": 0.06235784, "gate_b2a/L11_absmean": 0.0895283, "gate_b2a/L11_absmax": 0.59160435, "gate_b2a/L11_rms": 0.09755938, "gate_b2a/L14_absmean": 0.23045915, "gate_b2a/L14_absmax": 0.75165641, "gate_b2a/L14_rms": 0.23783083, "gate_b2c/L3_absmean": 0.04512526, "gate_b2c/L3_absmax": 0.63156205, "gate_b2c/L3_rms": 0.05769803, "gate_b2c/L5_absmean": 0.04944427, "gate_b2c/L5_absmax": 0.68513852, "gate_b2c/L5_rms": 0.06139258, "gate_b2c/L6_absmean": 0.07175738, "gate_b2c/L6_absmax": 0.44789749, "gate_b2c/L6_rms": 0.07976399}, "streams": {"cos_a2b/round0": 0.383204, "cos_c2b/round0": 0.309263, "cos_b2a/round0": 0.14495, "cos_b2c/round0": 0.131339, "rms_a/round0": 3.672635, "rms_b/round0": 6.935937, "rms_c/round0": 2.988478, "cos_a2b/round1": 0.429946, "cos_c2b/round1": 0.305184, "cos_b2a/round1": 0.248204, "cos_b2c/round1": 0.119079, "rms_a/round1": 6.840333, "rms_b/round1": 15.196651, "rms_c/round1": 5.854012, "cos_a2b/round2": 0.575611, "cos_c2b/round2": 0.51997, "cos_b2a/round2": 0.113088, "cos_b2c/round2": 0.178806, "rms_a/round2": 17.721989, "rms_b/round2": 38.464935, "rms_c/round2": 10.568221}, "alpha_mean": 0.4604275823191699, "alpha_hist": [0.00381, 0.03932, 0.13684, 0.18866, 0.21049, 0.21228, 0.14313, 0.05969, 0.00555, 0.00021], "alpha_by_type": {"special": {"share": 0.00078, "mean": 0.54878, "std": 0.09962}, "whitespace": {"share": 0.0529, "mean": 0.5232, "std": 0.12077}, "digit": {"share": 0.02943, "mean": 0.45078, "std": 0.12753}, "word_start": {"share": 0.58545, "mean": 0.47471, "std": 0.14823}, "word_cont": {"share": 0.22415, "mean": 0.34913, "std": 0.12687}, "punct": {"share": 0.10728, "mean": 0.58608, "std": 0.14122}}, "alpha_p05": 0.22203, "alpha_p50": 0.47037, "alpha_p95": 0.71741}
{"step": 36910, "total_steps": 72479, "tokens": 19351470080, "loss": 4.710919260978699, "l_mix": 2.921405076980591, "l_a": 3.0005783140659332, "l_c": 2.9644689857959747, "z_loss": 31.697449207305908, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1811375617980957, "tokens_per_second": 1467736.4736412268, "eta_seconds": 12705.550490093232, "data_wait_seconds": 1.609325408935547e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 36920, "total_steps": 72479, "tokens": 19356712960, "loss": 4.7773953676223755, "l_mix": 2.9641669392585754, "l_a": 3.0414519608020782, "l_c": 3.0026426315307617, "z_loss": 31.37433695793152, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19756662845611572, "tokens_per_second": 1177017.5256280946, "eta_seconds": 15839.319794368745, "data_wait_seconds": 3.510713577270508e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 36930, "total_steps": 72479, "tokens": 19361955840, "loss": 4.760847806930542, "l_mix": 2.9524362087249756, "l_a": 3.031102329492569, "l_c": 2.996936023235321, "z_loss": 30.908156871795654, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18115225434303284, "tokens_per_second": 1308490.3617535941, "eta_seconds": 14243.829879665374, "data_wait_seconds": 2.2172927856445312e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 36940, "total_steps": 72479, "tokens": 19367198720, "loss": 3.230337619781494, "l_mix": 1.9873589128255844, "l_a": 2.102853238582611, "l_c": 2.040408954024315, "z_loss": 33.54929542541504, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1821204572916031, "tokens_per_second": 1471646.316838822, "eta_seconds": 12661.106828999518, "data_wait_seconds": 3.400444984436035e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 36950, "total_steps": 72479, "tokens": 19372441600, "loss": 4.455084264278412, "l_mix": 2.7601530849933624, "l_a": 2.8511976301670074, "l_c": 2.7985728979110718, "z_loss": 32.090463161468506, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15955929458141327, "tokens_per_second": 1447273.504784179, "eta_seconds": 12870.703630256652, "data_wait_seconds": 1.6033649444580078e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 36960, "total_steps": 72479, "tokens": 19377684480, "loss": 4.337537229061127, "l_mix": 2.6866179704666138, "l_a": 2.7764554619789124, "l_c": 2.726608484983444, "z_loss": 33.405017375946045, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18333037197589874, "tokens_per_second": 1413457.0457594648, "eta_seconds": 13174.921394228935, "data_wait_seconds": 2.0384788513183594e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 36970, "total_steps": 72479, "tokens": 19382927360, "loss": 4.318904310464859, "l_mix": 2.6753980219364166, "l_a": 2.7642329335212708, "l_c": 2.7141208350658417, "z_loss": 32.67319631576538, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13156570494174957, "tokens_per_second": 1460179.264526306, "eta_seconds": 12749.765076303482, "data_wait_seconds": 2.136826515197754e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 36980, "total_steps": 72479, "tokens": 19388170240, "loss": 4.43804669380188, "l_mix": 2.748978912830353, "l_a": 2.8386711180210114, "l_c": 2.791554868221283, "z_loss": 32.014625787734985, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18243099749088287, "tokens_per_second": 1460578.35471806, "eta_seconds": 12742.691723370552, "data_wait_seconds": 1.5735626220703125e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 36990, "total_steps": 72479, "tokens": 19393413120, "loss": 4.39930534362793, "l_mix": 2.726243108510971, "l_a": 2.8142299950122833, "l_c": 2.7626437842845917, "z_loss": 32.188332319259644, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1835022121667862, "tokens_per_second": 1241159.1658053712, "eta_seconds": 14991.193188285826, "data_wait_seconds": 0.04557543992996216, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 37000, "total_steps": 72479, "tokens": 19398656000, "loss": 2.900832951068878, "l_mix": 1.7946098148822784, "l_a": 1.8611308336257935, "l_c": 1.8262796103954315, "z_loss": 37.621195793151855, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18132849037647247, "tokens_per_second": 1445157.2599372077, "eta_seconds": 12871.411622571944, "data_wait_seconds": 2.4646520614624023e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.23314154, "gate_a2b/L1_absmax": 0.7144534, "gate_a2b/L1_rms": 0.24666299, "gate_a2b/L3_absmean": 0.2375596, "gate_a2b/L3_absmax": 0.61225629, "gate_a2b/L3_rms": 0.24375668, "gate_a2b/L5_absmean": 0.37346289, "gate_a2b/L5_absmax": 0.68773657, "gate_a2b/L5_rms": 0.37932804, "gate_c2b/L1_absmean": 0.01763369, "gate_c2b/L1_absmax": 0.60564178, "gate_c2b/L1_rms": 0.06754662, "gate_c2b/L3_absmean": 0.21106783, "gate_c2b/L3_absmax": 0.47555462, "gate_c2b/L3_rms": 0.21534346, "gate_c2b/L5_absmean": 0.58859593, "gate_c2b/L5_absmax": 0.82151663, "gate_c2b/L5_rms": 0.60181791, "gate_b2a/L7_absmean": 0.05643588, "gate_b2a/L7_absmax": 0.33372012, "gate_b2a/L7_rms": 0.06202073, "gate_b2a/L11_absmean": 0.08935152, "gate_b2a/L11_absmax": 0.58986312, "gate_b2a/L11_rms": 0.09737831, "gate_b2a/L14_absmean": 0.23026296, "gate_b2a/L14_absmax": 0.75201654, "gate_b2a/L14_rms": 0.2376935, "gate_b2c/L3_absmean": 0.04481476, "gate_b2c/L3_absmax": 0.62870878, "gate_b2c/L3_rms": 0.05729229, "gate_b2c/L5_absmean": 0.04948668, "gate_b2c/L5_absmax": 0.67463154, "gate_b2c/L5_rms": 0.06114294, "gate_b2c/L6_absmean": 0.0715643, "gate_b2c/L6_absmax": 0.44883528, "gate_b2c/L6_rms": 0.0795358}, "streams": {"cos_a2b/round0": 0.395488, "cos_c2b/round0": 0.320584, "cos_b2a/round0": 0.128137, "cos_b2c/round0": 0.127683, "rms_a/round0": 3.790416, "rms_b/round0": 6.777813, "rms_c/round0": 2.991266, "cos_a2b/round1": 0.398911, "cos_c2b/round1": 0.263435, "cos_b2a/round1": 0.220418, "cos_b2c/round1": 0.099483, "rms_a/round1": 7.02231, "rms_b/round1": 15.472007, "rms_c/round1": 5.943727, "cos_a2b/round2": 0.498505, "cos_c2b/round2": 0.559503, "cos_b2a/round2": 0.080716, "cos_b2c/round2": 0.155467, "rms_a/round2": 17.062881, "rms_b/round2": 40.626911, "rms_c/round2": 10.720638}, "alpha_mean": 0.425893792580581, "alpha_hist": [0.0204, 0.06294, 0.14993, 0.21854, 0.21678, 0.17274, 0.1098, 0.04094, 0.00725, 0.00067], "alpha_by_type": {"special": {"share": 0.00063, "mean": 0.52183, "std": 0.14678}, "whitespace": {"share": 0.08844, "mean": 0.51345, "std": 0.14045}, "digit": {"share": 0.09314, "mean": 0.40718, "std": 0.13378}, "word_start": {"share": 0.37482, "mean": 0.42278, "std": 0.16311}, "word_cont": {"share": 0.2245, "mean": 0.31892, "std": 0.12395}, "punct": {"share": 0.21848, "mean": 0.51342, "std": 0.15334}}, "alpha_p05": 0.15078, "alpha_p50": 0.41893, "alpha_p95": 0.70457}
{"step": 37010, "total_steps": 72479, "tokens": 19403898880, "loss": 4.779792845249176, "l_mix": 2.9652887880802155, "l_a": 3.040842115879059, "l_c": 3.007504492998123, "z_loss": 31.456790924072266, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14177724719047546, "tokens_per_second": 1411147.275341505, "eta_seconds": 13177.909490346909, "data_wait_seconds": 2.4497509002685547e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 37020, "total_steps": 72479, "tokens": 19409141760, "loss": 4.739427208900452, "l_mix": 2.940016895532608, "l_a": 3.0199975967407227, "l_c": 2.978036344051361, "z_loss": 31.9489107131958, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15647462010383606, "tokens_per_second": 1410445.9999571547, "eta_seconds": 13180.744383382797, "data_wait_seconds": 3.421306610107422e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 37030, "total_steps": 72479, "tokens": 19414384640, "loss": 4.777403473854065, "l_mix": 2.962533622980118, "l_a": 3.0416920483112335, "l_c": 3.0078738927841187, "z_loss": 32.0516631603241, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17523586750030518, "tokens_per_second": 1272603.597056179, "eta_seconds": 14604.30047109127, "data_wait_seconds": 2.193450927734375e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 37040, "total_steps": 72479, "tokens": 19419627520, "loss": 4.742105841636658, "l_mix": 2.9424109160900116, "l_a": 3.015581637620926, "l_c": 2.983401209115982, "z_loss": 30.621131658554077, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16459612548351288, "tokens_per_second": 1391184.434912426, "eta_seconds": 13355.700341176986, "data_wait_seconds": 2.5451183319091797e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 37050, "total_steps": 72479, "tokens": 19424870400, "loss": 4.684120833873749, "l_mix": 2.906025230884552, "l_a": 2.9802242517471313, "l_c": 2.946760654449463, "z_loss": 31.909568071365356, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13802814483642578, "tokens_per_second": 1392661.603118313, "eta_seconds": 13337.769570446015, "data_wait_seconds": 1.633167266845703e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 37060, "total_steps": 72479, "tokens": 19430113280, "loss": 4.490465223789215, "l_mix": 2.7823846638202667, "l_a": 2.8691139221191406, "l_c": 2.824487864971161, "z_loss": 32.31165385246277, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13249249756336212, "tokens_per_second": 1181591.5970177997, "eta_seconds": 15715.884167480468, "data_wait_seconds": 2.256035804748535e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 37070, "total_steps": 72479, "tokens": 19435356160, "loss": 4.3416407108306885, "l_mix": 2.688999742269516, "l_a": 2.7796231508255005, "l_c": 2.729179859161377, "z_loss": 32.62794589996338, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11466038227081299, "tokens_per_second": 1457058.2864539672, "eta_seconds": 12741.092078876494, "data_wait_seconds": 1.5676021575927734e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 37080, "total_steps": 72479, "tokens": 19440599040, "loss": 4.301432758569717, "l_mix": 2.66391122341156, "l_a": 2.754194289445877, "l_c": 2.7042104601860046, "z_loss": 33.361478328704834, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13391175866127014, "tokens_per_second": 1421981.1094827226, "eta_seconds": 13051.700045967102, "data_wait_seconds": 1.6361474990844727e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 37090, "total_steps": 72479, "tokens": 19445841920, "loss": 4.3429935574531555, "l_mix": 2.6906016170978546, "l_a": 2.7784650921821594, "l_c": 2.729507714509964, "z_loss": 32.71457028388977, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1407187134027481, "tokens_per_second": 1412546.2946405516, "eta_seconds": 13135.16456232071, "data_wait_seconds": 1.6063451766967773e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 37100, "total_steps": 72479, "tokens": 19451084800, "loss": 4.465068697929382, "l_mix": 2.7670727372169495, "l_a": 2.854547470808029, "l_c": 2.8054392337799072, "z_loss": 32.16486716270447, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14922161400318146, "tokens_per_second": 1463094.7763690937, "eta_seconds": 12677.774161720276, "data_wait_seconds": 1.5050172805786133e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.23224032, "gate_a2b/L1_absmax": 0.71586484, "gate_a2b/L1_rms": 0.24574776, "gate_a2b/L3_absmean": 0.23623157, "gate_a2b/L3_absmax": 0.61260778, "gate_a2b/L3_rms": 0.2423954, "gate_a2b/L5_absmean": 0.37169853, "gate_a2b/L5_absmax": 0.68581504, "gate_a2b/L5_rms": 0.37750793, "gate_c2b/L1_absmean": 0.01735046, "gate_c2b/L1_absmax": 0.60739523, "gate_c2b/L1_rms": 0.06762935, "gate_c2b/L3_absmean": 0.21059802, "gate_c2b/L3_absmax": 0.47060812, "gate_c2b/L3_rms": 0.21486799, "gate_c2b/L5_absmean": 0.5882293, "gate_c2b/L5_absmax": 0.82321221, "gate_c2b/L5_rms": 0.6013661, "gate_b2a/L7_absmean": 0.05611484, "gate_b2a/L7_absmax": 0.33349404, "gate_b2a/L7_rms": 0.06175971, "gate_b2a/L11_absmean": 0.08907868, "gate_b2a/L11_absmax": 0.59023649, "gate_b2a/L11_rms": 0.09711707, "gate_b2a/L14_absmean": 0.22943702, "gate_b2a/L14_absmax": 0.75302052, "gate_b2a/L14_rms": 0.23688865, "gate_b2c/L3_absmean": 0.04423175, "gate_b2c/L3_absmax": 0.62698305, "gate_b2c/L3_rms": 0.05681583, "gate_b2c/L5_absmean": 0.04878438, "gate_b2c/L5_absmax": 0.66740513, "gate_b2c/L5_rms": 0.06035085, "gate_b2c/L6_absmean": 0.0709533, "gate_b2c/L6_absmax": 0.44838452, "gate_b2c/L6_rms": 0.07897075}, "streams": {"cos_a2b/round0": 0.389552, "cos_c2b/round0": 0.314293, "cos_b2a/round0": 0.130473, "cos_b2c/round0": 0.128854, "rms_a/round0": 3.738767, "rms_b/round0": 7.054199, "rms_c/round0": 3.041377, "cos_a2b/round1": 0.415832, "cos_c2b/round1": 0.289313, "cos_b2a/round1": 0.238285, "cos_b2c/round1": 0.115085, "rms_a/round1": 6.883692, "rms_b/round1": 15.786996, "rms_c/round1": 5.956416, "cos_a2b/round2": 0.552449, "cos_c2b/round2": 0.51366, "cos_b2a/round2": 0.101211, "cos_b2c/round2": 0.172111, "rms_a/round2": 18.189201, "rms_b/round2": 40.588974, "rms_c/round2": 10.734791}, "alpha_mean": 0.4209524575725254, "alpha_hist": [0.01324, 0.09264, 0.18327, 0.19574, 0.18137, 0.15398, 0.10493, 0.06348, 0.01103, 0.00032], "alpha_by_type": {"special": {"share": 0.00067, "mean": 0.54461, "std": 0.10783}, "whitespace": {"share": 0.03348, "mean": 0.50825, "std": 0.12566}, "digit": {"share": 0.09196, "mean": 0.45722, "std": 0.09628}, "word_start": {"share": 0.51312, "mean": 0.43089, "std": 0.17708}, "word_cont": {"share": 0.24664, "mean": 0.30233, "std": 0.12897}, "punct": {"share": 0.11412, "mean": 0.57709, "std": 0.16286}}, "alpha_p05": 0.16265, "alpha_p50": 0.42799, "alpha_p95": 0.71601}
{"step": 37110, "total_steps": 72479, "tokens": 19456327680, "loss": 3.049422115087509, "l_mix": 1.887576624751091, "l_a": 1.9574149399995804, "l_c": 1.9154032468795776, "z_loss": 38.06475353240967, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1482895463705063, "tokens_per_second": 1454496.3772258305, "eta_seconds": 12749.115475535393, "data_wait_seconds": 4.7326087951660156e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 37120, "total_steps": 72479, "tokens": 19461570560, "loss": 2.843948245048523, "l_mix": 1.759245604276657, "l_a": 1.8177474439144135, "l_c": 1.7979277670383453, "z_loss": 38.676448822021484, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2178817093372345, "tokens_per_second": 1436166.2248833575, "eta_seconds": 12908.185049057007, "data_wait_seconds": 1.683831214904785e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 37130, "total_steps": 72479, "tokens": 19466813440, "loss": 4.698554277420044, "l_mix": 2.9133937656879425, "l_a": 2.9979914724826813, "l_c": 2.952543318271637, "z_loss": 32.10961842536926, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15779486298561096, "tokens_per_second": 1467801.230678577, "eta_seconds": 12626.407530283926, "data_wait_seconds": 2.6345252990722656e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 37140, "total_steps": 72479, "tokens": 19472056320, "loss": 4.735539495944977, "l_mix": 2.936501204967499, "l_a": 3.0175781846046448, "l_c": 2.9792159497737885, "z_loss": 30.8626868724823, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16086918115615845, "tokens_per_second": 1418268.3645926306, "eta_seconds": 13063.6867426157, "data_wait_seconds": 1.7076730728149414e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 37150, "total_steps": 72479, "tokens": 19477299200, "loss": 4.672055184841156, "l_mix": 2.896879941225052, "l_a": 2.9808736443519592, "l_c": 2.9363768100738525, "z_loss": 32.17868208885193, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14316874742507935, "tokens_per_second": 1318152.4550686502, "eta_seconds": 14051.91841108799, "data_wait_seconds": 2.0176172256469727e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 37160, "total_steps": 72479, "tokens": 19482542080, "loss": 4.685584902763367, "l_mix": 2.906427949666977, "l_a": 2.982780337333679, "l_c": 2.947742462158203, "z_loss": 32.1252760887146, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1447324901819229, "tokens_per_second": 1280774.7466926337, "eta_seconds": 14457.911447596549, "data_wait_seconds": 1.6629695892333984e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 37170, "total_steps": 72479, "tokens": 19487784960, "loss": 4.545971393585205, "l_mix": 2.8175889551639557, "l_a": 2.9027383625507355, "l_c": 2.8585363924503326, "z_loss": 32.08284020423889, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.10182235389947891, "tokens_per_second": 1436662.290229761, "eta_seconds": 12885.481242108344, "data_wait_seconds": 1.7374753952026367e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08232}}
{"step": 37180, "total_steps": 72479, "tokens": 19493027840, "loss": 4.479210317134857, "l_mix": 2.7756319642066956, "l_a": 2.8629789650440216, "l_c": 2.815615177154541, "z_loss": 32.52130889892578, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11535579711198807, "tokens_per_second": 1434360.412509604, "eta_seconds": 12902.50480325222, "data_wait_seconds": 1.5854835510253906e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 37190, "total_steps": 72479, "tokens": 19498270720, "loss": 4.559981465339661, "l_mix": 2.826309859752655, "l_a": 2.9115211367607117, "l_c": 2.867383748292923, "z_loss": 32.30584168434143, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1291247010231018, "tokens_per_second": 1465492.908930009, "eta_seconds": 12624.830266499519, "data_wait_seconds": 1.704692840576172e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 37200, "total_steps": 72479, "tokens": 19503513600, "loss": 4.4724555015563965, "l_mix": 2.7709594666957855, "l_a": 2.8599683046340942, "l_c": 2.811685025691986, "z_loss": 31.392268180847168, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19880126416683197, "tokens_per_second": 1340689.8703458693, "eta_seconds": 13796.148356986047, "data_wait_seconds": 1.874566078186035e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.23184669, "gate_a2b/L1_absmax": 0.71385556, "gate_a2b/L1_rms": 0.24534121, "gate_a2b/L3_absmean": 0.23509319, "gate_a2b/L3_absmax": 0.61416996, "gate_a2b/L3_rms": 0.2413609, "gate_a2b/L5_absmean": 0.36987564, "gate_a2b/L5_absmax": 0.68652052, "gate_a2b/L5_rms": 0.37575349, "gate_c2b/L1_absmean": 0.01730177, "gate_c2b/L1_absmax": 0.60651213, "gate_c2b/L1_rms": 0.06774414, "gate_c2b/L3_absmean": 0.21046905, "gate_c2b/L3_absmax": 0.47174776, "gate_c2b/L3_rms": 0.21480963, "gate_c2b/L5_absmean": 0.588741, "gate_c2b/L5_absmax": 0.82166368, "gate_c2b/L5_rms": 0.60186195, "gate_b2a/L7_absmean": 0.05606176, "gate_b2a/L7_absmax": 0.33415821, "gate_b2a/L7_rms": 0.0616923, "gate_b2a/L11_absmean": 0.08908076, "gate_b2a/L11_absmax": 0.58993638, "gate_b2a/L11_rms": 0.0971427, "gate_b2a/L14_absmean": 0.22883008, "gate_b2a/L14_absmax": 0.75591803, "gate_b2a/L14_rms": 0.23630524, "gate_b2c/L3_absmean": 0.04455815, "gate_b2c/L3_absmax": 0.62470543, "gate_b2c/L3_rms": 0.05688943, "gate_b2c/L5_absmean": 0.04897639, "gate_b2c/L5_absmax": 0.66435117, "gate_b2c/L5_rms": 0.06032342, "gate_b2c/L6_absmean": 0.07062441, "gate_b2c/L6_absmax": 0.45119658, "gate_b2c/L6_rms": 0.07872254}, "streams": {"cos_a2b/round0": 0.390554, "cos_c2b/round0": 0.317993, "cos_b2a/round0": 0.140837, "cos_b2c/round0": 0.131192, "rms_a/round0": 3.748633, "rms_b/round0": 7.082306, "rms_c/round0": 3.042801, "cos_a2b/round1": 0.426893, "cos_c2b/round1": 0.29664, "cos_b2a/round1": 0.243624, "cos_b2c/round1": 0.108857, "rms_a/round1": 7.006538, "rms_b/round1": 15.638803, "rms_c/round1": 5.983005, "cos_a2b/round2": 0.558822, "cos_c2b/round2": 0.526046, "cos_b2a/round2": 0.099064, "cos_b2c/round2": 0.166003, "rms_a/round2": 18.064037, "rms_b/round2": 39.894379, "rms_c/round2": 10.871072}, "alpha_mean": 0.4513901940168026, "alpha_hist": [0.00667, 0.047, 0.15288, 0.19817, 0.19917, 0.18886, 0.13364, 0.06516, 0.00804, 0.00043], "alpha_by_type": {"special": {"share": 0.00078, "mean": 0.5744, "std": 0.11754}, "whitespace": {"share": 0.02904, "mean": 0.53385, "std": 0.12527}, "digit": {"share": 0.04747, "mean": 0.42299, "std": 0.12987}, "word_start": {"share": 0.59035, "mean": 0.46621, "std": 0.1545}, "word_cont": {"share": 0.2271, "mean": 0.34237, "std": 0.13554}, "punct": {"share": 0.10527, "mean": 0.59264, "std": 0.14791}}, "alpha_p05": 0.21834, "alpha_p50": 0.46259, "alpha_p95": 0.72115}
{"step": 37210, "total_steps": 72479, "tokens": 19508756480, "loss": 4.508796691894531, "l_mix": 2.793649524450302, "l_a": 2.8838514387607574, "l_c": 2.8333053588867188, "z_loss": 32.14206576347351, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12742751836776733, "tokens_per_second": 1378570.3076599115, "eta_seconds": 13413.253839325904, "data_wait_seconds": 2.181529998779297e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 37220, "total_steps": 72479, "tokens": 19513999360, "loss": 4.616694271564484, "l_mix": 2.86141899228096, "l_a": 2.946860820055008, "l_c": 2.9040560722351074, "z_loss": 32.5260705947876, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1585981249809265, "tokens_per_second": 1346840.006563249, "eta_seconds": 13725.3649296999, "data_wait_seconds": 2.3871660232543945e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 37230, "total_steps": 72479, "tokens": 19519242240, "loss": 3.210529237985611, "l_mix": 1.9869018346071243, "l_a": 2.061701714992523, "l_c": 2.0170561373233795, "z_loss": 38.00400400161743, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18323631584644318, "tokens_per_second": 1403767.045025655, "eta_seconds": 13165.024622488021, "data_wait_seconds": 2.5600194931030273e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 37240, "total_steps": 72479, "tokens": 19524485120, "loss": 4.371301233768463, "l_mix": 2.7078202664852142, "l_a": 2.796188861131668, "l_c": 2.7487475872039795, "z_loss": 32.707541942596436, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15532651543617249, "tokens_per_second": 1482453.1417980085, "eta_seconds": 12462.710834550859, "data_wait_seconds": 1.5914440155029297e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 37250, "total_steps": 72479, "tokens": 19529728000, "loss": 4.48281455039978, "l_mix": 2.7778112292289734, "l_a": 2.865661382675171, "l_c": 2.817682981491089, "z_loss": 32.759130001068115, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1342713087797165, "tokens_per_second": 1466575.0509691131, "eta_seconds": 12594.065295052527, "data_wait_seconds": 2.002716064453125e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 37260, "total_steps": 72479, "tokens": 19534970880, "loss": 4.5106441378593445, "l_mix": 2.7947438657283783, "l_a": 2.886026680469513, "l_c": 2.8336411118507385, "z_loss": 33.355671405792236, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15911121666431427, "tokens_per_second": 1435252.4931305717, "eta_seconds": 12865.261799144746, "data_wait_seconds": 1.9311904907226562e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 37270, "total_steps": 72479, "tokens": 19540213760, "loss": 4.522971928119659, "l_mix": 2.801710218191147, "l_a": 2.8915501534938812, "l_c": 2.8459885716438293, "z_loss": 32.00567293167114, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20826385915279388, "tokens_per_second": 1484907.4866852679, "eta_seconds": 12431.519375801085, "data_wait_seconds": 1.5854835510253906e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 37280, "total_steps": 72479, "tokens": 19545456640, "loss": 4.433228671550751, "l_mix": 2.7469464540481567, "l_a": 2.836649388074875, "l_c": 2.7842912077903748, "z_loss": 32.99491024017334, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1656687706708908, "tokens_per_second": 1424682.7040697099, "eta_seconds": 12953.349724316597, "data_wait_seconds": 1.5348196029663086e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 37290, "total_steps": 72479, "tokens": 19550699520, "loss": 4.403705835342407, "l_mix": 2.728680968284607, "l_a": 2.8161611557006836, "l_c": 2.767254561185837, "z_loss": 33.19303226470947, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11281876266002655, "tokens_per_second": 1389772.7556304794, "eta_seconds": 13274.95474152565, "data_wait_seconds": 2.41696834564209e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 37300, "total_steps": 72479, "tokens": 19555942400, "loss": 4.526646256446838, "l_mix": 2.8061822950839996, "l_a": 2.8898187279701233, "l_c": 2.845060855150223, "z_loss": 31.87173080444336, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14221331477165222, "tokens_per_second": 1427874.037227315, "eta_seconds": 12917.05505607128, "data_wait_seconds": 1.6391277313232422e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.23149665, "gate_a2b/L1_absmax": 0.71602583, "gate_a2b/L1_rms": 0.2449906, "gate_a2b/L3_absmean": 0.23492165, "gate_a2b/L3_absmax": 0.61670595, "gate_a2b/L3_rms": 0.24112214, "gate_a2b/L5_absmean": 0.36900744, "gate_a2b/L5_absmax": 0.68689448, "gate_a2b/L5_rms": 0.37486255, "gate_c2b/L1_absmean": 0.01750799, "gate_c2b/L1_absmax": 0.60985786, "gate_c2b/L1_rms": 0.0676265, "gate_c2b/L3_absmean": 0.20957689, "gate_c2b/L3_absmax": 0.47031292, "gate_c2b/L3_rms": 0.21390112, "gate_c2b/L5_absmean": 0.58775157, "gate_c2b/L5_absmax": 0.82132685, "gate_c2b/L5_rms": 0.60085511, "gate_b2a/L7_absmean": 0.05618665, "gate_b2a/L7_absmax": 0.33299255, "gate_b2a/L7_rms": 0.06179408, "gate_b2a/L11_absmean": 0.08867253, "gate_b2a/L11_absmax": 0.59069544, "gate_b2a/L11_rms": 0.0968101, "gate_b2a/L14_absmean": 0.22864032, "gate_b2a/L14_absmax": 0.75631702, "gate_b2a/L14_rms": 0.23611286, "gate_b2c/L3_absmean": 0.04410952, "gate_b2c/L3_absmax": 0.62449223, "gate_b2c/L3_rms": 0.05655608, "gate_b2c/L5_absmean": 0.0488242, "gate_b2c/L5_absmax": 0.66628921, "gate_b2c/L5_rms": 0.06033058, "gate_b2c/L6_absmean": 0.07043892, "gate_b2c/L6_absmax": 0.45304757, "gate_b2c/L6_rms": 0.07855364}, "streams": {"cos_a2b/round0": 0.380637, "cos_c2b/round0": 0.297489, "cos_b2a/round0": 0.142039, "cos_b2c/round0": 0.124303, "rms_a/round0": 3.745219, "rms_b/round0": 7.233536, "rms_c/round0": 2.992435, "cos_a2b/round1": 0.424314, "cos_c2b/round1": 0.297913, "cos_b2a/round1": 0.25691, "cos_b2c/round1": 0.112495, "rms_a/round1": 6.899076, "rms_b/round1": 15.691313, "rms_c/round1": 5.914913, "cos_a2b/round2": 0.56557, "cos_c2b/round2": 0.504892, "cos_b2a/round2": 0.121482, "cos_b2c/round2": 0.174762, "rms_a/round2": 18.126797, "rms_b/round2": 39.515923, "rms_c/round2": 10.484453}, "alpha_mean": 0.45160723797111757, "alpha_hist": [0.00278, 0.04312, 0.16548, 0.20796, 0.18248, 0.17908, 0.14581, 0.06706, 0.0062, 3e-05], "alpha_by_type": {"special": {"share": 0.00031, "mean": 0.55973, "std": 0.13294}, "whitespace": {"share": 0.05299, "mean": 0.52701, "std": 0.13527}, "digit": {"share": 0.01483, "mean": 0.49358, "std": 0.10173}, "word_start": {"share": 0.50081, "mean": 0.48739, "std": 0.15467}, "word_cont": {"share": 0.3271, "mean": 0.34607, "std": 0.12306}, "punct": {"share": 0.10396, "mean": 0.56656, "std": 0.16216}}, "alpha_p05": 0.20689, "alpha_p50": 0.42823, "alpha_p95": 0.73905}
{"step": 37310, "total_steps": 72479, "tokens": 19561185280, "loss": 4.485710859298706, "l_mix": 2.778433471918106, "l_a": 2.870609700679779, "l_c": 2.8203145265579224, "z_loss": 32.827444076538086, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14916954934597015, "tokens_per_second": 1380943.4058150353, "eta_seconds": 13352.237748742104, "data_wait_seconds": 1.8805265426635742e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 37320, "total_steps": 72479, "tokens": 19566428160, "loss": 4.373688340187073, "l_mix": 2.7096545100212097, "l_a": 2.795840710401535, "l_c": 2.7509383261203766, "z_loss": 32.97383117675781, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14541780948638916, "tokens_per_second": 1393913.205797857, "eta_seconds": 13224.239296483995, "data_wait_seconds": 2.053380012512207e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 37330, "total_steps": 72479, "tokens": 19571671040, "loss": 4.526781916618347, "l_mix": 2.8052287995815277, "l_a": 2.891646832227707, "l_c": 2.846863240003586, "z_loss": 32.563251972198486, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13689595460891724, "tokens_per_second": 1477908.4298911246, "eta_seconds": 12469.107381272315, "data_wait_seconds": 1.576542854309082e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 37340, "total_steps": 72479, "tokens": 19576913920, "loss": 2.9773149490356445, "l_mix": 1.841796651482582, "l_a": 1.9140017330646515, "l_c": 1.8710591942071915, "z_loss": 38.750532150268555, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.3949381709098816, "tokens_per_second": 1470137.4751541705, "eta_seconds": 12531.451203274728, "data_wait_seconds": 1.6868114471435547e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 37350, "total_steps": 72479, "tokens": 19582156800, "loss": 4.433295965194702, "l_mix": 2.7465887367725372, "l_a": 2.8361408412456512, "l_c": 2.7862167358398438, "z_loss": 31.810275554656982, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1399415135383606, "tokens_per_second": 1322040.1331741451, "eta_seconds": 13931.281426215171, "data_wait_seconds": 2.574920654296875e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 37360, "total_steps": 72479, "tokens": 19587399680, "loss": 4.420084536075592, "l_mix": 2.7392605245113373, "l_a": 2.824725776910782, "l_c": 2.778020828962326, "z_loss": 31.453202724456787, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14817996323108673, "tokens_per_second": 1408882.681069622, "eta_seconds": 13068.845631647111, "data_wait_seconds": 1.9878149032592773e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 37370, "total_steps": 72479, "tokens": 19592642560, "loss": 4.417880654335022, "l_mix": 2.7370521128177643, "l_a": 2.826090633869171, "l_c": 2.7766707241535187, "z_loss": 32.81177258491516, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1434343457221985, "tokens_per_second": 1470200.4783696954, "eta_seconds": 12520.215890836715, "data_wait_seconds": 1.8715858459472656e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 37380, "total_steps": 72479, "tokens": 19597885440, "loss": 4.460595011711121, "l_mix": 2.763665586709976, "l_a": 2.8501648902893066, "l_c": 2.806266278028488, "z_loss": 32.96626877784729, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11587647348642349, "tokens_per_second": 1412987.6775489557, "eta_seconds": 13023.457178282737, "data_wait_seconds": 1.9490718841552734e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 37390, "total_steps": 72479, "tokens": 19603128320, "loss": 4.4461639523506165, "l_mix": 2.75522843003273, "l_a": 2.8441321849823, "l_c": 2.792319566011429, "z_loss": 32.26156139373779, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13011598587036133, "tokens_per_second": 1454717.8737502145, "eta_seconds": 12646.260806965827, "data_wait_seconds": 2.1636486053466797e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 37400, "total_steps": 72479, "tokens": 19608371200, "loss": 4.496795237064362, "l_mix": 2.786753386259079, "l_a": 2.8726009130477905, "l_c": 2.827538251876831, "z_loss": 31.950181007385254, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15018579363822937, "tokens_per_second": 1401587.0689962977, "eta_seconds": 13121.909554409982, "data_wait_seconds": 1.4841556549072266e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.2308429, "gate_a2b/L1_absmax": 0.72004831, "gate_a2b/L1_rms": 0.24418172, "gate_a2b/L3_absmean": 0.23430048, "gate_a2b/L3_absmax": 0.61405134, "gate_a2b/L3_rms": 0.24051917, "gate_a2b/L5_absmean": 0.36766693, "gate_a2b/L5_absmax": 0.69020462, "gate_a2b/L5_rms": 0.37346154, "gate_c2b/L1_absmean": 0.01697194, "gate_c2b/L1_absmax": 0.6151098, "gate_c2b/L1_rms": 0.06804529, "gate_c2b/L3_absmean": 0.21021691, "gate_c2b/L3_absmax": 0.476879, "gate_c2b/L3_rms": 0.21450096, "gate_c2b/L5_absmean": 0.5871138, "gate_c2b/L5_absmax": 0.81688052, "gate_c2b/L5_rms": 0.60018641, "gate_b2a/L7_absmean": 0.05584597, "gate_b2a/L7_absmax": 0.33097956, "gate_b2a/L7_rms": 0.06147793, "gate_b2a/L11_absmean": 0.08925035, "gate_b2a/L11_absmax": 0.58914757, "gate_b2a/L11_rms": 0.09732991, "gate_b2a/L14_absmean": 0.22854909, "gate_b2a/L14_absmax": 0.75603384, "gate_b2a/L14_rms": 0.23593095, "gate_b2c/L3_absmean": 0.04409447, "gate_b2c/L3_absmax": 0.62723303, "gate_b2c/L3_rms": 0.05657505, "gate_b2c/L5_absmean": 0.04849422, "gate_b2c/L5_absmax": 0.66916096, "gate_b2c/L5_rms": 0.06011022, "gate_b2c/L6_absmean": 0.07016526, "gate_b2c/L6_absmax": 0.45057103, "gate_b2c/L6_rms": 0.0782427}, "streams": {"cos_a2b/round0": 0.381061, "cos_c2b/round0": 0.306931, "cos_b2a/round0": 0.138716, "cos_b2c/round0": 0.12731, "rms_a/round0": 3.794924, "rms_b/round0": 7.235878, "rms_c/round0": 3.061783, "cos_a2b/round1": 0.42655, "cos_c2b/round1": 0.297702, "cos_b2a/round1": 0.243897, "cos_b2c/round1": 0.117656, "rms_a/round1": 7.120674, "rms_b/round1": 16.129955, "rms_c/round1": 6.006103, "cos_a2b/round2": 0.564308, "cos_c2b/round2": 0.509958, "cos_b2a/round2": 0.106921, "cos_b2c/round2": 0.176203, "rms_a/round2": 18.650801, "rms_b/round2": 41.047638, "rms_c/round2": 10.81472}, "alpha_mean": 0.4603845173173333, "alpha_hist": [0.0094, 0.04681, 0.13387, 0.18631, 0.20271, 0.19688, 0.1443, 0.06897, 0.01068, 6e-05], "alpha_by_type": {"special": {"share": 0.00089, "mean": 0.53824, "std": 0.12389}, "whitespace": {"share": 0.03026, "mean": 0.51267, "std": 0.10954}, "digit": {"share": 0.03989, "mean": 0.46451, "std": 0.09562}, "word_start": {"share": 0.5526, "mean": 0.48353, "std": 0.15636}, "word_cont": {"share": 0.25511, "mean": 0.34153, "std": 0.1346}, "punct": {"share": 0.12126, "mean": 0.59, "std": 0.1498}}, "alpha_p05": 0.18803, "alpha_p50": 0.45169, "alpha_p95": 0.74614}
{"step": 37410, "total_steps": 72479, "tokens": 19613614080, "loss": 4.711791753768921, "l_mix": 2.922517031431198, "l_a": 2.9999858736991882, "l_c": 2.964262843132019, "z_loss": 32.08212947845459, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12722428143024445, "tokens_per_second": 1392506.2146356872, "eta_seconds": 13203.71548705101, "data_wait_seconds": 3.141164779663086e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 37420, "total_steps": 72479, "tokens": 19618856960, "loss": 4.807026505470276, "l_mix": 2.9814402163028717, "l_a": 3.06388983130455, "l_c": 3.0213973820209503, "z_loss": 33.07017660140991, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14811642467975616, "tokens_per_second": 1303538.1620555257, "eta_seconds": 14100.862964391708, "data_wait_seconds": 1.817941665649414e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 37430, "total_steps": 72479, "tokens": 19624099840, "loss": 4.676948606967926, "l_mix": 2.900365799665451, "l_a": 2.9815781116485596, "l_c": 2.9403644502162933, "z_loss": 32.42590045928955, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15018567442893982, "tokens_per_second": 1463948.3110895208, "eta_seconds": 12552.19871685505, "data_wait_seconds": 2.5898218154907227e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 37440, "total_steps": 72479, "tokens": 19629342720, "loss": 4.811892867088318, "l_mix": 2.985552132129669, "l_a": 3.05918288230896, "l_c": 3.028619408607483, "z_loss": 31.68498468399048, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1607007533311844, "tokens_per_second": 1434147.2226568605, "eta_seconds": 12809.373362636565, "data_wait_seconds": 3.0249357223510742e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 37450, "total_steps": 72479, "tokens": 19634585600, "loss": 4.774057924747467, "l_mix": 2.9612938463687897, "l_a": 3.0403581857681274, "l_c": 3.0021886825561523, "z_loss": 32.21886849403381, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13227614760398865, "tokens_per_second": 1374521.0592038527, "eta_seconds": 13361.224427247047, "data_wait_seconds": 2.4318695068359375e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 37460, "total_steps": 72479, "tokens": 19639828480, "loss": 4.371126770973206, "l_mix": 2.7078380584716797, "l_a": 2.79719740152359, "l_c": 2.747098386287689, "z_loss": 32.59713935852051, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12528948485851288, "tokens_per_second": 1474439.467861046, "eta_seconds": 12452.217857837677, "data_wait_seconds": 1.868605613708496e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 37470, "total_steps": 72479, "tokens": 19645071360, "loss": 4.389986038208008, "l_mix": 2.7204152047634125, "l_a": 2.8061141967773438, "l_c": 2.759121835231781, "z_loss": 33.04227352142334, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11293520778417587, "tokens_per_second": 1225562.782092839, "eta_seconds": 14976.628582549094, "data_wait_seconds": 2.0116567611694336e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 37480, "total_steps": 72479, "tokens": 19650314240, "loss": 4.448934495449066, "l_mix": 2.75551700592041, "l_a": 2.8478366136550903, "l_c": 2.796888291835785, "z_loss": 32.736220836639404, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13716323673725128, "tokens_per_second": 1450450.6213865264, "eta_seconds": 12650.934434747696, "data_wait_seconds": 3.510713577270508e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 37490, "total_steps": 72479, "tokens": 19655557120, "loss": 4.454384505748749, "l_mix": 2.760399639606476, "l_a": 2.8459119498729706, "l_c": 2.800704002380371, "z_loss": 32.6177544593811, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13957597315311432, "tokens_per_second": 1420796.8444506873, "eta_seconds": 12911.284891748428, "data_wait_seconds": 1.7017126083374023e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 37500, "total_steps": 72479, "tokens": 19660800000, "loss": 4.501422166824341, "l_mix": 2.7901427149772644, "l_a": 2.8765274584293365, "l_c": 2.8277375996112823, "z_loss": 32.156203508377075, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13681809604167938, "tokens_per_second": 1389756.6823755752, "eta_seconds": 13195.885426974297, "data_wait_seconds": 1.6063451766967773e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.2305018, "gate_a2b/L1_absmax": 0.71916765, "gate_a2b/L1_rms": 0.24385709, "gate_a2b/L3_absmean": 0.23376322, "gate_a2b/L3_absmax": 0.61429286, "gate_a2b/L3_rms": 0.2399957, "gate_a2b/L5_absmean": 0.36665654, "gate_a2b/L5_absmax": 0.68948853, "gate_a2b/L5_rms": 0.37252986, "gate_c2b/L1_absmean": 0.01793289, "gate_c2b/L1_absmax": 0.61306429, "gate_c2b/L1_rms": 0.06825715, "gate_c2b/L3_absmean": 0.20962124, "gate_c2b/L3_absmax": 0.46558452, "gate_c2b/L3_rms": 0.21390393, "gate_c2b/L5_absmean": 0.58654994, "gate_c2b/L5_absmax": 0.81675214, "gate_c2b/L5_rms": 0.59957302, "gate_b2a/L7_absmean": 0.05570382, "gate_b2a/L7_absmax": 0.33079755, "gate_b2a/L7_rms": 0.06129573, "gate_b2a/L11_absmean": 0.08887623, "gate_b2a/L11_absmax": 0.58791447, "gate_b2a/L11_rms": 0.09694678, "gate_b2a/L14_absmean": 0.22813892, "gate_b2a/L14_absmax": 0.75716192, "gate_b2a/L14_rms": 0.23555902, "gate_b2c/L3_absmean": 0.04423261, "gate_b2c/L3_absmax": 0.62533629, "gate_b2c/L3_rms": 0.05668463, "gate_b2c/L5_absmean": 0.04820142, "gate_b2c/L5_absmax": 0.66724825, "gate_b2c/L5_rms": 0.05985167, "gate_b2c/L6_absmean": 0.07009998, "gate_b2c/L6_absmax": 0.44961941, "gate_b2c/L6_rms": 0.07811684}, "streams": {"cos_a2b/round0": 0.382288, "cos_c2b/round0": 0.29746, "cos_b2a/round0": 0.133683, "cos_b2c/round0": 0.124553, "rms_a/round0": 3.819609, "rms_b/round0": 7.3002, "rms_c/round0": 3.107945, "cos_a2b/round1": 0.415552, "cos_c2b/round1": 0.295314, "cos_b2a/round1": 0.241281, "cos_b2c/round1": 0.11094, "rms_a/round1": 7.08414, "rms_b/round1": 16.064144, "rms_c/round1": 6.087204, "cos_a2b/round2": 0.54905, "cos_c2b/round2": 0.52445, "cos_b2a/round2": 0.10243, "cos_b2c/round2": 0.170805, "rms_a/round2": 18.302147, "rms_b/round2": 40.918839, "rms_c/round2": 10.91892}, "alpha_mean": 0.426809309104641, "alpha_hist": [0.00832, 0.05634, 0.16451, 0.23021, 0.21768, 0.17093, 0.10207, 0.04414, 0.00546, 0.00035], "alpha_by_type": {"special": {"share": 0.00084, "mean": 0.57109, "std": 0.14149}, "whitespace": {"share": 0.03421, "mean": 0.50693, "std": 0.12846}, "digit": {"share": 0.0585, "mean": 0.41564, "std": 0.1254}, "word_start": {"share": 0.5117, "mean": 0.44461, "std": 0.1497}, "word_cont": {"share": 0.27219, "mean": 0.33005, "std": 0.1257}, "punct": {"share": 0.12256, "mean": 0.54933, "std": 0.14384}}, "alpha_p05": 0.19529, "alpha_p50": 0.4038, "alpha_p95": 0.68447}
{"step": 37510, "total_steps": 72479, "tokens": 19666042880, "loss": 4.4229960441589355, "l_mix": 2.7404099702835083, "l_a": 2.8277520537376404, "l_c": 2.7808681428432465, "z_loss": 33.65095853805542, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15802080929279327, "tokens_per_second": 1441754.996262229, "eta_seconds": 12716.326365804673, "data_wait_seconds": 2.3573637008666992e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 37520, "total_steps": 72479, "tokens": 19671285760, "loss": 4.803886830806732, "l_mix": 2.9799822866916656, "l_a": 3.0573046505451202, "l_c": 3.022376775741577, "z_loss": 32.18994760513306, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14269189536571503, "tokens_per_second": 1435229.168268365, "eta_seconds": 12770.49310119152, "data_wait_seconds": 1.8596649169921875e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20768, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 37530, "total_steps": 72479, "tokens": 19676528640, "loss": 4.777951896190643, "l_mix": 2.964043229818344, "l_a": 3.0420326590538025, "l_c": 3.0043292343616486, "z_loss": 31.87759041786194, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1435961276292801, "tokens_per_second": 1414337.1334705935, "eta_seconds": 12955.426876926424, "data_wait_seconds": 2.4139881134033203e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 37540, "total_steps": 72479, "tokens": 19681771520, "loss": 4.841728389263153, "l_mix": 3.0032817721366882, "l_a": 3.084116369485855, "l_c": 3.044038861989975, "z_loss": 32.019986629486084, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16171298921108246, "tokens_per_second": 1377541.7807962683, "eta_seconds": 13297.671756577492, "data_wait_seconds": 3.0368566513061523e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 37550, "total_steps": 72479, "tokens": 19687014400, "loss": 4.637030810117722, "l_mix": 2.8766922503709793, "l_a": 2.953592538833618, "l_c": 2.914202481508255, "z_loss": 31.507016897201538, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13931477069854736, "tokens_per_second": 1396295.4024091586, "eta_seconds": 13115.316085982324, "data_wait_seconds": 3.361701965332031e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 37560, "total_steps": 72479, "tokens": 19692257280, "loss": 4.741069495677948, "l_mix": 2.9413804411888123, "l_a": 3.01963272690773, "l_c": 2.9793304800987244, "z_loss": 31.561869859695435, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1564902812242508, "tokens_per_second": 1407050.9691637708, "eta_seconds": 13011.33581740856, "data_wait_seconds": 3.2961368560791016e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 37570, "total_steps": 72479, "tokens": 19697500160, "loss": 4.463059842586517, "l_mix": 2.7642884254455566, "l_a": 2.856902003288269, "l_c": 2.8056691586971283, "z_loss": 32.903191566467285, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13462011516094208, "tokens_per_second": 1395740.7036667543, "eta_seconds": 13113.015722703933, "data_wait_seconds": 1.7851591110229492e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 37580, "total_steps": 72479, "tokens": 19702743040, "loss": 4.458680033683777, "l_mix": 2.762460470199585, "l_a": 2.852841019630432, "l_c": 2.8012241423130035, "z_loss": 33.54522371292114, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11585142463445663, "tokens_per_second": 1463456.6010525064, "eta_seconds": 12502.678179073333, "data_wait_seconds": 1.6480684280395508e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 37590, "total_steps": 72479, "tokens": 19707985920, "loss": 4.469684302806854, "l_mix": 2.7686564922332764, "l_a": 2.858848750591278, "l_c": 2.811244249343872, "z_loss": 33.161723613739014, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11700710654258728, "tokens_per_second": 1431237.3892996123, "eta_seconds": 12780.468263864517, "data_wait_seconds": 1.6480684280395508e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 37600, "total_steps": 72479, "tokens": 19713228800, "loss": 4.384924292564392, "l_mix": 2.7168634235858917, "l_a": 2.806058883666992, "l_c": 2.754143625497818, "z_loss": 32.81638145446777, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18222267925739288, "tokens_per_second": 1311573.6024191866, "eta_seconds": 13942.519976210593, "data_wait_seconds": 2.321600914001465e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.22969519, "gate_a2b/L1_absmax": 0.72629392, "gate_a2b/L1_rms": 0.2430633, "gate_a2b/L3_absmean": 0.2322882, "gate_a2b/L3_absmax": 0.618119, "gate_a2b/L3_rms": 0.23857786, "gate_a2b/L5_absmean": 0.3649635, "gate_a2b/L5_absmax": 0.69834286, "gate_a2b/L5_rms": 0.37088034, "gate_c2b/L1_absmean": 0.01780069, "gate_c2b/L1_absmax": 0.62084836, "gate_c2b/L1_rms": 0.06798322, "gate_c2b/L3_absmean": 0.20954747, "gate_c2b/L3_absmax": 0.47170454, "gate_c2b/L3_rms": 0.21389647, "gate_c2b/L5_absmean": 0.58579791, "gate_c2b/L5_absmax": 0.82088315, "gate_c2b/L5_rms": 0.59884018, "gate_b2a/L7_absmean": 0.05563071, "gate_b2a/L7_absmax": 0.33108526, "gate_b2a/L7_rms": 0.06133432, "gate_b2a/L11_absmean": 0.08866853, "gate_b2a/L11_absmax": 0.59122247, "gate_b2a/L11_rms": 0.09685741, "gate_b2a/L14_absmean": 0.22818419, "gate_b2a/L14_absmax": 0.75619572, "gate_b2a/L14_rms": 0.23560497, "gate_b2c/L3_absmean": 0.04347609, "gate_b2c/L3_absmax": 0.62612271, "gate_b2c/L3_rms": 0.05603113, "gate_b2c/L5_absmean": 0.04771454, "gate_b2c/L5_absmax": 0.65665567, "gate_b2c/L5_rms": 0.05912201, "gate_b2c/L6_absmean": 0.06963766, "gate_b2c/L6_absmax": 0.44751224, "gate_b2c/L6_rms": 0.07762442}, "streams": {"cos_a2b/round0": 0.381765, "cos_c2b/round0": 0.30562, "cos_b2a/round0": 0.138512, "cos_b2c/round0": 0.126754, "rms_a/round0": 3.902033, "rms_b/round0": 7.412981, "rms_c/round0": 3.124551, "cos_a2b/round1": 0.419358, "cos_c2b/round1": 0.295252, "cos_b2a/round1": 0.241624, "cos_b2c/round1": 0.115786, "rms_a/round1": 7.287296, "rms_b/round1": 16.337997, "rms_c/round1": 6.186607, "cos_a2b/round2": 0.560001, "cos_c2b/round2": 0.517309, "cos_b2a/round2": 0.100382, "cos_b2c/round2": 0.175776, "rms_a/round2": 18.899502, "rms_b/round2": 41.488884, "rms_c/round2": 11.184954}, "alpha_mean": 0.4360848024691961, "alpha_hist": [0.0088, 0.05711, 0.16736, 0.21431, 0.2036, 0.16296, 0.11412, 0.0623, 0.00919, 0.00024], "alpha_by_type": {"special": {"share": 0.00125, "mean": 0.53963, "std": 0.11259}, "whitespace": {"share": 0.0345, "mean": 0.50984, "std": 0.11038}, "digit": {"share": 0.05568, "mean": 0.39016, "std": 0.11157}, "word_start": {"share": 0.52969, "mean": 0.4611, "std": 0.15898}, "word_cont": {"share": 0.25549, "mean": 0.32392, "std": 0.12692}, "punct": {"share": 0.12338, "mean": 0.56, "std": 0.16167}}, "alpha_p05": 0.17753, "alpha_p50": 0.42131, "alpha_p95": 0.72855}
{"step": 37610, "total_steps": 72479, "tokens": 19718471680, "loss": 4.490794062614441, "l_mix": 2.783056855201721, "l_a": 2.869017541408539, "l_c": 2.823439747095108, "z_loss": 32.95330047607422, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11916059255599976, "tokens_per_second": 1449924.6268392387, "eta_seconds": 12608.516286706923, "data_wait_seconds": 2.2530555725097656e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 37620, "total_steps": 72479, "tokens": 19723714560, "loss": 4.504349231719971, "l_mix": 2.791850358247757, "l_a": 2.8775715231895447, "l_c": 2.830757826566696, "z_loss": 32.93037033081055, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13215863704681396, "tokens_per_second": 1462504.8096559145, "eta_seconds": 12496.47541077137, "data_wait_seconds": 1.65402889251709e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 37630, "total_steps": 72479, "tokens": 19728957440, "loss": 3.041371524333954, "l_mix": 1.8795934170484543, "l_a": 1.9626810997724533, "l_c": 1.9099125415086746, "z_loss": 39.54193663597107, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2225428968667984, "tokens_per_second": 1409783.4704168178, "eta_seconds": 12960.084222435951, "data_wait_seconds": 3.1828880310058594e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 37640, "total_steps": 72479, "tokens": 19734200320, "loss": 4.660695493221283, "l_mix": 2.8919650316238403, "l_a": 2.965234190225601, "l_c": 2.930533707141876, "z_loss": 32.59572672843933, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17960131168365479, "tokens_per_second": 1407751.935831773, "eta_seconds": 12975.062698960304, "data_wait_seconds": 1.9609928131103516e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 37650, "total_steps": 72479, "tokens": 19739443200, "loss": 4.758986234664917, "l_mix": 2.952261596918106, "l_a": 3.030625730752945, "l_c": 2.9917895793914795, "z_loss": 31.891362190246582, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16033588349819183, "tokens_per_second": 1411090.0465828048, "eta_seconds": 12940.65307612419, "data_wait_seconds": 1.958012580871582e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 37660, "total_steps": 72479, "tokens": 19744686080, "loss": 4.725071847438812, "l_mix": 2.9307934045791626, "l_a": 3.0099468529224396, "l_c": 2.970981150865555, "z_loss": 32.190829277038574, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12848713994026184, "tokens_per_second": 1424682.5194677017, "eta_seconds": 12813.510113692284, "data_wait_seconds": 1.6391277313232422e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 37670, "total_steps": 72479, "tokens": 19749928960, "loss": 4.788524150848389, "l_mix": 2.9709072411060333, "l_a": 3.047371566295624, "l_c": 3.0113512873649597, "z_loss": 31.988136291503906, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1460018903017044, "tokens_per_second": 1090625.8085322736, "eta_seconds": 16733.457845234872, "data_wait_seconds": 1.7374753952026367e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 37680, "total_steps": 72479, "tokens": 19755171840, "loss": 4.8312729597091675, "l_mix": 2.996864825487137, "l_a": 3.077429711818695, "l_c": 3.037264049053192, "z_loss": 32.51371765136719, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14495164155960083, "tokens_per_second": 1062695.806784905, "eta_seconds": 17168.316648578646, "data_wait_seconds": 1.874566078186035e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 37690, "total_steps": 72479, "tokens": 19760414720, "loss": 4.514029622077942, "l_mix": 2.7972123324871063, "l_a": 2.8849593698978424, "l_c": 2.8377644419670105, "z_loss": 32.94450378417969, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1081276386976242, "tokens_per_second": 1384844.068806437, "eta_seconds": 13170.76459569931, "data_wait_seconds": 1.7434358596801758e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 37700, "total_steps": 72479, "tokens": 19765657600, "loss": 4.473464846611023, "l_mix": 2.7722465991973877, "l_a": 2.857657343149185, "l_c": 2.8130700886249542, "z_loss": 33.461402893066406, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12230733036994934, "tokens_per_second": 1373050.0385917265, "eta_seconds": 13280.078540110588, "data_wait_seconds": 1.6689300537109375e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.22884277, "gate_a2b/L1_absmax": 0.72303069, "gate_a2b/L1_rms": 0.24223971, "gate_a2b/L3_absmean": 0.23158899, "gate_a2b/L3_absmax": 0.6173445, "gate_a2b/L3_rms": 0.23787121, "gate_a2b/L5_absmean": 0.36381185, "gate_a2b/L5_absmax": 0.69692725, "gate_a2b/L5_rms": 0.36972499, "gate_c2b/L1_absmean": 0.01775311, "gate_c2b/L1_absmax": 0.61704397, "gate_c2b/L1_rms": 0.06786492, "gate_c2b/L3_absmean": 0.20912527, "gate_c2b/L3_absmax": 0.47709915, "gate_c2b/L3_rms": 0.21343753, "gate_c2b/L5_absmean": 0.58560169, "gate_c2b/L5_absmax": 0.82168239, "gate_c2b/L5_rms": 0.59866267, "gate_b2a/L7_absmean": 0.05543751, "gate_b2a/L7_absmax": 0.33007658, "gate_b2a/L7_rms": 0.06103261, "gate_b2a/L11_absmean": 0.08825503, "gate_b2a/L11_absmax": 0.58902812, "gate_b2a/L11_rms": 0.09646037, "gate_b2a/L14_absmean": 0.22743869, "gate_b2a/L14_absmax": 0.75464928, "gate_b2a/L14_rms": 0.2348526, "gate_b2c/L3_absmean": 0.04356579, "gate_b2c/L3_absmax": 0.62471557, "gate_b2c/L3_rms": 0.05596436, "gate_b2c/L5_absmean": 0.04780079, "gate_b2c/L5_absmax": 0.65697902, "gate_b2c/L5_rms": 0.05915553, "gate_b2c/L6_absmean": 0.06944267, "gate_b2c/L6_absmax": 0.44321239, "gate_b2c/L6_rms": 0.07738006}, "streams": {"cos_a2b/round0": 0.382316, "cos_c2b/round0": 0.325814, "cos_b2a/round0": 0.143014, "cos_b2c/round0": 0.133102, "rms_a/round0": 3.874376, "rms_b/round0": 7.278241, "rms_c/round0": 3.025517, "cos_a2b/round1": 0.431064, "cos_c2b/round1": 0.296636, "cos_b2a/round1": 0.241309, "cos_b2c/round1": 0.114463, "rms_a/round1": 7.318351, "rms_b/round1": 16.072788, "rms_c/round1": 6.07338, "cos_a2b/round2": 0.555312, "cos_c2b/round2": 0.516438, "cos_b2a/round2": 0.104459, "cos_b2c/round2": 0.157315, "rms_a/round2": 18.923683, "rms_b/round2": 41.59882, "rms_c/round2": 11.097903}, "alpha_mean": 0.46126304597976286, "alpha_hist": [0.01537, 0.06383, 0.14566, 0.16995, 0.15909, 0.18425, 0.16191, 0.08385, 0.01587, 0.00023], "alpha_by_type": {"special": {"share": 0.00053, "mean": 0.56404, "std": 0.15755}, "whitespace": {"share": 0.05342, "mean": 0.54092, "std": 0.1373}, "digit": {"share": 0.0854, "mean": 0.33094, "std": 0.16602}, "word_start": {"share": 0.55136, "mean": 0.4886, "std": 0.16955}, "word_cont": {"share": 0.20459, "mean": 0.34388, "std": 0.14069}, "punct": {"share": 0.10469, "mean": 0.61185, "std": 0.15457}}, "alpha_p05": 0.16425, "alpha_p50": 0.45689, "alpha_p95": 0.75815}
{"step": 37710, "total_steps": 72479, "tokens": 19770900480, "loss": 4.492016136646271, "l_mix": 2.7836729288101196, "l_a": 2.871151566505432, "l_c": 2.823325604200363, "z_loss": 32.995158195495605, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11772440373897552, "tokens_per_second": 1460380.2856138921, "eta_seconds": 12482.344257569313, "data_wait_seconds": 1.9937753677368164e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 37720, "total_steps": 72479, "tokens": 19776143360, "loss": 4.425776183605194, "l_mix": 2.741902232170105, "l_a": 2.830094575881958, "l_c": 2.7828183472156525, "z_loss": 32.94714403152466, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12074804306030273, "tokens_per_second": 1427163.637703642, "eta_seconds": 12769.192060780526, "data_wait_seconds": 1.6808509826660156e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 37730, "total_steps": 72479, "tokens": 19781386240, "loss": 4.47484964132309, "l_mix": 2.772083818912506, "l_a": 2.865648090839386, "l_c": 2.8102378249168396, "z_loss": 33.163036823272705, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1689993292093277, "tokens_per_second": 1413665.1279064124, "eta_seconds": 12887.41113603115, "data_wait_seconds": 2.485513687133789e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 37740, "total_steps": 72479, "tokens": 19786629120, "loss": 2.8792795836925507, "l_mix": 1.7810825407505035, "l_a": 1.8495763391256332, "l_c": 1.8110801875591278, "z_loss": 38.574913024902344, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.21588890254497528, "tokens_per_second": 1463555.6569235534, "eta_seconds": 12444.515345788, "data_wait_seconds": 3.072619438171387e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 37750, "total_steps": 72479, "tokens": 19791872000, "loss": 3.018547832965851, "l_mix": 1.8679460436105728, "l_a": 1.9339833557605743, "l_c": 1.9013559222221375, "z_loss": 38.9600887298584, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.25352850556373596, "tokens_per_second": 1367584.5196243816, "eta_seconds": 13313.983662962914, "data_wait_seconds": 1.8984079360961914e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08236}}
{"step": 37760, "total_steps": 72479, "tokens": 19797114880, "loss": 4.5549076795578, "l_mix": 2.8235265612602234, "l_a": 2.908680558204651, "l_c": 2.862589567899704, "z_loss": 32.83855438232422, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11298732459545135, "tokens_per_second": 1295232.0741258366, "eta_seconds": 14053.662996482848, "data_wait_seconds": 2.872943878173828e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 37770, "total_steps": 72479, "tokens": 19802357760, "loss": 4.533873438835144, "l_mix": 2.809580475091934, "l_a": 2.899073749780655, "l_c": 2.8485694527626038, "z_loss": 32.90752863883972, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12748099863529205, "tokens_per_second": 1409350.6804592295, "eta_seconds": 12911.983116984366, "data_wait_seconds": 2.3305416107177734e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 37780, "total_steps": 72479, "tokens": 19807600640, "loss": 4.488156795501709, "l_mix": 2.7809237837791443, "l_a": 2.871598929166794, "l_c": 2.819177508354187, "z_loss": 32.526612281799316, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14592081308364868, "tokens_per_second": 1444250.077335851, "eta_seconds": 12596.342972373963, "data_wait_seconds": 3.871321678161621e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 37790, "total_steps": 72479, "tokens": 19812843520, "loss": 4.441390812397003, "l_mix": 2.7517795860767365, "l_a": 2.8428269028663635, "l_c": 2.789210468530655, "z_loss": 32.75450134277344, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12173297256231308, "tokens_per_second": 1281231.6611030689, "eta_seconds": 14194.955513620376, "data_wait_seconds": 1.6123056411743164e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 37800, "total_steps": 72479, "tokens": 19818086400, "loss": 4.508014559745789, "l_mix": 2.793687790632248, "l_a": 2.8826271295547485, "l_c": 2.8317950665950775, "z_loss": 33.11868190765381, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14105316996574402, "tokens_per_second": 1305305.0416663329, "eta_seconds": 13929.145273804663, "data_wait_seconds": 0.02365940809249878, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.22840782, "gate_a2b/L1_absmax": 0.72331035, "gate_a2b/L1_rms": 0.24179021, "gate_a2b/L3_absmean": 0.23093088, "gate_a2b/L3_absmax": 0.6144172, "gate_a2b/L3_rms": 0.23725164, "gate_a2b/L5_absmean": 0.36251289, "gate_a2b/L5_absmax": 0.69753438, "gate_a2b/L5_rms": 0.36839578, "gate_c2b/L1_absmean": 0.01762523, "gate_c2b/L1_absmax": 0.61639929, "gate_c2b/L1_rms": 0.06784998, "gate_c2b/L3_absmean": 0.20881429, "gate_c2b/L3_absmax": 0.47201791, "gate_c2b/L3_rms": 0.213163, "gate_c2b/L5_absmean": 0.58537012, "gate_c2b/L5_absmax": 0.82028788, "gate_c2b/L5_rms": 0.59834707, "gate_b2a/L7_absmean": 0.05539398, "gate_b2a/L7_absmax": 0.33050993, "gate_b2a/L7_rms": 0.06100785, "gate_b2a/L11_absmean": 0.08816281, "gate_b2a/L11_absmax": 0.59058613, "gate_b2a/L11_rms": 0.09640803, "gate_b2a/L14_absmean": 0.22695984, "gate_b2a/L14_absmax": 0.7568056, "gate_b2a/L14_rms": 0.23441465, "gate_b2c/L3_absmean": 0.04328448, "gate_b2c/L3_absmax": 0.62326223, "gate_b2c/L3_rms": 0.05580528, "gate_b2c/L5_absmean": 0.04772899, "gate_b2c/L5_absmax": 0.65886122, "gate_b2c/L5_rms": 0.05918569, "gate_b2c/L6_absmean": 0.06918475, "gate_b2c/L6_absmax": 0.4436574, "gate_b2c/L6_rms": 0.07714601}, "streams": {"cos_a2b/round0": 0.382896, "cos_c2b/round0": 0.306902, "cos_b2a/round0": 0.13986, "cos_b2c/round0": 0.125739, "rms_a/round0": 3.952212, "rms_b/round0": 7.499332, "rms_c/round0": 3.144846, "cos_a2b/round1": 0.422285, "cos_c2b/round1": 0.297719, "cos_b2a/round1": 0.24904, "cos_b2c/round1": 0.120199, "rms_a/round1": 7.381366, "rms_b/round1": 16.572582, "rms_c/round1": 6.23842, "cos_a2b/round2": 0.561265, "cos_c2b/round2": 0.517908, "cos_b2a/round2": 0.09777, "cos_b2c/round2": 0.177144, "rms_a/round2": 19.196909, "rms_b/round2": 42.043938, "rms_c/round2": 11.381232}, "alpha_mean": 0.43742642994290293, "alpha_hist": [0.00777, 0.06224, 0.16614, 0.20988, 0.20589, 0.15688, 0.10944, 0.06892, 0.01241, 0.00044], "alpha_by_type": {"special": {"share": 0.00121, "mean": 0.54608, "std": 0.10672}, "whitespace": {"share": 0.03217, "mean": 0.53308, "std": 0.12272}, "digit": {"share": 0.04034, "mean": 0.45285, "std": 0.10226}, "word_start": {"share": 0.55547, "mean": 0.45, "std": 0.16359}, "word_cont": {"share": 0.26381, "mean": 0.33282, "std": 0.12744}, "punct": {"share": 0.10701, "mean": 0.59426, "std": 0.16043}}, "alpha_p05": 0.17667, "alpha_p50": 0.42393, "alpha_p95": 0.75491}
{"step": 37810, "total_steps": 72479, "tokens": 19823329280, "loss": 4.5631521344184875, "l_mix": 2.828147768974304, "l_a": 2.9168078005313873, "l_c": 2.866539865732193, "z_loss": 32.51499056816101, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12331987172365189, "tokens_per_second": 1457550.4394292454, "eta_seconds": 12470.608344173432, "data_wait_seconds": 1.9878149032592773e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 37820, "total_steps": 72479, "tokens": 19828572160, "loss": 4.502957880496979, "l_mix": 2.790589392185211, "l_a": 2.8779881596565247, "l_c": 2.829906791448593, "z_loss": 32.394487619400024, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12712997198104858, "tokens_per_second": 1427113.2527006648, "eta_seconds": 12732.905225014687, "data_wait_seconds": 1.9758939743041992e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 37830, "total_steps": 72479, "tokens": 19833815040, "loss": 4.507934331893921, "l_mix": 2.7929261922836304, "l_a": 2.8815130293369293, "l_c": 2.835180550813675, "z_loss": 33.12147331237793, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11644826084375381, "tokens_per_second": 1394798.841594707, "eta_seconds": 13024.139661049843, "data_wait_seconds": 1.990795135498047e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 37840, "total_steps": 72479, "tokens": 19839057920, "loss": 4.502183377742767, "l_mix": 2.7902153730392456, "l_a": 2.879089832305908, "l_c": 2.827469915151596, "z_loss": 32.56277942657471, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13619999587535858, "tokens_per_second": 1458956.3918313372, "eta_seconds": 12447.810046744346, "data_wait_seconds": 2.5719404220581055e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 37850, "total_steps": 72479, "tokens": 19844300800, "loss": 2.929961860179901, "l_mix": 1.8131089806556702, "l_a": 1.8788678348064423, "l_c": 1.8439751118421555, "z_loss": 38.87478160858154, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1978686898946762, "tokens_per_second": 1399612.3869236214, "eta_seconds": 12971.855151915552, "data_wait_seconds": 1.722574234008789e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 37860, "total_steps": 72479, "tokens": 19849543680, "loss": 2.9687307476997375, "l_mix": 1.8383903950452805, "l_a": 1.9037083685398102, "l_c": 1.8640926033258438, "z_loss": 39.453773498535156, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.24999083578586578, "tokens_per_second": 1392320.3584158132, "eta_seconds": 13036.027349805832, "data_wait_seconds": 3.0487775802612305e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08236}}
{"step": 37870, "total_steps": 72479, "tokens": 19854786560, "loss": 4.534255087375641, "l_mix": 2.810056149959564, "l_a": 2.8977657556533813, "l_c": 2.8495639264583588, "z_loss": 32.04545259475708, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12320590019226074, "tokens_per_second": 1362581.2354112715, "eta_seconds": 13316.698425340654, "data_wait_seconds": 2.339482307434082e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 37880, "total_steps": 72479, "tokens": 19860029440, "loss": 4.518104434013367, "l_mix": 2.8001746237277985, "l_a": 2.8872794210910797, "l_c": 2.8391532003879547, "z_loss": 32.77115058898926, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1210196390748024, "tokens_per_second": 1380813.8574928516, "eta_seconds": 13137.06435778141, "data_wait_seconds": 1.683831214904785e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 37890, "total_steps": 72479, "tokens": 19865272320, "loss": 4.54428505897522, "l_mix": 2.8169021010398865, "l_a": 2.9017040133476257, "l_c": 2.8562387228012085, "z_loss": 33.10530090332031, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12095649540424347, "tokens_per_second": 1479182.7944891304, "eta_seconds": 12259.875993394851, "data_wait_seconds": 3.197789192199707e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 37900, "total_steps": 72479, "tokens": 19870515200, "loss": 4.491740703582764, "l_mix": 2.783262997865677, "l_a": 2.8739642202854156, "l_c": 2.8209610879421234, "z_loss": 32.56050729751587, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12771283090114594, "tokens_per_second": 1367216.8587259627, "eta_seconds": 13260.043303513527, "data_wait_seconds": 1.615285873413086e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.22766495, "gate_a2b/L1_absmax": 0.72210366, "gate_a2b/L1_rms": 0.24095957, "gate_a2b/L3_absmean": 0.23045121, "gate_a2b/L3_absmax": 0.6105544, "gate_a2b/L3_rms": 0.23667544, "gate_a2b/L5_absmean": 0.36103341, "gate_a2b/L5_absmax": 0.69671828, "gate_a2b/L5_rms": 0.36693799, "gate_c2b/L1_absmean": 0.01758857, "gate_c2b/L1_absmax": 0.6151014, "gate_c2b/L1_rms": 0.06784134, "gate_c2b/L3_absmean": 0.20860413, "gate_c2b/L3_absmax": 0.47434652, "gate_c2b/L3_rms": 0.21300553, "gate_c2b/L5_absmean": 0.58530045, "gate_c2b/L5_absmax": 0.81913304, "gate_c2b/L5_rms": 0.59838849, "gate_b2a/L7_absmean": 0.05552603, "gate_b2a/L7_absmax": 0.32978976, "gate_b2a/L7_rms": 0.06107321, "gate_b2a/L11_absmean": 0.0882405, "gate_b2a/L11_absmax": 0.58855885, "gate_b2a/L11_rms": 0.09634811, "gate_b2a/L14_absmean": 0.22647035, "gate_b2a/L14_absmax": 0.75761938, "gate_b2a/L14_rms": 0.23394679, "gate_b2c/L3_absmean": 0.04347718, "gate_b2c/L3_absmax": 0.62025636, "gate_b2c/L3_rms": 0.05578598, "gate_b2c/L5_absmean": 0.04746498, "gate_b2c/L5_absmax": 0.65540427, "gate_b2c/L5_rms": 0.05883093, "gate_b2c/L6_absmean": 0.06886314, "gate_b2c/L6_absmax": 0.44207284, "gate_b2c/L6_rms": 0.07676142}, "streams": {"cos_a2b/round0": 0.38323, "cos_c2b/round0": 0.310899, "cos_b2a/round0": 0.143864, "cos_b2c/round0": 0.128445, "rms_a/round0": 3.922286, "rms_b/round0": 7.482716, "rms_c/round0": 3.157527, "cos_a2b/round1": 0.426703, "cos_c2b/round1": 0.300375, "cos_b2a/round1": 0.250607, "cos_b2c/round1": 0.117567, "rms_a/round1": 7.42598, "rms_b/round1": 16.702499, "rms_c/round1": 6.256771, "cos_a2b/round2": 0.557603, "cos_c2b/round2": 0.52475, "cos_b2a/round2": 0.102135, "cos_b2c/round2": 0.177142, "rms_a/round2": 19.48774, "rms_b/round2": 42.692715, "rms_c/round2": 11.486911}, "alpha_mean": 0.43272060075256746, "alpha_hist": [0.00832, 0.07286, 0.17975, 0.20428, 0.17694, 0.15671, 0.12691, 0.06744, 0.00676, 3e-05], "alpha_by_type": {"special": {"share": 0.00111, "mean": 0.58495, "std": 0.11323}, "whitespace": {"share": 0.02502, "mean": 0.54366, "std": 0.12207}, "digit": {"share": 0.03378, "mean": 0.47267, "std": 0.11428}, "word_start": {"share": 0.59569, "mean": 0.44894, "std": 0.16557}, "word_cont": {"share": 0.24901, "mean": 0.3181, "std": 0.12767}, "punct": {"share": 0.09538, "mean": 0.58563, "std": 0.16016}}, "alpha_p05": 0.16506, "alpha_p50": 0.41822, "alpha_p95": 0.73564}
{"step": 37910, "total_steps": 72479, "tokens": 19875758080, "loss": 4.5190624594688416, "l_mix": 2.800059288740158, "l_a": 2.893444836139679, "l_c": 2.8365656435489655, "z_loss": 32.3137001991272, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15126942098140717, "tokens_per_second": 1476260.5551191194, "eta_seconds": 12277.04134554863, "data_wait_seconds": 2.4110078811645508e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 37920, "total_steps": 72479, "tokens": 19881000960, "loss": 4.9875470995903015, "l_mix": 3.094841957092285, "l_a": 3.1737902462482452, "l_c": 3.135226845741272, "z_loss": 31.661063194274902, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12699665129184723, "tokens_per_second": 1347185.398963297, "eta_seconds": 13449.425005602836, "data_wait_seconds": 2.8967857360839844e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 37930, "total_steps": 72479, "tokens": 19886243840, "loss": 4.738388121128082, "l_mix": 2.938924789428711, "l_a": 3.018968790769577, "l_c": 2.9792418777942657, "z_loss": 31.741026878356934, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11937703937292099, "tokens_per_second": 1342243.064984203, "eta_seconds": 13495.041683983802, "data_wait_seconds": 2.1964311599731445e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 37940, "total_steps": 72479, "tokens": 19891486720, "loss": 4.704011499881744, "l_mix": 2.918396085500717, "l_a": 2.994422435760498, "l_c": 2.957628607749939, "z_loss": 32.03747606277466, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13359764218330383, "tokens_per_second": 1435341.0220969033, "eta_seconds": 12616.084228920938, "data_wait_seconds": 1.9311904907226562e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 37950, "total_steps": 72479, "tokens": 19896729600, "loss": 4.659904181957245, "l_mix": 2.8906520009040833, "l_a": 2.96888467669487, "l_c": 2.9286221861839294, "z_loss": 32.41029119491577, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15350712835788727, "tokens_per_second": 1430170.4737059532, "eta_seconds": 12658.029713821412, "data_wait_seconds": 2.5391578674316406e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 37960, "total_steps": 72479, "tokens": 19901972480, "loss": 4.683804750442505, "l_mix": 2.904583126306534, "l_a": 2.985938638448715, "l_c": 2.944799840450287, "z_loss": 33.2290096282959, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1410425901412964, "tokens_per_second": 1444773.1960364357, "eta_seconds": 12526.462645936013, "data_wait_seconds": 2.0563602447509766e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 37970, "total_steps": 72479, "tokens": 19907215360, "loss": 2.8763516545295715, "l_mix": 1.780894696712494, "l_a": 1.8433716595172882, "l_c": 1.8081514984369278, "z_loss": 39.748878479003906, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2013881802558899, "tokens_per_second": 1440470.3938003962, "eta_seconds": 12560.24050884247, "data_wait_seconds": 2.3603439331054688e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 37980, "total_steps": 72479, "tokens": 19912458240, "loss": 4.572983205318451, "l_mix": 2.833886504173279, "l_a": 2.923148989677429, "l_c": 2.8738399147987366, "z_loss": 33.00900363922119, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12229195237159729, "tokens_per_second": 1410526.4283943472, "eta_seconds": 12823.163996005058, "data_wait_seconds": 1.4901161193847656e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 37990, "total_steps": 72479, "tokens": 19917701120, "loss": 4.545456886291504, "l_mix": 2.815729409456253, "l_a": 2.9103364050388336, "l_c": 2.855421543121338, "z_loss": 32.476051807403564, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11365462094545364, "tokens_per_second": 1429479.3458033232, "eta_seconds": 12649.478906488419, "data_wait_seconds": 1.8477439880371094e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 38000, "total_steps": 72479, "tokens": 19922944000, "loss": 4.509086728096008, "l_mix": 2.79451784491539, "l_a": 2.882264196872711, "l_c": 2.8329654037952423, "z_loss": 32.51043224334717, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14997494220733643, "tokens_per_second": 1396362.3434661983, "eta_seconds": 12945.727186489104, "data_wait_seconds": 1.558661460876465e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.22692934, "gate_a2b/L1_absmax": 0.72146297, "gate_a2b/L1_rms": 0.2401851, "gate_a2b/L3_absmean": 0.22946861, "gate_a2b/L3_absmax": 0.6108011, "gate_a2b/L3_rms": 0.23576997, "gate_a2b/L5_absmean": 0.3594749, "gate_a2b/L5_absmax": 0.69880539, "gate_a2b/L5_rms": 0.3653914, "gate_c2b/L1_absmean": 0.01748342, "gate_c2b/L1_absmax": 0.61484069, "gate_c2b/L1_rms": 0.06788052, "gate_c2b/L3_absmean": 0.20845187, "gate_c2b/L3_absmax": 0.47550887, "gate_c2b/L3_rms": 0.21279491, "gate_c2b/L5_absmean": 0.58508128, "gate_c2b/L5_absmax": 0.81532729, "gate_c2b/L5_rms": 0.59807754, "gate_b2a/L7_absmean": 0.05523018, "gate_b2a/L7_absmax": 0.32930824, "gate_b2a/L7_rms": 0.06085709, "gate_b2a/L11_absmean": 0.08807895, "gate_b2a/L11_absmax": 0.58884579, "gate_b2a/L11_rms": 0.0962513, "gate_b2a/L14_absmean": 0.22607857, "gate_b2a/L14_absmax": 0.75852245, "gate_b2a/L14_rms": 0.23362304, "gate_b2c/L3_absmean": 0.04337404, "gate_b2c/L3_absmax": 0.61889994, "gate_b2c/L3_rms": 0.05560191, "gate_b2c/L5_absmean": 0.04718337, "gate_b2c/L5_absmax": 0.65300494, "gate_b2c/L5_rms": 0.05856973, "gate_b2c/L6_absmean": 0.06821302, "gate_b2c/L6_absmax": 0.44361895, "gate_b2c/L6_rms": 0.07620197}, "streams": {"cos_a2b/round0": 0.382921, "cos_c2b/round0": 0.309105, "cos_b2a/round0": 0.143445, "cos_b2c/round0": 0.125797, "rms_a/round0": 3.968557, "rms_b/round0": 7.541821, "rms_c/round0": 3.190674, "cos_a2b/round1": 0.424258, "cos_c2b/round1": 0.297328, "cos_b2a/round1": 0.249212, "cos_b2c/round1": 0.118047, "rms_a/round1": 7.469923, "rms_b/round1": 16.817894, "rms_c/round1": 6.333436, "cos_a2b/round2": 0.554504, "cos_c2b/round2": 0.52829, "cos_b2a/round2": 0.09416, "cos_b2c/round2": 0.177875, "rms_a/round2": 19.407776, "rms_b/round2": 42.648159, "rms_c/round2": 11.648254}, "alpha_mean": 0.451510365972851, "alpha_hist": [0.0051, 0.0452, 0.15765, 0.21135, 0.19437, 0.17392, 0.12857, 0.07191, 0.01169, 0.00024], "alpha_by_type": {"special": {"share": 0.00136, "mean": 0.5644, "std": 0.11066}, "whitespace": {"share": 0.0294, "mean": 0.53963, "std": 0.11142}, "digit": {"share": 0.03606, "mean": 0.51175, "std": 0.10605}, "word_start": {"share": 0.56931, "mean": 0.46374, "std": 0.16224}, "word_cont": {"share": 0.25931, "mean": 0.34942, "std": 0.12649}, "punct": {"share": 0.10457, "mean": 0.59107, "std": 0.15926}}, "alpha_p05": 0.18922, "alpha_p50": 0.45145, "alpha_p95": 0.74017}
{"step": 38010, "total_steps": 72479, "tokens": 19928186880, "loss": 4.468886196613312, "l_mix": 2.7689473032951355, "l_a": 2.8598223328590393, "l_c": 2.8066404461860657, "z_loss": 32.982555866241455, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1274873912334442, "tokens_per_second": 1344341.9791023363, "eta_seconds": 13442.772265481948, "data_wait_seconds": 4.1991472244262695e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 38020, "total_steps": 72479, "tokens": 19933429760, "loss": 4.5477758049964905, "l_mix": 2.818412661552429, "l_a": 2.9077762365341187, "l_c": 2.856767475605011, "z_loss": 32.220877170562744, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14100606739521027, "tokens_per_second": 1340747.253251308, "eta_seconds": 13474.903751015663, "data_wait_seconds": 1.940131187438965e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 38030, "total_steps": 72479, "tokens": 19938672640, "loss": 3.0724038183689117, "l_mix": 1.9017544090747833, "l_a": 1.9696730822324753, "l_c": 1.9324913769960403, "z_loss": 38.26366186141968, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2458702027797699, "tokens_per_second": 1332546.8959148265, "eta_seconds": 13553.892450141906, "data_wait_seconds": 2.6285648345947266e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08236}}
{"step": 38040, "total_steps": 72479, "tokens": 19943915520, "loss": 4.737456679344177, "l_mix": 2.9383186995983124, "l_a": 3.0178735852241516, "l_c": 2.9792530834674835, "z_loss": 32.63026762008667, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12126374989748001, "tokens_per_second": 1287347.692866091, "eta_seconds": 14025.701472926141, "data_wait_seconds": 2.053380012512207e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20768, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 38050, "total_steps": 72479, "tokens": 19949158400, "loss": 4.777240991592407, "l_mix": 2.963362991809845, "l_a": 3.0417794287204742, "l_c": 3.004480481147766, "z_loss": 33.18299865722656, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13450376689434052, "tokens_per_second": 1348962.8496566438, "eta_seconds": 13381.177662968636, "data_wait_seconds": 2.181529998779297e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 38060, "total_steps": 72479, "tokens": 19954401280, "loss": 4.69473123550415, "l_mix": 2.9121013283729553, "l_a": 2.989678591489792, "l_c": 2.952421188354492, "z_loss": 33.28197240829468, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12113756686449051, "tokens_per_second": 1359113.9434419274, "eta_seconds": 13277.377337694168, "data_wait_seconds": 3.3468008041381836e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 38070, "total_steps": 72479, "tokens": 19959644160, "loss": 4.777355909347534, "l_mix": 2.9633378386497498, "l_a": 3.0425735116004944, "l_c": 3.0041528046131134, "z_loss": 32.759052753448486, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14698198437690735, "tokens_per_second": 1431576.9167609648, "eta_seconds": 12601.646185255051, "data_wait_seconds": 3.62396240234375e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 38080, "total_steps": 72479, "tokens": 19964887040, "loss": 3.1304425299167633, "l_mix": 1.937314659357071, "l_a": 2.010025680065155, "l_c": 1.9670671075582504, "z_loss": 39.85381364822388, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1820727437734604, "tokens_per_second": 1460606.1973363713, "eta_seconds": 12347.601252746583, "data_wait_seconds": 1.6808509826660156e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 38090, "total_steps": 72479, "tokens": 19970129920, "loss": 4.450472891330719, "l_mix": 2.7569713592529297, "l_a": 2.849368393421173, "l_c": 2.7956365942955017, "z_loss": 32.68347787857056, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15045641362667084, "tokens_per_second": 1468274.0027723978, "eta_seconds": 12279.547276568414, "data_wait_seconds": 2.682209014892578e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 38100, "total_steps": 72479, "tokens": 19975372800, "loss": 4.4598318338394165, "l_mix": 2.763511747121811, "l_a": 2.852201610803604, "l_c": 2.802198737859726, "z_loss": 33.46076250076294, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11815136671066284, "tokens_per_second": 1334352.2131669135, "eta_seconds": 13508.050553774834, "data_wait_seconds": 1.9818544387817383e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.22633505, "gate_a2b/L1_absmax": 0.72068429, "gate_a2b/L1_rms": 0.23962735, "gate_a2b/L3_absmean": 0.22871195, "gate_a2b/L3_absmax": 0.61090863, "gate_a2b/L3_rms": 0.23501559, "gate_a2b/L5_absmean": 0.35840532, "gate_a2b/L5_absmax": 0.69864851, "gate_a2b/L5_rms": 0.3643305, "gate_c2b/L1_absmean": 0.01745159, "gate_c2b/L1_absmax": 0.61501211, "gate_c2b/L1_rms": 0.06762189, "gate_c2b/L3_absmean": 0.20781933, "gate_c2b/L3_absmax": 0.47782466, "gate_c2b/L3_rms": 0.21217079, "gate_c2b/L5_absmean": 0.58472675, "gate_c2b/L5_absmax": 0.81812209, "gate_c2b/L5_rms": 0.59770602, "gate_b2a/L7_absmean": 0.05522317, "gate_b2a/L7_absmax": 0.32813942, "gate_b2a/L7_rms": 0.06077921, "gate_b2a/L11_absmean": 0.08782098, "gate_b2a/L11_absmax": 0.58875811, "gate_b2a/L11_rms": 0.09606729, "gate_b2a/L14_absmean": 0.22567044, "gate_b2a/L14_absmax": 0.75873214, "gate_b2a/L14_rms": 0.23319995, "gate_b2c/L3_absmean": 0.04275404, "gate_b2c/L3_absmax": 0.61815065, "gate_b2c/L3_rms": 0.055073, "gate_b2c/L5_absmean": 0.04694084, "gate_b2c/L5_absmax": 0.65140921, "gate_b2c/L5_rms": 0.0583264, "gate_b2c/L6_absmean": 0.06806417, "gate_b2c/L6_absmax": 0.44441471, "gate_b2c/L6_rms": 0.07608441}, "streams": {"cos_a2b/round0": 0.38054, "cos_c2b/round0": 0.304802, "cos_b2a/round0": 0.130537, "cos_b2c/round0": 0.122867, "rms_a/round0": 4.004312, "rms_b/round0": 7.547942, "rms_c/round0": 3.164512, "cos_a2b/round1": 0.407687, "cos_c2b/round1": 0.281779, "cos_b2a/round1": 0.233458, "cos_b2c/round1": 0.109393, "rms_a/round1": 7.440375, "rms_b/round1": 16.918989, "rms_c/round1": 6.286366, "cos_a2b/round2": 0.548033, "cos_c2b/round2": 0.513141, "cos_b2a/round2": 0.095472, "cos_b2c/round2": 0.17039, "rms_a/round2": 19.560186, "rms_b/round2": 43.629333, "rms_c/round2": 11.275236}, "alpha_mean": 0.4300090885185952, "alpha_hist": [0.00822, 0.05617, 0.18439, 0.22818, 0.18285, 0.15381, 0.12241, 0.05638, 0.00742, 0.00018], "alpha_by_type": {"special": {"share": 0.00053, "mean": 0.55562, "std": 0.092}, "whitespace": {"share": 0.03133, "mean": 0.53135, "std": 0.13908}, "digit": {"share": 0.08307, "mean": 0.48636, "std": 0.12382}, "word_start": {"share": 0.46867, "mean": 0.4391, "std": 0.16246}, "word_cont": {"share": 0.28545, "mean": 0.32334, "std": 0.11376}, "punct": {"share": 0.13095, "mean": 0.56947, "std": 0.15613}}, "alpha_p05": 0.18477, "alpha_p50": 0.41324, "alpha_p95": 0.71859}
{"step": 38110, "total_steps": 72479, "tokens": 19980615680, "loss": 4.515985727310181, "l_mix": 2.798560321331024, "l_a": 2.8859823048114777, "l_c": 2.838769167661667, "z_loss": 33.01273775100708, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12799598276615143, "tokens_per_second": 1326005.1803212299, "eta_seconds": 13589.128111577034, "data_wait_seconds": 2.238154411315918e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 38120, "total_steps": 72479, "tokens": 19985858560, "loss": 4.506936728954315, "l_mix": 2.793005108833313, "l_a": 2.8810400664806366, "l_c": 2.8320652544498444, "z_loss": 32.69294881820679, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11618413031101227, "tokens_per_second": 1446515.3199758036, "eta_seconds": 12453.384449672698, "data_wait_seconds": 3.072619438171387e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 38130, "total_steps": 72479, "tokens": 19991101440, "loss": 4.543222427368164, "l_mix": 2.8152669966220856, "l_a": 2.9050772190093994, "l_c": 2.854774057865143, "z_loss": 32.91051769256592, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1320752054452896, "tokens_per_second": 1461570.380982638, "eta_seconds": 12321.519884586334, "data_wait_seconds": 1.609325408935547e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 38140, "total_steps": 72479, "tokens": 19996344320, "loss": 3.036191940307617, "l_mix": 1.879594847559929, "l_a": 1.9467281252145767, "l_c": 1.9085953086614609, "z_loss": 39.52716875076294, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15777015686035156, "tokens_per_second": 1403381.4661613782, "eta_seconds": 12828.675642442702, "data_wait_seconds": 1.7017126083374023e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 38150, "total_steps": 72479, "tokens": 20001587200, "loss": 3.1675414741039276, "l_mix": 1.947972759604454, "l_a": 2.0659790486097336, "l_c": 1.999249905347824, "z_loss": 36.40266132354736, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20937761664390564, "tokens_per_second": 1443086.7789003125, "eta_seconds": 12472.07237648964, "data_wait_seconds": 1.6450881958007812e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20045, "finemath": 0.08233}}
{"step": 38160, "total_steps": 72479, "tokens": 20006830080, "loss": 3.159744918346405, "l_mix": 1.9446217715740204, "l_a": 2.0564173758029938, "l_c": 1.9939929395914078, "z_loss": 34.431626319885254, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18650716543197632, "tokens_per_second": 1464678.3500129948, "eta_seconds": 12284.63564839363, "data_wait_seconds": 1.9878149032592773e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20045, "finemath": 0.08234}}
{"step": 38170, "total_steps": 72479, "tokens": 20012072960, "loss": 3.1478756070137024, "l_mix": 1.9367274940013885, "l_a": 2.0493034273386, "l_c": 1.9878565967082977, "z_loss": 34.40664625167847, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1336417943239212, "tokens_per_second": 1436259.369339237, "eta_seconds": 12524.058938097953, "data_wait_seconds": 1.9758939743041992e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20045, "finemath": 0.08234}}
{"step": 38180, "total_steps": 72479, "tokens": 20017315840, "loss": 4.707723259925842, "l_mix": 2.9194479882717133, "l_a": 3.0010368525981903, "l_c": 2.9598802626132965, "z_loss": 33.673184871673584, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15316766500473022, "tokens_per_second": 1368554.616577558, "eta_seconds": 13139.814731669427, "data_wait_seconds": 2.5779008865356445e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 38190, "total_steps": 72479, "tokens": 20022558720, "loss": 4.710166096687317, "l_mix": 2.921062558889389, "l_a": 3.002953380346298, "l_c": 2.9607246220111847, "z_loss": 32.11918830871582, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12935061752796173, "tokens_per_second": 1462418.247296633, "eta_seconds": 12292.86578257084, "data_wait_seconds": 2.0831823348999023e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 38200, "total_steps": 72479, "tokens": 20027801600, "loss": 4.526105761528015, "l_mix": 2.8052156567573547, "l_a": 2.892852157354355, "l_c": 2.8434478044509888, "z_loss": 32.49052047729492, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13440048694610596, "tokens_per_second": 1440131.6338117328, "eta_seconds": 12479.46224501133, "data_wait_seconds": 2.2470951080322266e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.22550116, "gate_a2b/L1_absmax": 0.71964628, "gate_a2b/L1_rms": 0.23880453, "gate_a2b/L3_absmean": 0.22819372, "gate_a2b/L3_absmax": 0.61019367, "gate_a2b/L3_rms": 0.23444796, "gate_a2b/L5_absmean": 0.35772997, "gate_a2b/L5_absmax": 0.69756126, "gate_a2b/L5_rms": 0.36367372, "gate_c2b/L1_absmean": 0.01738205, "gate_c2b/L1_absmax": 0.61337239, "gate_c2b/L1_rms": 0.06768458, "gate_c2b/L3_absmean": 0.20780371, "gate_c2b/L3_absmax": 0.47543931, "gate_c2b/L3_rms": 0.21216644, "gate_c2b/L5_absmean": 0.58398026, "gate_c2b/L5_absmax": 0.8153733, "gate_c2b/L5_rms": 0.59695899, "gate_b2a/L7_absmean": 0.05493142, "gate_b2a/L7_absmax": 0.3284193, "gate_b2a/L7_rms": 0.06047622, "gate_b2a/L11_absmean": 0.08777382, "gate_b2a/L11_absmax": 0.58692765, "gate_b2a/L11_rms": 0.0959006, "gate_b2a/L14_absmean": 0.22521004, "gate_b2a/L14_absmax": 0.75969779, "gate_b2a/L14_rms": 0.23277162, "gate_b2c/L3_absmean": 0.04295754, "gate_b2c/L3_absmax": 0.61807609, "gate_b2c/L3_rms": 0.05520038, "gate_b2c/L5_absmean": 0.04670938, "gate_b2c/L5_absmax": 0.65292549, "gate_b2c/L5_rms": 0.05816308, "gate_b2c/L6_absmean": 0.06793942, "gate_b2c/L6_absmax": 0.44199854, "gate_b2c/L6_rms": 0.07585312}, "streams": {"cos_a2b/round0": 0.380607, "cos_c2b/round0": 0.306123, "cos_b2a/round0": 0.139852, "cos_b2c/round0": 0.124449, "rms_a/round0": 4.010273, "rms_b/round0": 7.613032, "rms_c/round0": 3.214106, "cos_a2b/round1": 0.423803, "cos_c2b/round1": 0.295902, "cos_b2a/round1": 0.247893, "cos_b2c/round1": 0.11533, "rms_a/round1": 7.549888, "rms_b/round1": 16.96921, "rms_c/round1": 6.349518, "cos_a2b/round2": 0.554026, "cos_c2b/round2": 0.525236, "cos_b2a/round2": 0.097744, "cos_b2c/round2": 0.174648, "rms_a/round2": 19.651131, "rms_b/round2": 43.410389, "rms_c/round2": 11.69136}, "alpha_mean": 0.43567697797202243, "alpha_hist": [0.00613, 0.06226, 0.1813, 0.2081, 0.18451, 0.16133, 0.12083, 0.06714, 0.00835, 5e-05], "alpha_by_type": {"special": {"share": 0.00119, "mean": 0.5637, "std": 0.13132}, "whitespace": {"share": 0.02817, "mean": 0.52096, "std": 0.1322}, "digit": {"share": 0.04379, "mean": 0.45139, "std": 0.12411}, "word_start": {"share": 0.5658, "mean": 0.44815, "std": 0.16686}, "word_cont": {"share": 0.25818, "mean": 0.33644, "std": 0.12718}, "punct": {"share": 0.10287, "mean": 0.58459, "std": 0.15394}}, "alpha_p05": 0.19042, "alpha_p50": 0.42919, "alpha_p95": 0.73373}
{"step": 38210, "total_steps": 72479, "tokens": 20033044480, "loss": 4.47208696603775, "l_mix": 2.771058201789856, "l_a": 2.860109359025955, "l_c": 2.809986263513565, "z_loss": 32.22821092605591, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14702463150024414, "tokens_per_second": 1420086.6815566465, "eta_seconds": 12651.921678686142, "data_wait_seconds": 1.58846378326416e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 38220, "total_steps": 72479, "tokens": 20038287360, "loss": 4.40161007642746, "l_mix": 2.7269244492053986, "l_a": 2.8163914382457733, "l_c": 2.765893667936325, "z_loss": 33.1437292098999, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11285298317670822, "tokens_per_second": 1413435.5141180726, "eta_seconds": 12707.74818701744, "data_wait_seconds": 1.952052116394043e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 38230, "total_steps": 72479, "tokens": 20043530240, "loss": 4.491862714290619, "l_mix": 2.7831433415412903, "l_a": 2.8760500252246857, "l_c": 2.8196811974048615, "z_loss": 32.93808126449585, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15100136399269104, "tokens_per_second": 1375810.5750677232, "eta_seconds": 13051.46219792366, "data_wait_seconds": 1.9609928131103516e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 38240, "total_steps": 72479, "tokens": 20048773120, "loss": 4.594813168048859, "l_mix": 2.8485049307346344, "l_a": 2.9320541322231293, "l_c": 2.8889734148979187, "z_loss": 32.247154712677, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.10810660570859909, "tokens_per_second": 1409961.7234076753, "eta_seconds": 12731.619968104364, "data_wait_seconds": 1.5616416931152344e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 38250, "total_steps": 72479, "tokens": 20054016000, "loss": 2.980372726917267, "l_mix": 1.8443372547626495, "l_a": 1.914163276553154, "l_c": 1.872621476650238, "z_loss": 38.45034122467041, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18672989308834076, "tokens_per_second": 1442837.2846155083, "eta_seconds": 12437.891745209694, "data_wait_seconds": 1.749396324157715e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 38260, "total_steps": 72479, "tokens": 20059258880, "loss": 3.2255617678165436, "l_mix": 1.9837495684623718, "l_a": 2.104605048894882, "l_c": 2.03476881980896, "z_loss": 35.21910762786865, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2137591540813446, "tokens_per_second": 1399283.2204884968, "eta_seconds": 12821.28650534153, "data_wait_seconds": 1.6450881958007812e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 38270, "total_steps": 72479, "tokens": 20064501760, "loss": 3.250534564256668, "l_mix": 2.0004986822605133, "l_a": 2.113971024751663, "l_c": 2.0528151094913483, "z_loss": 34.25647592544556, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19146351516246796, "tokens_per_second": 1295816.2581158548, "eta_seconds": 13840.980987596511, "data_wait_seconds": 2.22623348236084e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 38280, "total_steps": 72479, "tokens": 20069744640, "loss": 3.2325853407382965, "l_mix": 1.989460438489914, "l_a": 2.1051370203495026, "l_c": 2.0386126935482025, "z_loss": 34.576870918273926, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15122957527637482, "tokens_per_second": 1302170.9699549284, "eta_seconds": 13769.409490537644, "data_wait_seconds": 2.4318695068359375e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 38290, "total_steps": 72479, "tokens": 20074987520, "loss": 3.2143694460392, "l_mix": 1.9765930473804474, "l_a": 2.0974605083465576, "l_c": 2.0284606367349625, "z_loss": 34.98538780212402, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15310777723789215, "tokens_per_second": 1456350.2893708944, "eta_seconds": 12308.084506058693, "data_wait_seconds": 2.041459083557129e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 38300, "total_steps": 72479, "tokens": 20080230400, "loss": 4.5488258600234985, "l_mix": 2.8189183473587036, "l_a": 2.906533181667328, "l_c": 2.8598246574401855, "z_loss": 32.62484574317932, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13338446617126465, "tokens_per_second": 1415899.2768314583, "eta_seconds": 12656.012927770615, "data_wait_seconds": 2.7477741241455078e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.22514449, "gate_a2b/L1_absmax": 0.72064674, "gate_a2b/L1_rms": 0.23838249, "gate_a2b/L3_absmean": 0.22720142, "gate_a2b/L3_absmax": 0.61148691, "gate_a2b/L3_rms": 0.23347472, "gate_a2b/L5_absmean": 0.35660076, "gate_a2b/L5_absmax": 0.70150411, "gate_a2b/L5_rms": 0.36255351, "gate_c2b/L1_absmean": 0.01760448, "gate_c2b/L1_absmax": 0.61537462, "gate_c2b/L1_rms": 0.06749988, "gate_c2b/L3_absmean": 0.20774484, "gate_c2b/L3_absmax": 0.469758, "gate_c2b/L3_rms": 0.21209961, "gate_c2b/L5_absmean": 0.58313644, "gate_c2b/L5_absmax": 0.8146013, "gate_c2b/L5_rms": 0.59609324, "gate_b2a/L7_absmean": 0.05467283, "gate_b2a/L7_absmax": 0.32944205, "gate_b2a/L7_rms": 0.06028828, "gate_b2a/L11_absmean": 0.08758557, "gate_b2a/L11_absmax": 0.58817053, "gate_b2a/L11_rms": 0.09582434, "gate_b2a/L14_absmean": 0.22478887, "gate_b2a/L14_absmax": 0.75991088, "gate_b2a/L14_rms": 0.23234162, "gate_b2c/L3_absmean": 0.04260693, "gate_b2c/L3_absmax": 0.61669326, "gate_b2c/L3_rms": 0.05483454, "gate_b2c/L5_absmean": 0.04655426, "gate_b2c/L5_absmax": 0.65364915, "gate_b2c/L5_rms": 0.05803012, "gate_b2c/L6_absmean": 0.06763162, "gate_b2c/L6_absmax": 0.44154927, "gate_b2c/L6_rms": 0.07559694}, "streams": {"cos_a2b/round0": 0.37801, "cos_c2b/round0": 0.308289, "cos_b2a/round0": 0.141433, "cos_b2c/round0": 0.125388, "rms_a/round0": 4.019407, "rms_b/round0": 7.59986, "rms_c/round0": 3.231889, "cos_a2b/round1": 0.422265, "cos_c2b/round1": 0.299057, "cos_b2a/round1": 0.24621, "cos_b2c/round1": 0.116786, "rms_a/round1": 7.564825, "rms_b/round1": 17.112238, "rms_c/round1": 6.339973, "cos_a2b/round2": 0.551884, "cos_c2b/round2": 0.522207, "cos_b2a/round2": 0.09813, "cos_b2c/round2": 0.175678, "rms_a/round2": 19.813372, "rms_b/round2": 43.685463, "rms_c/round2": 11.566788}, "alpha_mean": 0.4265846978055521, "alpha_hist": [0.01416, 0.08125, 0.18549, 0.19925, 0.17976, 0.14252, 0.11246, 0.07277, 0.01221, 0.00014], "alpha_by_type": {"special": {"share": 0.00085, "mean": 0.56565, "std": 0.10842}, "whitespace": {"share": 0.02361, "mean": 0.50844, "std": 0.12888}, "digit": {"share": 0.02721, "mean": 0.45953, "std": 0.11824}, "word_start": {"share": 0.59334, "mean": 0.44152, "std": 0.17369}, "word_cont": {"share": 0.25153, "mean": 0.31878, "std": 0.13749}, "punct": {"share": 0.10347, "mean": 0.57452, "std": 0.17206}}, "alpha_p05": 0.16886, "alpha_p50": 0.40216, "alpha_p95": 0.72506}
{"step": 38310, "total_steps": 72479, "tokens": 20085473280, "loss": 4.4671995639801025, "l_mix": 2.7679027020931244, "l_a": 2.859706610441208, "l_c": 2.8046161234378815, "z_loss": 32.7251296043396, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11164535582065582, "tokens_per_second": 1416111.452879567, "eta_seconds": 12650.414369273185, "data_wait_seconds": 1.8030405044555664e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 38320, "total_steps": 72479, "tokens": 20090716160, "loss": 4.56354421377182, "l_mix": 2.8293173015117645, "l_a": 2.913594961166382, "l_c": 2.867161214351654, "z_loss": 33.02542495727539, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1122974157333374, "tokens_per_second": 1458957.843764053, "eta_seconds": 12275.305875730513, "data_wait_seconds": 1.7195940017700195e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 38330, "total_steps": 72479, "tokens": 20095959040, "loss": 4.5147881507873535, "l_mix": 2.7973307371139526, "l_a": 2.888299882411957, "l_c": 2.8365582525730133, "z_loss": 33.2201042175293, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1128167062997818, "tokens_per_second": 1442481.892453256, "eta_seconds": 12411.87914085388, "data_wait_seconds": 1.96993350982666e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 38340, "total_steps": 72479, "tokens": 20101201920, "loss": 4.574571073055267, "l_mix": 2.8355250656604767, "l_a": 2.921597868204117, "l_c": 2.8752219676971436, "z_loss": 32.84452152252197, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12422726303339005, "tokens_per_second": 1471400.3384733663, "eta_seconds": 12164.376726031303, "data_wait_seconds": 1.8477439880371094e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 38350, "total_steps": 72479, "tokens": 20106444800, "loss": 4.539915382862091, "l_mix": 2.813500255346298, "l_a": 2.9027852714061737, "l_c": 2.851931571960449, "z_loss": 33.16528367996216, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11181016266345978, "tokens_per_second": 1409979.1714821744, "eta_seconds": 12690.559913158417, "data_wait_seconds": 1.7702579498291016e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 38360, "total_steps": 72479, "tokens": 20111687680, "loss": 3.0326395332813263, "l_mix": 1.875853955745697, "l_a": 1.9485532194375992, "l_c": 1.9073984026908875, "z_loss": 40.52723550796509, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1662360578775406, "tokens_per_second": 1418896.047027099, "eta_seconds": 12607.11262779236, "data_wait_seconds": 2.422928810119629e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 38370, "total_steps": 72479, "tokens": 20116930560, "loss": 3.218844711780548, "l_mix": 1.9794635623693466, "l_a": 2.0996855795383453, "l_c": 2.03158475458622, "z_loss": 34.70189046859741, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18075762689113617, "tokens_per_second": 1404047.8517637406, "eta_seconds": 12736.702221035957, "data_wait_seconds": 2.7447938919067383e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 38380, "total_steps": 72479, "tokens": 20122173440, "loss": 3.2035029232501984, "l_mix": 1.9696549326181412, "l_a": 2.087617665529251, "l_c": 2.0252088755369186, "z_loss": 33.97377395629883, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19888921082019806, "tokens_per_second": 1419353.0423288038, "eta_seconds": 12595.665756750108, "data_wait_seconds": 3.159046173095703e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 38390, "total_steps": 72479, "tokens": 20127416320, "loss": 3.1772142946720123, "l_mix": 1.9541151225566864, "l_a": 2.0727555453777313, "l_c": 2.004241719841957, "z_loss": 36.20558786392212, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18519236147403717, "tokens_per_second": 1435115.4587135876, "eta_seconds": 12453.669510340691, "data_wait_seconds": 1.8209218978881836e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 38400, "total_steps": 72479, "tokens": 20132659200, "loss": 3.2134073078632355, "l_mix": 1.9777765721082687, "l_a": 2.090332567691803, "l_c": 2.028436377644539, "z_loss": 35.19373035430908, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20907263457775116, "tokens_per_second": 1433068.9616590005, "eta_seconds": 12467.795500445367, "data_wait_seconds": 1.8388032913208008e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.22420238, "gate_a2b/L1_absmax": 0.71862864, "gate_a2b/L1_rms": 0.23751856, "gate_a2b/L3_absmean": 0.22682719, "gate_a2b/L3_absmax": 0.61019528, "gate_a2b/L3_rms": 0.2331599, "gate_a2b/L5_absmean": 0.35538027, "gate_a2b/L5_absmax": 0.70157999, "gate_a2b/L5_rms": 0.36129841, "gate_c2b/L1_absmean": 0.01741042, "gate_c2b/L1_absmax": 0.61434501, "gate_c2b/L1_rms": 0.06760576, "gate_c2b/L3_absmean": 0.20768757, "gate_c2b/L3_absmax": 0.47382075, "gate_c2b/L3_rms": 0.21201836, "gate_c2b/L5_absmean": 0.58297962, "gate_c2b/L5_absmax": 0.81765568, "gate_c2b/L5_rms": 0.59582174, "gate_b2a/L7_absmean": 0.05485141, "gate_b2a/L7_absmax": 0.32670581, "gate_b2a/L7_rms": 0.06038305, "gate_b2a/L11_absmean": 0.08709987, "gate_b2a/L11_absmax": 0.58925092, "gate_b2a/L11_rms": 0.09536956, "gate_b2a/L14_absmean": 0.22433564, "gate_b2a/L14_absmax": 0.76180631, "gate_b2a/L14_rms": 0.2319337, "gate_b2c/L3_absmean": 0.0425648, "gate_b2c/L3_absmax": 0.61605859, "gate_b2c/L3_rms": 0.05476582, "gate_b2c/L5_absmean": 0.04657272, "gate_b2c/L5_absmax": 0.65320879, "gate_b2c/L5_rms": 0.0580682, "gate_b2c/L6_absmean": 0.0674189, "gate_b2c/L6_absmax": 0.44165024, "gate_b2c/L6_rms": 0.07529458}, "streams": {"cos_a2b/round0": 0.377124, "cos_c2b/round0": 0.275686, "cos_b2a/round0": 0.129385, "cos_b2c/round0": 0.103367, "rms_a/round0": 4.233341, "rms_b/round0": 7.781231, "rms_c/round0": 3.290687, "cos_a2b/round1": 0.420634, "cos_c2b/round1": 0.298028, "cos_b2a/round1": 0.234077, "cos_b2c/round1": 0.123277, "rms_a/round1": 8.069952, "rms_b/round1": 17.996344, "rms_c/round1": 6.74018, "cos_a2b/round2": 0.526414, "cos_c2b/round2": 0.510871, "cos_b2a/round2": 0.083435, "cos_b2c/round2": 0.163927, "rms_a/round2": 20.312859, "rms_b/round2": 44.95927, "rms_c/round2": 12.532954}, "alpha_mean": 0.4132147196542064, "alpha_hist": [0.02798, 0.15443, 0.19754, 0.15671, 0.12346, 0.10625, 0.09799, 0.0876, 0.04544, 0.00259], "alpha_by_type": {"special": {"share": 0.00108, "mean": 0.61873, "std": 0.11884}, "whitespace": {"share": 0.03896, "mean": 0.60185, "std": 0.17031}, "digit": {"share": 0.01291, "mean": 0.4074, "std": 0.17275}, "word_start": {"share": 0.57242, "mean": 0.41592, "std": 0.20767}, "word_cont": {"share": 0.25536, "mean": 0.28023, "std": 0.13951}, "punct": {"share": 0.11928, "mean": 0.62209, "std": 0.18316}}, "alpha_p05": 0.12003, "alpha_p50": 0.35981, "alpha_p95": 0.80958}
{"step": 38410, "total_steps": 72479, "tokens": 20137902080, "loss": 4.595299422740936, "l_mix": 2.8485048413276672, "l_a": 2.9380965530872345, "l_c": 2.884551703929901, "z_loss": 32.03101396560669, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1437249630689621, "tokens_per_second": 1408941.3558242163, "eta_seconds": 12677.580793666839, "data_wait_seconds": 1.913309097290039e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 38420, "total_steps": 72479, "tokens": 20143144960, "loss": 4.50744241476059, "l_mix": 2.7938331067562103, "l_a": 2.8808279633522034, "l_c": 2.8312030136585236, "z_loss": 31.870509386062622, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13186456263065338, "tokens_per_second": 1435373.3449171546, "eta_seconds": 12440.474149274825, "data_wait_seconds": 1.665949821472168e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 38430, "total_steps": 72479, "tokens": 20148387840, "loss": 4.587264657020569, "l_mix": 2.8424653708934784, "l_a": 2.933529108762741, "l_c": 2.8824680745601654, "z_loss": 32.88297510147095, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12099593132734299, "tokens_per_second": 1414348.5951994467, "eta_seconds": 12621.698902654649, "data_wait_seconds": 2.1517276763916016e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 38440, "total_steps": 72479, "tokens": 20153630720, "loss": 3.1630521714687347, "l_mix": 1.9456994831562042, "l_a": 2.0616031289100647, "l_c": 1.99623903632164, "z_loss": 35.02698278427124, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15601076185703278, "tokens_per_second": 1317918.695496038, "eta_seconds": 13541.229282951355, "data_wait_seconds": 2.0891427993774414e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 38450, "total_steps": 72479, "tokens": 20158873600, "loss": 3.1877012848854065, "l_mix": 1.9614781439304352, "l_a": 2.0788681507110596, "l_c": 2.008542224764824, "z_loss": 36.47127103805542, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.264780730009079, "tokens_per_second": 1468592.0995869888, "eta_seconds": 12148.367376494407, "data_wait_seconds": 1.7017126083374023e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 38460, "total_steps": 72479, "tokens": 20164116480, "loss": 4.749632835388184, "l_mix": 2.9465711414813995, "l_a": 3.0228933691978455, "l_c": 2.9873120188713074, "z_loss": 32.86701154708862, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1362617015838623, "tokens_per_second": 1412224.7117650304, "eta_seconds": 12629.54353044033, "data_wait_seconds": 2.2649765014648438e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 38470, "total_steps": 72479, "tokens": 20169359360, "loss": 4.696749627590179, "l_mix": 2.9133094251155853, "l_a": 2.99236398935318, "l_c": 2.9524363577365875, "z_loss": 32.81808662414551, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17675358057022095, "tokens_per_second": 1383947.8570917451, "eta_seconds": 12883.802305579185, "data_wait_seconds": 1.850724220275879e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 38480, "total_steps": 72479, "tokens": 20174602240, "loss": 3.200766235589981, "l_mix": 1.9686319082975388, "l_a": 2.087892174720764, "l_c": 2.01922208070755, "z_loss": 34.74853515625, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14486686885356903, "tokens_per_second": 1470118.309880548, "eta_seconds": 12125.056597280503, "data_wait_seconds": 2.092123031616211e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 38490, "total_steps": 72479, "tokens": 20179845120, "loss": 3.2121199667453766, "l_mix": 1.9760535210371017, "l_a": 2.0930487513542175, "l_c": 2.02717262506485, "z_loss": 34.28227996826172, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1509416699409485, "tokens_per_second": 1441277.3236333358, "eta_seconds": 12364.049957489968, "data_wait_seconds": 3.713369369506836e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 38500, "total_steps": 72479, "tokens": 20185088000, "loss": 3.218459129333496, "l_mix": 1.9796713292598724, "l_a": 2.0968681275844574, "l_c": 2.032424345612526, "z_loss": 34.877756118774414, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16652145981788635, "tokens_per_second": 1384363.7021485174, "eta_seconds": 12868.570538473128, "data_wait_seconds": 2.6851892471313477e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.22384082, "gate_a2b/L1_absmax": 0.71820891, "gate_a2b/L1_rms": 0.23692952, "gate_a2b/L3_absmean": 0.22633815, "gate_a2b/L3_absmax": 0.61165786, "gate_a2b/L3_rms": 0.23259376, "gate_a2b/L5_absmean": 0.35418305, "gate_a2b/L5_absmax": 0.70146018, "gate_a2b/L5_rms": 0.36016938, "gate_c2b/L1_absmean": 0.01774902, "gate_c2b/L1_absmax": 0.61335909, "gate_c2b/L1_rms": 0.06765988, "gate_c2b/L3_absmean": 0.20724578, "gate_c2b/L3_absmax": 0.46937469, "gate_c2b/L3_rms": 0.21153748, "gate_c2b/L5_absmean": 0.58213788, "gate_c2b/L5_absmax": 0.81166053, "gate_c2b/L5_rms": 0.59505987, "gate_b2a/L7_absmean": 0.05463304, "gate_b2a/L7_absmax": 0.32979974, "gate_b2a/L7_rms": 0.06018151, "gate_b2a/L11_absmean": 0.08733578, "gate_b2a/L11_absmax": 0.58858043, "gate_b2a/L11_rms": 0.0955792, "gate_b2a/L14_absmean": 0.2245034, "gate_b2a/L14_absmax": 0.76296341, "gate_b2a/L14_rms": 0.23206681, "gate_b2c/L3_absmean": 0.04209073, "gate_b2c/L3_absmax": 0.61541855, "gate_b2c/L3_rms": 0.05437096, "gate_b2c/L5_absmean": 0.04650127, "gate_b2c/L5_absmax": 0.65306681, "gate_b2c/L5_rms": 0.05792273, "gate_b2c/L6_absmean": 0.06740366, "gate_b2c/L6_absmax": 0.44001046, "gate_b2c/L6_rms": 0.07523029}, "streams": {"cos_a2b/round0": 0.374423, "cos_c2b/round0": 0.276675, "cos_b2a/round0": 0.129748, "cos_b2c/round0": 0.103505, "rms_a/round0": 4.23497, "rms_b/round0": 7.83092, "rms_c/round0": 3.290591, "cos_a2b/round1": 0.417864, "cos_c2b/round1": 0.294448, "cos_b2a/round1": 0.229636, "cos_b2c/round1": 0.118573, "rms_a/round1": 8.114104, "rms_b/round1": 18.146477, "rms_c/round1": 6.703202, "cos_a2b/round2": 0.524472, "cos_c2b/round2": 0.51235, "cos_b2a/round2": 0.082701, "cos_b2c/round2": 0.160425, "rms_a/round2": 20.552608, "rms_b/round2": 45.594933, "rms_c/round2": 12.477746}, "alpha_mean": 0.41201346183704146, "alpha_hist": [0.03801, 0.15814, 0.18686, 0.15202, 0.12245, 0.10777, 0.09653, 0.08586, 0.0484, 0.00395], "alpha_by_type": {"special": {"share": 0.00104, "mean": 0.60434, "std": 0.1259}, "whitespace": {"share": 0.04419, "mean": 0.60153, "std": 0.16869}, "digit": {"share": 0.0118, "mean": 0.41184, "std": 0.17162}, "word_start": {"share": 0.55487, "mean": 0.41482, "std": 0.20938}, "word_cont": {"share": 0.26012, "mean": 0.27013, "std": 0.14123}, "punct": {"share": 0.12799, "mean": 0.62124, "std": 0.19094}}, "alpha_p05": 0.11556, "alpha_p50": 0.39607, "alpha_p95": 0.80837}
{"step": 38510, "total_steps": 72479, "tokens": 20190330880, "loss": 3.2190661430358887, "l_mix": 1.981093019247055, "l_a": 2.097752094268799, "l_c": 2.028824806213379, "z_loss": 35.313660621643066, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1720939427614212, "tokens_per_second": 1444094.5337705717, "eta_seconds": 12332.668433761597, "data_wait_seconds": 2.181529998779297e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 38520, "total_steps": 72479, "tokens": 20195573760, "loss": 4.5710238218307495, "l_mix": 2.833259791135788, "l_a": 2.918327182531357, "l_c": 2.8742194175720215, "z_loss": 32.45271301269531, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14636734127998352, "tokens_per_second": 1435948.0908961855, "eta_seconds": 12398.983156061171, "data_wait_seconds": 3.606081008911133e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 38530, "total_steps": 72479, "tokens": 20200816640, "loss": 4.4179277420043945, "l_mix": 2.7376388609409332, "l_a": 2.8260088860988617, "l_c": 2.7749538719654083, "z_loss": 33.03924560546875, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.151161789894104, "tokens_per_second": 1405163.493453822, "eta_seconds": 12666.89135813713, "data_wait_seconds": 1.5944242477416992e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 38540, "total_steps": 72479, "tokens": 20206059520, "loss": 4.514891743659973, "l_mix": 2.797340154647827, "l_a": 2.888630300760269, "l_c": 2.8365414142608643, "z_loss": 33.16292977333069, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11000298708677292, "tokens_per_second": 1475924.5660528992, "eta_seconds": 12056.043270277978, "data_wait_seconds": 1.8030405044555664e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 38550, "total_steps": 72479, "tokens": 20211302400, "loss": 3.225382089614868, "l_mix": 1.984779253602028, "l_a": 2.100992977619171, "l_c": 2.0343497544527054, "z_loss": 34.04957723617554, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1493067443370819, "tokens_per_second": 1466556.271883986, "eta_seconds": 12129.481761479377, "data_wait_seconds": 1.952052116394043e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 38560, "total_steps": 72479, "tokens": 20216545280, "loss": 3.1867313385009766, "l_mix": 1.9601452201604843, "l_a": 2.0749599635601044, "l_c": 2.0136601328849792, "z_loss": 35.40596961975098, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1546943336725235, "tokens_per_second": 1356988.9502230985, "eta_seconds": 13104.988562417031, "data_wait_seconds": 2.390146255493164e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 38570, "total_steps": 72479, "tokens": 20221788160, "loss": 3.2129806578159332, "l_mix": 1.977957844734192, "l_a": 2.0913059413433075, "l_c": 2.0254365354776382, "z_loss": 35.08621168136597, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18057182431221008, "tokens_per_second": 1408540.2985279597, "eta_seconds": 12621.635185432433, "data_wait_seconds": 2.205371856689453e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 38580, "total_steps": 72479, "tokens": 20227031040, "loss": 3.2499670684337616, "l_mix": 1.9992493391036987, "l_a": 2.1137149333953857, "l_c": 2.0553439557552338, "z_loss": 34.975637912750244, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17190046608448029, "tokens_per_second": 1390863.1789989092, "eta_seconds": 12778.279833960532, "data_wait_seconds": 3.063678741455078e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 38590, "total_steps": 72479, "tokens": 20232273920, "loss": 4.72198873758316, "l_mix": 2.9286336600780487, "l_a": 3.0088744461536407, "l_c": 2.9689754843711853, "z_loss": 32.34741830825806, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16223573684692383, "tokens_per_second": 1383973.290260059, "eta_seconds": 12838.106166529655, "data_wait_seconds": 2.1159648895263672e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 38600, "total_steps": 72479, "tokens": 20237516800, "loss": 3.169086456298828, "l_mix": 1.9489970654249191, "l_a": 2.0649584531784058, "l_c": 2.0020060390233994, "z_loss": 34.80367183685303, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1719447821378708, "tokens_per_second": 1441608.0226978422, "eta_seconds": 12321.208589529992, "data_wait_seconds": 1.7076730728149414e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.22357415, "gate_a2b/L1_absmax": 0.71742219, "gate_a2b/L1_rms": 0.23662296, "gate_a2b/L3_absmean": 0.22582822, "gate_a2b/L3_absmax": 0.60980678, "gate_a2b/L3_rms": 0.23202923, "gate_a2b/L5_absmean": 0.35328269, "gate_a2b/L5_absmax": 0.70096028, "gate_a2b/L5_rms": 0.35923168, "gate_c2b/L1_absmean": 0.01659441, "gate_c2b/L1_absmax": 0.61307877, "gate_c2b/L1_rms": 0.06734977, "gate_c2b/L3_absmean": 0.20639718, "gate_c2b/L3_absmax": 0.46594092, "gate_c2b/L3_rms": 0.21080792, "gate_c2b/L5_absmean": 0.58186591, "gate_c2b/L5_absmax": 0.81238246, "gate_c2b/L5_rms": 0.59478086, "gate_b2a/L7_absmean": 0.05433252, "gate_b2a/L7_absmax": 0.32658461, "gate_b2a/L7_rms": 0.0598479, "gate_b2a/L11_absmean": 0.08724293, "gate_b2a/L11_absmax": 0.58833939, "gate_b2a/L11_rms": 0.09550281, "gate_b2a/L14_absmean": 0.22419512, "gate_b2a/L14_absmax": 0.76135945, "gate_b2a/L14_rms": 0.23177938, "gate_b2c/L3_absmean": 0.04220343, "gate_b2c/L3_absmax": 0.61344361, "gate_b2c/L3_rms": 0.05442449, "gate_b2c/L5_absmean": 0.04651461, "gate_b2c/L5_absmax": 0.6486178, "gate_b2c/L5_rms": 0.05779017, "gate_b2c/L6_absmean": 0.06738582, "gate_b2c/L6_absmax": 0.43786502, "gate_b2c/L6_rms": 0.07517554}, "streams": {"cos_a2b/round0": 0.37634, "cos_c2b/round0": 0.273653, "cos_b2a/round0": 0.12584, "cos_b2c/round0": 0.101709, "rms_a/round0": 4.290656, "rms_b/round0": 7.867526, "rms_c/round0": 3.325662, "cos_a2b/round1": 0.417962, "cos_c2b/round1": 0.294059, "cos_b2a/round1": 0.230515, "cos_b2c/round1": 0.12039, "rms_a/round1": 8.183646, "rms_b/round1": 18.436806, "rms_c/round1": 6.820049, "cos_a2b/round2": 0.521893, "cos_c2b/round2": 0.511424, "cos_b2a/round2": 0.081317, "cos_b2c/round2": 0.161279, "rms_a/round2": 20.604559, "rms_b/round2": 45.921501, "rms_c/round2": 12.610425}, "alpha_mean": 0.40908600786545435, "alpha_hist": [0.03676, 0.15817, 0.19627, 0.15617, 0.11652, 0.10333, 0.09683, 0.08597, 0.04623, 0.00374], "alpha_by_type": {"special": {"share": 0.00105, "mean": 0.5748, "std": 0.15075}, "whitespace": {"share": 0.04106, "mean": 0.60128, "std": 0.16896}, "digit": {"share": 0.009, "mean": 0.40181, "std": 0.17202}, "word_start": {"share": 0.55891, "mean": 0.41369, "std": 0.21075}, "word_cont": {"share": 0.26335, "mean": 0.27031, "std": 0.13968}, "punct": {"share": 0.12662, "mean": 0.61422, "std": 0.18848}}, "alpha_p05": 0.11398, "alpha_p50": 0.36342, "alpha_p95": 0.79343}
{"step": 38610, "total_steps": 72479, "tokens": 20242759680, "loss": 3.192193776369095, "l_mix": 1.9634175598621368, "l_a": 2.0832685232162476, "l_c": 2.0126520693302155, "z_loss": 36.2121376991272, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.196363165974617, "tokens_per_second": 1465724.4108333418, "eta_seconds": 12114.90382554531, "data_wait_seconds": 2.154707908630371e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 38620, "total_steps": 72479, "tokens": 20248002560, "loss": 3.166917234659195, "l_mix": 1.9486036747694016, "l_a": 2.063592165708542, "l_c": 1.9974530637264252, "z_loss": 35.52594184875488, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16740219295024872, "tokens_per_second": 1443387.7070693022, "eta_seconds": 12298.75196044445, "data_wait_seconds": 1.7583370208740234e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 38630, "total_steps": 72479, "tokens": 20253245440, "loss": 4.5821614265441895, "l_mix": 2.840065509080887, "l_a": 2.925698310136795, "l_c": 2.881287842988968, "z_loss": 32.586909532547, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16076385974884033, "tokens_per_second": 1417830.350205551, "eta_seconds": 12516.74751455784, "data_wait_seconds": 1.7821788787841797e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 38640, "total_steps": 72479, "tokens": 20258488320, "loss": 4.438892364501953, "l_mix": 2.7503042817115784, "l_a": 2.8409222066402435, "l_c": 2.7877043783664703, "z_loss": 33.376790046691895, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12171322852373123, "tokens_per_second": 1444458.596206034, "eta_seconds": 12282.374640989303, "data_wait_seconds": 1.77919864654541e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 38650, "total_steps": 72479, "tokens": 20263731200, "loss": 4.526478588581085, "l_mix": 2.804987460374832, "l_a": 2.893041104078293, "l_c": 2.8452623188495636, "z_loss": 33.31774854660034, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1331714540719986, "tokens_per_second": 1409889.2235990777, "eta_seconds": 12579.810140490532, "data_wait_seconds": 2.6494264602661133e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 38660, "total_steps": 72479, "tokens": 20268974080, "loss": 3.2214584350585938, "l_mix": 1.9821578711271286, "l_a": 2.0961417853832245, "l_c": 2.034859851002693, "z_loss": 34.53328466415405, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17294980585575104, "tokens_per_second": 1455499.134322749, "eta_seconds": 12182.003722214698, "data_wait_seconds": 1.6123056411743164e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 38670, "total_steps": 72479, "tokens": 20274216960, "loss": 3.2215530276298523, "l_mix": 1.9826378673315048, "l_a": 2.094947963953018, "l_c": 2.0347691774368286, "z_loss": 35.40181016921997, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1684836596250534, "tokens_per_second": 1447916.8328737777, "eta_seconds": 12242.176200699805, "data_wait_seconds": 2.065300941467285e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 38680, "total_steps": 72479, "tokens": 20279459840, "loss": 3.1926844716072083, "l_mix": 1.9635025411844254, "l_a": 2.0838151425123215, "l_c": 2.0134575963020325, "z_loss": 35.542325019836426, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18329539895057678, "tokens_per_second": 1464107.2838746833, "eta_seconds": 12103.218327760696, "data_wait_seconds": 2.625584602355957e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 38690, "total_steps": 72479, "tokens": 20284702720, "loss": 3.219413071870804, "l_mix": 1.9824559390544891, "l_a": 2.0951696038246155, "l_c": 2.0280207097530365, "z_loss": 34.487783432006836, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.22890231013298035, "tokens_per_second": 1403630.4914799444, "eta_seconds": 12620.962097597123, "data_wait_seconds": 3.6329030990600586e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 38700, "total_steps": 72479, "tokens": 20289945600, "loss": 3.2222855985164642, "l_mix": 1.9816934615373611, "l_a": 2.0976775884628296, "l_c": 2.0376293808221817, "z_loss": 35.067978382110596, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20582036674022675, "tokens_per_second": 1287072.2223381752, "eta_seconds": 13759.85282304287, "data_wait_seconds": 2.47955322265625e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.22319727, "gate_a2b/L1_absmax": 0.7191757, "gate_a2b/L1_rms": 0.23627195, "gate_a2b/L3_absmean": 0.22445254, "gate_a2b/L3_absmax": 0.61164039, "gate_a2b/L3_rms": 0.23077612, "gate_a2b/L5_absmean": 0.35150024, "gate_a2b/L5_absmax": 0.70147991, "gate_a2b/L5_rms": 0.35748136, "gate_c2b/L1_absmean": 0.01765618, "gate_c2b/L1_absmax": 0.6154381, "gate_c2b/L1_rms": 0.06767514, "gate_c2b/L3_absmean": 0.20604722, "gate_c2b/L3_absmax": 0.46727353, "gate_c2b/L3_rms": 0.21040574, "gate_c2b/L5_absmean": 0.58061409, "gate_c2b/L5_absmax": 0.81388301, "gate_c2b/L5_rms": 0.59346795, "gate_b2a/L7_absmean": 0.054435, "gate_b2a/L7_absmax": 0.32915825, "gate_b2a/L7_rms": 0.05998853, "gate_b2a/L11_absmean": 0.08692214, "gate_b2a/L11_absmax": 0.5893144, "gate_b2a/L11_rms": 0.09528936, "gate_b2a/L14_absmean": 0.22314534, "gate_b2a/L14_absmax": 0.76283711, "gate_b2a/L14_rms": 0.23082614, "gate_b2c/L3_absmean": 0.04207356, "gate_b2c/L3_absmax": 0.61322272, "gate_b2c/L3_rms": 0.05429694, "gate_b2c/L5_absmean": 0.04610571, "gate_b2c/L5_absmax": 0.64770609, "gate_b2c/L5_rms": 0.05749102, "gate_b2c/L6_absmean": 0.06675246, "gate_b2c/L6_absmax": 0.43885443, "gate_b2c/L6_rms": 0.07464381}, "streams": {"cos_a2b/round0": 0.371768, "cos_c2b/round0": 0.268102, "cos_b2a/round0": 0.131353, "cos_b2c/round0": 0.100055, "rms_a/round0": 4.298198, "rms_b/round0": 8.016662, "rms_c/round0": 3.340025, "cos_a2b/round1": 0.418144, "cos_c2b/round1": 0.299608, "cos_b2a/round1": 0.234815, "cos_b2c/round1": 0.121984, "rms_a/round1": 8.258886, "rms_b/round1": 18.497046, "rms_c/round1": 6.859989, "cos_a2b/round2": 0.521381, "cos_c2b/round2": 0.512572, "cos_b2a/round2": 0.084907, "cos_b2c/round2": 0.16291, "rms_a/round2": 20.874523, "rms_b/round2": 46.093304, "rms_c/round2": 12.785192}, "alpha_mean": 0.4051373003236236, "alpha_hist": [0.0372, 0.17126, 0.19539, 0.15047, 0.11357, 0.09804, 0.09789, 0.08748, 0.04575, 0.00296], "alpha_by_type": {"special": {"share": 0.00116, "mean": 0.59984, "std": 0.14061}, "whitespace": {"share": 0.03015, "mean": 0.62162, "std": 0.17199}, "digit": {"share": 0.01237, "mean": 0.418, "std": 0.17047}, "word_start": {"share": 0.58377, "mean": 0.40592, "std": 0.21076}, "word_cont": {"share": 0.25363, "mean": 0.26865, "std": 0.14088}, "punct": {"share": 0.11891, "mean": 0.63431, "std": 0.18016}}, "alpha_p05": 0.10933, "alpha_p50": 0.35712, "alpha_p95": 0.77899}
{"step": 38710, "total_steps": 72479, "tokens": 20295188480, "loss": 4.708212494850159, "l_mix": 2.9202995002269745, "l_a": 2.9985466301441193, "l_c": 2.961163103580475, "z_loss": 34.05988025665283, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18072783946990967, "tokens_per_second": 1351507.08381474, "eta_seconds": 13099.954624009131, "data_wait_seconds": 2.467632293701172e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 38720, "total_steps": 72479, "tokens": 20300431360, "loss": 4.593002647161484, "l_mix": 2.8486598134040833, "l_a": 2.9245121479034424, "l_c": 2.8899638652801514, "z_loss": 32.33261227607727, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17464111745357513, "tokens_per_second": 1385020.6066414278, "eta_seconds": 12779.18791036606, "data_wait_seconds": 1.9073486328125e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 38730, "total_steps": 72479, "tokens": 20305674240, "loss": 3.1680329144001007, "l_mix": 1.948928713798523, "l_a": 2.0646957010030746, "l_c": 1.998984694480896, "z_loss": 34.61020374298096, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1861962229013443, "tokens_per_second": 1449151.916074233, "eta_seconds": 12210.03506653309, "data_wait_seconds": 1.761317253112793e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 38740, "total_steps": 72479, "tokens": 20310917120, "loss": 4.456282258033752, "l_mix": 2.761081635951996, "l_a": 2.8516293466091156, "l_c": 2.7990393340587616, "z_loss": 33.283421993255615, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11169516295194626, "tokens_per_second": 1422115.6469493937, "eta_seconds": 12438.477046465874, "data_wait_seconds": 1.749396324157715e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08234}}
{"step": 38750, "total_steps": 72479, "tokens": 20316160000, "loss": 4.524664103984833, "l_mix": 2.8037586212158203, "l_a": 2.8925039172172546, "l_c": 2.8438476026058197, "z_loss": 33.184704303741455, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.10796508938074112, "tokens_per_second": 1362907.7186442812, "eta_seconds": 12974.987014961243, "data_wait_seconds": 1.9609928131103516e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 38760, "total_steps": 72479, "tokens": 20321402880, "loss": 4.45245760679245, "l_mix": 2.758213698863983, "l_a": 2.851670950651169, "l_c": 2.7958085536956787, "z_loss": 33.58126163482666, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1422739028930664, "tokens_per_second": 1477792.9221738903, "eta_seconds": 11962.749859428406, "data_wait_seconds": 1.800060272216797e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 38770, "total_steps": 72479, "tokens": 20326645760, "loss": 3.1950516998767853, "l_mix": 1.9653519690036774, "l_a": 2.0800683796405792, "l_c": 2.0189305245876312, "z_loss": 36.200305461883545, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15488672256469727, "tokens_per_second": 1454976.3074613626, "eta_seconds": 12146.743628311156, "data_wait_seconds": 2.613663673400879e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 38780, "total_steps": 72479, "tokens": 20331888640, "loss": 3.212271958589554, "l_mix": 1.9761677384376526, "l_a": 2.0948179364204407, "l_c": 2.0255292654037476, "z_loss": 35.26438331604004, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14969393610954285, "tokens_per_second": 1446394.67776588, "eta_seconds": 12215.186894416809, "data_wait_seconds": 3.0606985092163086e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 38790, "total_steps": 72479, "tokens": 20337131520, "loss": 3.2829552590847015, "l_mix": 2.0201273411512375, "l_a": 2.136870354413986, "l_c": 2.072556108236313, "z_loss": 34.23177766799927, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14537568390369415, "tokens_per_second": 1382049.631178911, "eta_seconds": 12780.104298377037, "data_wait_seconds": 2.008676528930664e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 38800, "total_steps": 72479, "tokens": 20342374400, "loss": 3.1989704966545105, "l_mix": 1.9677552282810211, "l_a": 2.0863721817731857, "l_c": 2.0176787078380585, "z_loss": 36.133299350738525, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16131195425987244, "tokens_per_second": 1444225.6054564486, "eta_seconds": 12226.272325658798, "data_wait_seconds": 1.5437602996826172e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.22217607, "gate_a2b/L1_absmax": 0.72308558, "gate_a2b/L1_rms": 0.2353932, "gate_a2b/L3_absmean": 0.22447418, "gate_a2b/L3_absmax": 0.60712421, "gate_a2b/L3_rms": 0.23080648, "gate_a2b/L5_absmean": 0.35076895, "gate_a2b/L5_absmax": 0.70563656, "gate_a2b/L5_rms": 0.35673675, "gate_c2b/L1_absmean": 0.0187339, "gate_c2b/L1_absmax": 0.61758524, "gate_c2b/L1_rms": 0.06788997, "gate_c2b/L3_absmean": 0.20662656, "gate_c2b/L3_absmax": 0.47472903, "gate_c2b/L3_rms": 0.21091861, "gate_c2b/L5_absmean": 0.58045024, "gate_c2b/L5_absmax": 0.81614298, "gate_c2b/L5_rms": 0.59329301, "gate_b2a/L7_absmean": 0.05409764, "gate_b2a/L7_absmax": 0.33060616, "gate_b2a/L7_rms": 0.05994703, "gate_b2a/L11_absmean": 0.08698343, "gate_b2a/L11_absmax": 0.59295106, "gate_b2a/L11_rms": 0.09542026, "gate_b2a/L14_absmean": 0.22318992, "gate_b2a/L14_absmax": 0.76721102, "gate_b2a/L14_rms": 0.23099723, "gate_b2c/L3_absmean": 0.04247774, "gate_b2c/L3_absmax": 0.61564916, "gate_b2c/L3_rms": 0.05478653, "gate_b2c/L5_absmean": 0.04578473, "gate_b2c/L5_absmax": 0.64830786, "gate_b2c/L5_rms": 0.0572545, "gate_b2c/L6_absmean": 0.0665954, "gate_b2c/L6_absmax": 0.4395597, "gate_b2c/L6_rms": 0.07457319}, "streams": {"cos_a2b/round0": 0.37444, "cos_c2b/round0": 0.26925, "cos_b2a/round0": 0.129791, "cos_b2c/round0": 0.099128, "rms_a/round0": 4.291892, "rms_b/round0": 7.993601, "rms_c/round0": 3.392018, "cos_a2b/round1": 0.420806, "cos_c2b/round1": 0.300414, "cos_b2a/round1": 0.238018, "cos_b2c/round1": 0.122526, "rms_a/round1": 8.243279, "rms_b/round1": 18.568739, "rms_c/round1": 6.921673, "cos_a2b/round2": 0.521563, "cos_c2b/round2": 0.509462, "cos_b2a/round2": 0.082541, "cos_b2c/round2": 0.164809, "rms_a/round2": 20.900686, "rms_b/round2": 46.553955, "rms_c/round2": 12.806012}, "alpha_mean": 0.39568262571560453, "alpha_hist": [0.0421, 0.18951, 0.19202, 0.14795, 0.1109, 0.09319, 0.09196, 0.0836, 0.04514, 0.00363], "alpha_by_type": {"special": {"share": 0.00104, "mean": 0.62464, "std": 0.09591}, "whitespace": {"share": 0.03326, "mean": 0.62677, "std": 0.16979}, "digit": {"share": 0.01479, "mean": 0.42827, "std": 0.18345}, "word_start": {"share": 0.55971, "mean": 0.39672, "std": 0.21596}, "word_cont": {"share": 0.2765, "mean": 0.26987, "std": 0.14245}, "punct": {"share": 0.1147, "mean": 0.62065, "std": 0.19264}}, "alpha_p05": 0.10781, "alpha_p50": 0.36162, "alpha_p95": 0.78858}
{"step": 38810, "total_steps": 72479, "tokens": 20347617280, "loss": 3.2736754417419434, "l_mix": 2.0137918889522552, "l_a": 2.1343158781528473, "l_c": 2.0652956068515778, "z_loss": 35.38037109375, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14813700318336487, "tokens_per_second": 1390897.7524190233, "eta_seconds": 12691.26550912857, "data_wait_seconds": 1.659989356994629e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 38820, "total_steps": 72479, "tokens": 20352860160, "loss": 4.709984600543976, "l_mix": 2.920145571231842, "l_a": 3.0090350210666656, "l_c": 2.9570952653884888, "z_loss": 32.80525541305542, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11896944791078568, "tokens_per_second": 1422635.4599893345, "eta_seconds": 12404.449550366402, "data_wait_seconds": 1.9222497940063477e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 38830, "total_steps": 72479, "tokens": 20358103040, "loss": 4.5896719098091125, "l_mix": 2.843641519546509, "l_a": 2.9328417778015137, "l_c": 2.887259215116501, "z_loss": 34.16650199890137, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13105270266532898, "tokens_per_second": 1239914.8767416982, "eta_seconds": 14228.208115673066, "data_wait_seconds": 1.6421079635620117e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 38840, "total_steps": 72479, "tokens": 20363345920, "loss": 4.4986960887908936, "l_mix": 2.7876594364643097, "l_a": 2.8765152990818024, "l_c": 2.8269400000572205, "z_loss": 33.40543174743652, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12615816295146942, "tokens_per_second": 1136782.2854655006, "eta_seconds": 15514.425459909438, "data_wait_seconds": 2.1278858184814453e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 38850, "total_steps": 72479, "tokens": 20368588800, "loss": 4.414774537086487, "l_mix": 2.7338537871837616, "l_a": 2.8278037011623383, "l_c": 2.775265246629715, "z_loss": 32.77654838562012, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.23086664080619812, "tokens_per_second": 1420855.5977254661, "eta_seconds": 12408.91838707924, "data_wait_seconds": 1.84476375579834e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 38860, "total_steps": 72479, "tokens": 20373831680, "loss": 4.5205846428871155, "l_mix": 2.801545262336731, "l_a": 2.892672896385193, "l_c": 2.837458163499832, "z_loss": 32.84455347061157, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14370356500148773, "tokens_per_second": 1430080.7213513607, "eta_seconds": 12325.205150198935, "data_wait_seconds": 2.3156404495239258e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 38870, "total_steps": 72479, "tokens": 20379074560, "loss": 4.55384749174118, "l_mix": 2.822552442550659, "l_a": 2.911508560180664, "l_c": 2.85947448015213, "z_loss": 33.09560775756836, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15111064910888672, "tokens_per_second": 1429712.3431970305, "eta_seconds": 12324.71376206875, "data_wait_seconds": 3.263354301452637e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 38880, "total_steps": 72479, "tokens": 20384317440, "loss": 3.161607265472412, "l_mix": 1.9431169480085373, "l_a": 2.0635657906532288, "l_c": 1.9980683773756027, "z_loss": 36.68279695510864, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18120944499969482, "tokens_per_second": 1426613.670520083, "eta_seconds": 12347.808573555947, "data_wait_seconds": 1.6868114471435547e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 38890, "total_steps": 72479, "tokens": 20389560320, "loss": 3.2311432361602783, "l_mix": 1.9875873923301697, "l_a": 2.1088655292987823, "l_c": 2.0363206565380096, "z_loss": 35.51370716094971, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1888752579689026, "tokens_per_second": 1428674.1521418581, "eta_seconds": 12326.330399131775, "data_wait_seconds": 2.0772218704223633e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 38900, "total_steps": 72479, "tokens": 20394803200, "loss": 3.1907020211219788, "l_mix": 1.9625124037265778, "l_a": 2.0781890153884888, "l_c": 2.015776216983795, "z_loss": 34.40304899215698, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20011058449745178, "tokens_per_second": 1448862.3260326115, "eta_seconds": 12150.959021902085, "data_wait_seconds": 1.582503318786621e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.22068258, "gate_a2b/L1_absmax": 0.73109466, "gate_a2b/L1_rms": 0.23394844, "gate_a2b/L3_absmean": 0.22314899, "gate_a2b/L3_absmax": 0.60808414, "gate_a2b/L3_rms": 0.22937679, "gate_a2b/L5_absmean": 0.34994268, "gate_a2b/L5_absmax": 0.70574313, "gate_a2b/L5_rms": 0.35580459, "gate_c2b/L1_absmean": 0.01724366, "gate_c2b/L1_absmax": 0.62537062, "gate_c2b/L1_rms": 0.06793401, "gate_c2b/L3_absmean": 0.20625119, "gate_c2b/L3_absmax": 0.46949777, "gate_c2b/L3_rms": 0.21053694, "gate_c2b/L5_absmean": 0.58026314, "gate_c2b/L5_absmax": 0.81558001, "gate_c2b/L5_rms": 0.59297121, "gate_b2a/L7_absmean": 0.05468995, "gate_b2a/L7_absmax": 0.32716712, "gate_b2a/L7_rms": 0.06015966, "gate_b2a/L11_absmean": 0.08661905, "gate_b2a/L11_absmax": 0.58886445, "gate_b2a/L11_rms": 0.09498525, "gate_b2a/L14_absmean": 0.22346745, "gate_b2a/L14_absmax": 0.76939905, "gate_b2a/L14_rms": 0.2312014, "gate_b2c/L3_absmean": 0.04186993, "gate_b2c/L3_absmax": 0.60848677, "gate_b2c/L3_rms": 0.05385426, "gate_b2c/L5_absmean": 0.04575005, "gate_b2c/L5_absmax": 0.63411587, "gate_b2c/L5_rms": 0.05678489, "gate_b2c/L6_absmean": 0.0664234, "gate_b2c/L6_absmax": 0.44145566, "gate_b2c/L6_rms": 0.07426551}, "streams": {"cos_a2b/round0": 0.371792, "cos_c2b/round0": 0.272458, "cos_b2a/round0": 0.129532, "cos_b2c/round0": 0.101153, "rms_a/round0": 4.360386, "rms_b/round0": 8.036032, "rms_c/round0": 3.397686, "cos_a2b/round1": 0.418445, "cos_c2b/round1": 0.300876, "cos_b2a/round1": 0.237608, "cos_b2c/round1": 0.120899, "rms_a/round1": 8.276232, "rms_b/round1": 18.68574, "rms_c/round1": 6.884456, "cos_a2b/round2": 0.522242, "cos_c2b/round2": 0.516322, "cos_b2a/round2": 0.084507, "cos_b2c/round2": 0.162936, "rms_a/round2": 21.008581, "rms_b/round2": 46.897682, "rms_c/round2": 12.763765}, "alpha_mean": 0.41175512112396717, "alpha_hist": [0.03838, 0.16045, 0.18509, 0.14879, 0.12372, 0.10594, 0.10129, 0.08862, 0.04475, 0.00298], "alpha_by_type": {"special": {"share": 0.0011, "mean": 0.59734, "std": 0.13616}, "whitespace": {"share": 0.03418, "mean": 0.62259, "std": 0.16564}, "digit": {"share": 0.00952, "mean": 0.39031, "std": 0.16738}, "word_start": {"share": 0.57312, "mean": 0.4198, "std": 0.21098}, "word_cont": {"share": 0.25549, "mean": 0.27085, "std": 0.14574}, "punct": {"share": 0.12659, "mean": 0.6028, "std": 0.19146}}, "alpha_p05": 0.10781, "alpha_p50": 0.37869, "alpha_p95": 0.77287}
{"step": 38910, "total_steps": 72479, "tokens": 20400046080, "loss": 3.200474292039871, "l_mix": 1.9689224362373352, "l_a": 2.0863867104053497, "l_c": 2.018785774707794, "z_loss": 35.47503137588501, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17097365856170654, "tokens_per_second": 1452032.2396294035, "eta_seconds": 12120.821695041657, "data_wait_seconds": 2.1845102310180664e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 38920, "total_steps": 72479, "tokens": 20405288960, "loss": 3.2565313279628754, "l_mix": 2.0036421418190002, "l_a": 2.1218769550323486, "l_c": 2.054420158267021, "z_loss": 35.706209659576416, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.22238503396511078, "tokens_per_second": 1401481.8430049748, "eta_seconds": 12554.269668078423, "data_wait_seconds": 1.8924474716186523e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 38930, "total_steps": 72479, "tokens": 20410531840, "loss": 4.560780584812164, "l_mix": 2.826649308204651, "l_a": 2.9169363379478455, "l_c": 2.863500863313675, "z_loss": 32.79675483703613, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1359775960445404, "tokens_per_second": 1426826.0150079227, "eta_seconds": 12327.598408627511, "data_wait_seconds": 1.58846378326416e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 38940, "total_steps": 72479, "tokens": 20415774720, "loss": 4.53613543510437, "l_mix": 2.811516612768173, "l_a": 2.901452511548996, "l_c": 2.84727680683136, "z_loss": 32.588395833969116, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1440689116716385, "tokens_per_second": 1340618.679864864, "eta_seconds": 13116.403266716005, "data_wait_seconds": 2.7954578399658203e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 38950, "total_steps": 72479, "tokens": 20421017600, "loss": 4.28203558921814, "l_mix": 2.652744084596634, "l_a": 2.739368200302124, "l_c": 2.691603124141693, "z_loss": 32.37669658660889, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17507663369178772, "tokens_per_second": 1405208.0302872485, "eta_seconds": 12509.786432409286, "data_wait_seconds": 1.806020736694336e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 38960, "total_steps": 72479, "tokens": 20426260480, "loss": 4.538944482803345, "l_mix": 2.8124020099639893, "l_a": 2.9041011035442352, "l_c": 2.8510401844978333, "z_loss": 33.09452676773071, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12320562452077866, "tokens_per_second": 1287396.9072077072, "eta_seconds": 13650.498438835144, "data_wait_seconds": 2.6881694793701172e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 38970, "total_steps": 72479, "tokens": 20431503360, "loss": 4.561061978340149, "l_mix": 2.8270999789237976, "l_a": 2.9129084944725037, "l_c": 2.8669648468494415, "z_loss": 33.89327907562256, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17372943460941315, "tokens_per_second": 1367861.1608558479, "eta_seconds": 12843.676752257346, "data_wait_seconds": 2.3633241653442383e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 38980, "total_steps": 72479, "tokens": 20436746240, "loss": 4.4619545340538025, "l_mix": 2.7649225294589996, "l_a": 2.8544284999370575, "l_c": 2.8023446202278137, "z_loss": 32.71336650848389, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16894082725048065, "tokens_per_second": 1414199.0615271053, "eta_seconds": 12419.131216955186, "data_wait_seconds": 3.0517578125e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 38990, "total_steps": 72479, "tokens": 20441989120, "loss": 4.748844623565674, "l_mix": 2.9452511966228485, "l_a": 3.024508535861969, "l_c": 2.9874693751335144, "z_loss": 32.99304819107056, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15803544223308563, "tokens_per_second": 1451002.847836478, "eta_seconds": 12100.514384365082, "data_wait_seconds": 2.7745962142944336e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 39000, "total_steps": 72479, "tokens": 20447232000, "loss": 4.687419950962067, "l_mix": 2.9080885350704193, "l_a": 2.986282706260681, "l_c": 2.944821923971176, "z_loss": 33.218082427978516, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16979245841503143, "tokens_per_second": 1370486.901955818, "eta_seconds": 12807.592635107041, "data_wait_seconds": 2.512335777282715e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.22109406, "gate_a2b/L1_absmax": 0.72966003, "gate_a2b/L1_rms": 0.23425467, "gate_a2b/L3_absmean": 0.22240658, "gate_a2b/L3_absmax": 0.60904896, "gate_a2b/L3_rms": 0.22878115, "gate_a2b/L5_absmean": 0.34830114, "gate_a2b/L5_absmax": 0.70443571, "gate_a2b/L5_rms": 0.35421264, "gate_c2b/L1_absmean": 0.01729548, "gate_c2b/L1_absmax": 0.62572432, "gate_c2b/L1_rms": 0.06806877, "gate_c2b/L3_absmean": 0.20592391, "gate_c2b/L3_absmax": 0.46556872, "gate_c2b/L3_rms": 0.21021456, "gate_c2b/L5_absmean": 0.57991523, "gate_c2b/L5_absmax": 0.81070304, "gate_c2b/L5_rms": 0.59264338, "gate_b2a/L7_absmean": 0.05416138, "gate_b2a/L7_absmax": 0.32609999, "gate_b2a/L7_rms": 0.05970756, "gate_b2a/L11_absmean": 0.08668382, "gate_b2a/L11_absmax": 0.58961397, "gate_b2a/L11_rms": 0.09500093, "gate_b2a/L14_absmean": 0.22296235, "gate_b2a/L14_absmax": 0.76844603, "gate_b2a/L14_rms": 0.23070945, "gate_b2c/L3_absmean": 0.04177663, "gate_b2c/L3_absmax": 0.60852414, "gate_b2c/L3_rms": 0.05374083, "gate_b2c/L5_absmean": 0.04561055, "gate_b2c/L5_absmax": 0.64010978, "gate_b2c/L5_rms": 0.0568294, "gate_b2c/L6_absmean": 0.06599194, "gate_b2c/L6_absmax": 0.43683708, "gate_b2c/L6_rms": 0.07381643}, "streams": {"cos_a2b/round0": 0.376151, "cos_c2b/round0": 0.315892, "cos_b2a/round0": 0.146235, "cos_b2c/round0": 0.133048, "rms_a/round0": 4.209314, "rms_b/round0": 7.995105, "rms_c/round0": 3.258168, "cos_a2b/round1": 0.426494, "cos_c2b/round1": 0.304457, "cos_b2a/round1": 0.246802, "cos_b2c/round1": 0.12178, "rms_a/round1": 7.903739, "rms_b/round1": 17.618603, "rms_c/round1": 6.515487, "cos_a2b/round2": 0.577916, "cos_c2b/round2": 0.509588, "cos_b2a/round2": 0.116719, "cos_b2c/round2": 0.182853, "rms_a/round2": 21.006443, "rms_b/round2": 44.890167, "rms_c/round2": 11.945535}, "alpha_mean": 0.47923167671904565, "alpha_hist": [0.00713, 0.04465, 0.12909, 0.16222, 0.18187, 0.20427, 0.161, 0.09061, 0.01843, 0.00075], "alpha_by_type": {"special": {"share": 0.00052, "mean": 0.52771, "std": 0.20954}, "whitespace": {"share": 0.0482, "mean": 0.55176, "std": 0.12162}, "digit": {"share": 0.0191, "mean": 0.48029, "std": 0.15481}, "word_start": {"share": 0.60387, "mean": 0.50018, "std": 0.15669}, "word_cont": {"share": 0.22408, "mean": 0.33935, "std": 0.14011}, "punct": {"share": 0.10423, "mean": 0.6246, "std": 0.15353}}, "alpha_p05": 0.18243, "alpha_p50": 0.47963, "alpha_p95": 0.7773}
{"step": 39010, "total_steps": 72479, "tokens": 20452474880, "loss": 3.183267116546631, "l_mix": 1.9586202055215836, "l_a": 2.074453055858612, "l_c": 2.007703050971031, "z_loss": 35.191657066345215, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16765254735946655, "tokens_per_second": 1461900.1572985274, "eta_seconds": 12003.141927576065, "data_wait_seconds": 2.536177635192871e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 39020, "total_steps": 72479, "tokens": 20457717760, "loss": 3.180926412343979, "l_mix": 1.956242486834526, "l_a": 2.071247100830078, "l_c": 2.0110325813293457, "z_loss": 35.811208724975586, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18408288061618805, "tokens_per_second": 1415470.9242895204, "eta_seconds": 12393.15615105629, "data_wait_seconds": 2.2202730178833008e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 39030, "total_steps": 72479, "tokens": 20462960640, "loss": 3.1920759081840515, "l_mix": 1.9633748531341553, "l_a": 2.082184284925461, "l_c": 2.013485908508301, "z_loss": 36.12913656234741, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18748106062412262, "tokens_per_second": 1443641.8461272633, "eta_seconds": 12147.687017416954, "data_wait_seconds": 2.47955322265625e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 39040, "total_steps": 72479, "tokens": 20468203520, "loss": 4.470552206039429, "l_mix": 2.7708562314510345, "l_a": 2.857106864452362, "l_c": 2.808546096086502, "z_loss": 32.35870862007141, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14665096998214722, "tokens_per_second": 1475605.067540595, "eta_seconds": 11881.001778626442, "data_wait_seconds": 1.6003847122192383e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 39050, "total_steps": 72479, "tokens": 20473446400, "loss": 4.55123895406723, "l_mix": 2.8205674290657043, "l_a": 2.9103300869464874, "l_c": 2.8585745990276337, "z_loss": 33.93602275848389, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1642942726612091, "tokens_per_second": 1467396.1278047806, "eta_seconds": 11943.893826556206, "data_wait_seconds": 1.6748905181884766e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 39060, "total_steps": 72479, "tokens": 20478689280, "loss": 4.513282299041748, "l_mix": 2.7966666221618652, "l_a": 2.8859189450740814, "l_c": 2.836133152246475, "z_loss": 33.94119930267334, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11375042796134949, "tokens_per_second": 1336799.728941949, "eta_seconds": 13106.810461330413, "data_wait_seconds": 3.072619438171387e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 39070, "total_steps": 72479, "tokens": 20483932160, "loss": 4.424290060997009, "l_mix": 2.7410106360912323, "l_a": 2.829471528530121, "l_c": 2.7814595997333527, "z_loss": 32.94850730895996, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16065557301044464, "tokens_per_second": 1349088.7247755593, "eta_seconds": 12983.532862091064, "data_wait_seconds": 2.1159648895263672e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 39080, "total_steps": 72479, "tokens": 20489175040, "loss": 4.526448905467987, "l_mix": 2.805308699607849, "l_a": 2.894896298646927, "l_c": 2.8422376215457916, "z_loss": 33.776944637298584, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1764412373304367, "tokens_per_second": 1410909.878749009, "eta_seconds": 12410.92374200821, "data_wait_seconds": 1.6927719116210938e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 39090, "total_steps": 72479, "tokens": 20494417920, "loss": 4.481049954891205, "l_mix": 2.776084840297699, "l_a": 2.869074285030365, "l_c": 2.814142405986786, "z_loss": 34.47210502624512, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19684535264968872, "tokens_per_second": 1409525.1197615769, "eta_seconds": 12419.396991634369, "data_wait_seconds": 2.4080276489257812e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 39100, "total_steps": 72479, "tokens": 20499660800, "loss": 4.787073493003845, "l_mix": 2.9697462022304535, "l_a": 3.0460946559906006, "l_c": 3.0116629004478455, "z_loss": 31.908303022384644, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.22914640605449677, "tokens_per_second": 1349837.7575718253, "eta_seconds": 12964.6759796381, "data_wait_seconds": 1.8268823623657227e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.22056094, "gate_a2b/L1_absmax": 0.72771114, "gate_a2b/L1_rms": 0.23360385, "gate_a2b/L3_absmean": 0.22167854, "gate_a2b/L3_absmax": 0.60775775, "gate_a2b/L3_rms": 0.2279599, "gate_a2b/L5_absmean": 0.34733126, "gate_a2b/L5_absmax": 0.70481831, "gate_a2b/L5_rms": 0.35326737, "gate_c2b/L1_absmean": 0.01737363, "gate_c2b/L1_absmax": 0.62384349, "gate_c2b/L1_rms": 0.06786038, "gate_c2b/L3_absmean": 0.20576476, "gate_c2b/L3_absmax": 0.46842927, "gate_c2b/L3_rms": 0.21006963, "gate_c2b/L5_absmean": 0.5794735, "gate_c2b/L5_absmax": 0.81542408, "gate_c2b/L5_rms": 0.59224373, "gate_b2a/L7_absmean": 0.05412225, "gate_b2a/L7_absmax": 0.32700646, "gate_b2a/L7_rms": 0.05962014, "gate_b2a/L11_absmean": 0.08620057, "gate_b2a/L11_absmax": 0.58893186, "gate_b2a/L11_rms": 0.09456602, "gate_b2a/L14_absmean": 0.22185168, "gate_b2a/L14_absmax": 0.76861143, "gate_b2a/L14_rms": 0.22968724, "gate_b2c/L3_absmean": 0.04164144, "gate_b2c/L3_absmax": 0.6084016, "gate_b2c/L3_rms": 0.05371461, "gate_b2c/L5_absmean": 0.0452772, "gate_b2c/L5_absmax": 0.64124632, "gate_b2c/L5_rms": 0.05659883, "gate_b2c/L6_absmean": 0.06569117, "gate_b2c/L6_absmax": 0.43757612, "gate_b2c/L6_rms": 0.07351615}, "streams": {"cos_a2b/round0": 0.375251, "cos_c2b/round0": 0.313156, "cos_b2a/round0": 0.147529, "cos_b2c/round0": 0.129991, "rms_a/round0": 4.210495, "rms_b/round0": 8.001341, "rms_c/round0": 3.275173, "cos_a2b/round1": 0.423201, "cos_c2b/round1": 0.303726, "cos_b2a/round1": 0.2488, "cos_b2c/round1": 0.12373, "rms_a/round1": 7.893475, "rms_b/round1": 17.622826, "rms_c/round1": 6.580837, "cos_a2b/round2": 0.575577, "cos_c2b/round2": 0.516129, "cos_b2a/round2": 0.110028, "cos_b2c/round2": 0.182064, "rms_a/round2": 20.883398, "rms_b/round2": 44.649578, "rms_c/round2": 12.033304}, "alpha_mean": 0.47153877698719526, "alpha_hist": [0.00554, 0.04654, 0.12758, 0.16649, 0.19598, 0.21826, 0.14886, 0.07964, 0.01088, 0.00023], "alpha_by_type": {"special": {"share": 0.00061, "mean": 0.54977, "std": 0.12688}, "whitespace": {"share": 0.05637, "mean": 0.54708, "std": 0.10057}, "digit": {"share": 0.01945, "mean": 0.47096, "std": 0.11679}, "word_start": {"share": 0.60156, "mean": 0.48965, "std": 0.15589}, "word_cont": {"share": 0.22781, "mean": 0.34872, "std": 0.13659}, "punct": {"share": 0.09419, "mean": 0.60735, "std": 0.15417}}, "alpha_p05": 0.21834, "alpha_p50": 0.47701, "alpha_p95": 0.74466}
{"step": 39110, "total_steps": 72479, "tokens": 20504903680, "loss": 4.706900715827942, "l_mix": 2.9196732938289642, "l_a": 2.999035209417343, "l_c": 2.958389312028885, "z_loss": 33.35111474990845, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15061664581298828, "tokens_per_second": 1407899.838718744, "eta_seconds": 12426.286153936386, "data_wait_seconds": 2.193450927734375e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 39120, "total_steps": 72479, "tokens": 20510146560, "loss": 4.770471155643463, "l_mix": 2.9593698978424072, "l_a": 3.0356540381908417, "l_c": 3.001349836587906, "z_loss": 33.141016721725464, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17877104878425598, "tokens_per_second": 1453254.4761462926, "eta_seconds": 12034.866349339485, "data_wait_seconds": 1.8864870071411133e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 39130, "total_steps": 72479, "tokens": 20515389440, "loss": 4.7923743724823, "l_mix": 2.972890555858612, "l_a": 3.050868481397629, "l_c": 3.01407727599144, "z_loss": 32.94742488861084, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1409682184457779, "tokens_per_second": 1452938.8602340457, "eta_seconds": 12033.872168016433, "data_wait_seconds": 2.09808349609375e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 39140, "total_steps": 72479, "tokens": 20520632320, "loss": 4.697229981422424, "l_mix": 2.913798362016678, "l_a": 2.9911748468875885, "l_c": 2.953596979379654, "z_loss": 32.844902992248535, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11650627106428146, "tokens_per_second": 1435428.343902017, "eta_seconds": 12177.018592572213, "data_wait_seconds": 1.7344951629638672e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 39150, "total_steps": 72479, "tokens": 20525875200, "loss": 3.1888451278209686, "l_mix": 1.9619060456752777, "l_a": 2.080349773168564, "l_c": 2.0094470232725143, "z_loss": 34.46426343917847, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17728805541992188, "tokens_per_second": 1396629.5509791388, "eta_seconds": 12511.545914053917, "data_wait_seconds": 2.2739171981811523e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 39160, "total_steps": 72479, "tokens": 20531118080, "loss": 4.451653957366943, "l_mix": 2.758155435323715, "l_a": 2.8483352065086365, "l_c": 2.796659380197525, "z_loss": 33.81186008453369, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12129757553339005, "tokens_per_second": 1439006.8795135356, "eta_seconds": 12139.44986691475, "data_wait_seconds": 1.7136335372924805e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 39170, "total_steps": 72479, "tokens": 20536360960, "loss": 4.504519522190094, "l_mix": 2.7908636033535004, "l_a": 2.8817870020866394, "l_c": 2.8303993344306946, "z_loss": 33.48247003555298, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16141252219676971, "tokens_per_second": 1472428.0259160663, "eta_seconds": 11860.348135614395, "data_wait_seconds": 1.874566078186035e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 39180, "total_steps": 72479, "tokens": 20541603840, "loss": 4.4698721170425415, "l_mix": 2.7707227766513824, "l_a": 2.856967478990555, "l_c": 2.8068631291389465, "z_loss": 33.281840324401855, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15587778389453888, "tokens_per_second": 1385495.580701429, "eta_seconds": 12600.73749434948, "data_wait_seconds": 1.6570091247558594e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 39190, "total_steps": 72479, "tokens": 20546846720, "loss": 4.510858356952667, "l_mix": 2.7949467599391937, "l_a": 2.884416699409485, "l_c": 2.8352881371974945, "z_loss": 33.17004990577698, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20727227628231049, "tokens_per_second": 1275922.8224855028, "eta_seconds": 13678.745237898826, "data_wait_seconds": 2.065300941467285e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 39200, "total_steps": 72479, "tokens": 20552089600, "loss": 4.402324557304382, "l_mix": 2.7289308309555054, "l_a": 2.8149674236774445, "l_c": 2.7630112171173096, "z_loss": 33.57379770278931, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.23360584676265717, "tokens_per_second": 1408594.3431167577, "eta_seconds": 12386.660813498498, "data_wait_seconds": 1.9043684005737305e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.22001722, "gate_a2b/L1_absmax": 0.72724462, "gate_a2b/L1_rms": 0.23312472, "gate_a2b/L3_absmean": 0.22116041, "gate_a2b/L3_absmax": 0.60582674, "gate_a2b/L3_rms": 0.22749545, "gate_a2b/L5_absmean": 0.34673277, "gate_a2b/L5_absmax": 0.70612091, "gate_a2b/L5_rms": 0.35256067, "gate_c2b/L1_absmean": 0.01734581, "gate_c2b/L1_absmax": 0.62343234, "gate_c2b/L1_rms": 0.06779588, "gate_c2b/L3_absmean": 0.20580025, "gate_c2b/L3_absmax": 0.46720085, "gate_c2b/L3_rms": 0.21005711, "gate_c2b/L5_absmean": 0.57905847, "gate_c2b/L5_absmax": 0.81992656, "gate_c2b/L5_rms": 0.5916813, "gate_b2a/L7_absmean": 0.05392493, "gate_b2a/L7_absmax": 0.32661924, "gate_b2a/L7_rms": 0.05950305, "gate_b2a/L11_absmean": 0.08660628, "gate_b2a/L11_absmax": 0.59060001, "gate_b2a/L11_rms": 0.09495687, "gate_b2a/L14_absmean": 0.221921, "gate_b2a/L14_absmax": 0.76667869, "gate_b2a/L14_rms": 0.22972898, "gate_b2c/L3_absmean": 0.04144452, "gate_b2c/L3_absmax": 0.60708624, "gate_b2c/L3_rms": 0.05347494, "gate_b2c/L5_absmean": 0.0452354, "gate_b2c/L5_absmax": 0.64038914, "gate_b2c/L5_rms": 0.05648723, "gate_b2c/L6_absmean": 0.06566495, "gate_b2c/L6_absmax": 0.43349037, "gate_b2c/L6_rms": 0.07347719}, "streams": {"cos_a2b/round0": 0.377913, "cos_c2b/round0": 0.308964, "cos_b2a/round0": 0.146552, "cos_b2c/round0": 0.129317, "rms_a/round0": 4.257457, "rms_b/round0": 8.086956, "rms_c/round0": 3.346312, "cos_a2b/round1": 0.428054, "cos_c2b/round1": 0.306973, "cos_b2a/round1": 0.256441, "cos_b2c/round1": 0.11927, "rms_a/round1": 7.974694, "rms_b/round1": 18.027113, "rms_c/round1": 6.621762, "cos_a2b/round2": 0.555389, "cos_c2b/round2": 0.527455, "cos_b2a/round2": 0.107835, "cos_b2c/round2": 0.183266, "rms_a/round2": 21.140327, "rms_b/round2": 46.368179, "rms_c/round2": 12.185323}, "alpha_mean": 0.44558462223471196, "alpha_hist": [0.00493, 0.05817, 0.18222, 0.19833, 0.17627, 0.15645, 0.12898, 0.08101, 0.01358, 6e-05], "alpha_by_type": {"special": {"share": 0.00108, "mean": 0.55281, "std": 0.09405}, "whitespace": {"share": 0.0238, "mean": 0.53715, "std": 0.11855}, "digit": {"share": 0.02979, "mean": 0.47539, "std": 0.10111}, "word_start": {"share": 0.58736, "mean": 0.46629, "std": 0.17027}, "word_cont": {"share": 0.25926, "mean": 0.33067, "std": 0.1247}, "punct": {"share": 0.09871, "mean": 0.59193, "std": 0.1705}}, "alpha_p05": 0.19714, "alpha_p50": 0.4225, "alpha_p95": 0.76873}
{"step": 39210, "total_steps": 72479, "tokens": 20557332480, "loss": 3.0087777376174927, "l_mix": 1.86268450319767, "l_a": 1.9306256771087646, "l_c": 1.8896849155426025, "z_loss": 39.592233180999756, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16490530967712402, "tokens_per_second": 1415993.3668282947, "eta_seconds": 12318.233884859084, "data_wait_seconds": 2.3752450942993164e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 39220, "total_steps": 72479, "tokens": 20562575360, "loss": 4.813579976558685, "l_mix": 2.9867470264434814, "l_a": 3.0627387166023254, "l_c": 3.026704281568527, "z_loss": 32.29431629180908, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1518717259168625, "tokens_per_second": 1400496.45166864, "eta_seconds": 12450.79526708126, "data_wait_seconds": 2.3812055587768555e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 39230, "total_steps": 72479, "tokens": 20567818240, "loss": 4.732405126094818, "l_mix": 2.934827208518982, "l_a": 3.019063174724579, "l_c": 2.972862958908081, "z_loss": 33.334322929382324, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15218935906887054, "tokens_per_second": 1417048.2671899702, "eta_seconds": 12301.664040398597, "data_wait_seconds": 1.627206802368164e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 39240, "total_steps": 72479, "tokens": 20573061120, "loss": 4.823017954826355, "l_mix": 2.9916664958000183, "l_a": 3.072471559047699, "l_c": 3.0320331156253815, "z_loss": 33.20476698875427, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19384212791919708, "tokens_per_second": 1416582.5340325683, "eta_seconds": 12302.007411026956, "data_wait_seconds": 2.3663043975830078e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 39250, "total_steps": 72479, "tokens": 20578304000, "loss": 4.7415531873703, "l_mix": 2.941513776779175, "l_a": 3.0164464116096497, "l_c": 2.9836845993995667, "z_loss": 32.882439613342285, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15234684944152832, "tokens_per_second": 1469639.4386050082, "eta_seconds": 11854.313033771514, "data_wait_seconds": 1.6301870346069336e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 39260, "total_steps": 72479, "tokens": 20583546880, "loss": 4.8208059668540955, "l_mix": 2.9909638464450836, "l_a": 3.07103830575943, "l_c": 3.028434932231903, "z_loss": 33.10229682922363, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17541919648647308, "tokens_per_second": 1476948.0702436864, "eta_seconds": 11792.102527427674, "data_wait_seconds": 1.558661460876465e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 39270, "total_steps": 72479, "tokens": 20588789760, "loss": 4.508980572223663, "l_mix": 2.793944388628006, "l_a": 2.8809227347373962, "l_c": 2.835864692926407, "z_loss": 33.488205432891846, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1678311973810196, "tokens_per_second": 1281040.5128721655, "eta_seconds": 13591.357975840569, "data_wait_seconds": 3.072619438171387e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 39280, "total_steps": 72479, "tokens": 20594032640, "loss": 4.481118381023407, "l_mix": 2.7759933173656464, "l_a": 2.868252217769623, "l_c": 2.815497875213623, "z_loss": 33.33765888214111, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19418324530124664, "tokens_per_second": 1324528.4876333475, "eta_seconds": 13141.157381296156, "data_wait_seconds": 2.3931264877319336e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 39290, "total_steps": 72479, "tokens": 20599275520, "loss": 4.520676970481873, "l_mix": 2.8020781576633453, "l_a": 2.888101279735565, "l_c": 2.840561032295227, "z_loss": 33.67834711074829, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13254554569721222, "tokens_per_second": 1410530.771248132, "eta_seconds": 12336.20335457325, "data_wait_seconds": 2.4765729904174805e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 39300, "total_steps": 72479, "tokens": 20604518400, "loss": 4.4784621596336365, "l_mix": 2.7749279737472534, "l_a": 2.8664384186267853, "l_c": 2.8120086789131165, "z_loss": 32.81495666503906, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1678280532360077, "tokens_per_second": 1458837.7303007077, "eta_seconds": 11924.116843628884, "data_wait_seconds": 1.4662742614746094e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.21940635, "gate_a2b/L1_absmax": 0.72666866, "gate_a2b/L1_rms": 0.23250024, "gate_a2b/L3_absmean": 0.22072625, "gate_a2b/L3_absmax": 0.60352302, "gate_a2b/L3_rms": 0.22703069, "gate_a2b/L5_absmean": 0.34517977, "gate_a2b/L5_absmax": 0.70564938, "gate_a2b/L5_rms": 0.35102203, "gate_c2b/L1_absmean": 0.01757717, "gate_c2b/L1_absmax": 0.62334025, "gate_c2b/L1_rms": 0.06761668, "gate_c2b/L3_absmean": 0.20490232, "gate_c2b/L3_absmax": 0.46502915, "gate_c2b/L3_rms": 0.20916742, "gate_c2b/L5_absmean": 0.57851875, "gate_c2b/L5_absmax": 0.81716776, "gate_c2b/L5_rms": 0.59119487, "gate_b2a/L7_absmean": 0.05405478, "gate_b2a/L7_absmax": 0.32673174, "gate_b2a/L7_rms": 0.05954476, "gate_b2a/L11_absmean": 0.08601309, "gate_b2a/L11_absmax": 0.59218353, "gate_b2a/L11_rms": 0.094461, "gate_b2a/L14_absmean": 0.22155052, "gate_b2a/L14_absmax": 0.76672351, "gate_b2a/L14_rms": 0.2293722, "gate_b2c/L3_absmean": 0.04145847, "gate_b2c/L3_absmax": 0.60681558, "gate_b2c/L3_rms": 0.05346253, "gate_b2c/L5_absmean": 0.04523287, "gate_b2c/L5_absmax": 0.63882113, "gate_b2c/L5_rms": 0.05646581, "gate_b2c/L6_absmean": 0.06550588, "gate_b2c/L6_absmax": 0.43296134, "gate_b2c/L6_rms": 0.07324839}, "streams": {"cos_a2b/round0": 0.375834, "cos_c2b/round0": 0.301379, "cos_b2a/round0": 0.13434, "cos_b2c/round0": 0.122313, "rms_a/round0": 4.291719, "rms_b/round0": 8.099125, "rms_c/round0": 3.376172, "cos_a2b/round1": 0.411602, "cos_c2b/round1": 0.284801, "cos_b2a/round1": 0.234773, "cos_b2c/round1": 0.112001, "rms_a/round1": 8.086962, "rms_b/round1": 18.240349, "rms_c/round1": 6.669836, "cos_a2b/round2": 0.548976, "cos_c2b/round2": 0.512129, "cos_b2a/round2": 0.091611, "cos_b2c/round2": 0.170127, "rms_a/round2": 21.147459, "rms_b/round2": 46.488453, "rms_c/round2": 12.022281}, "alpha_mean": 0.4286740120538113, "alpha_hist": [0.01215, 0.07404, 0.18074, 0.20755, 0.18523, 0.14517, 0.11578, 0.06761, 0.01147, 0.00026], "alpha_by_type": {"special": {"share": 0.00081, "mean": 0.56118, "std": 0.10275}, "whitespace": {"share": 0.03087, "mean": 0.52242, "std": 0.11968}, "digit": {"share": 0.05063, "mean": 0.46683, "std": 0.12134}, "word_start": {"share": 0.52238, "mean": 0.44715, "std": 0.16877}, "word_cont": {"share": 0.25935, "mean": 0.3067, "std": 0.12594}, "punct": {"share": 0.13596, "mean": 0.55406, "std": 0.16898}}, "alpha_p05": 0.16185, "alpha_p50": 0.40333, "alpha_p95": 0.74503}
{"step": 39310, "total_steps": 72479, "tokens": 20609761280, "loss": 4.472906470298767, "l_mix": 2.7724631130695343, "l_a": 2.8587229251861572, "l_c": 2.8094215393066406, "z_loss": 33.237833976745605, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14350783824920654, "tokens_per_second": 1339905.3128676284, "eta_seconds": 12978.610133862496, "data_wait_seconds": 2.0176172256469727e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 39320, "total_steps": 72479, "tokens": 20615004160, "loss": 2.844404846429825, "l_mix": 1.759410783648491, "l_a": 1.8261148631572723, "l_c": 1.7905318588018417, "z_loss": 41.05156850814819, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16034075617790222, "tokens_per_second": 1473966.5707059111, "eta_seconds": 11794.613349795341, "data_wait_seconds": 3.835558891296387e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 39330, "total_steps": 72479, "tokens": 20620247040, "loss": 3.123805671930313, "l_mix": 1.934133619070053, "l_a": 2.0039033591747284, "l_c": 1.9616699516773224, "z_loss": 39.46668291091919, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17362385988235474, "tokens_per_second": 1440636.6718356085, "eta_seconds": 12063.848749494553, "data_wait_seconds": 3.018975257873535e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 39340, "total_steps": 72479, "tokens": 20625489920, "loss": 4.345790207386017, "l_mix": 2.6931498646736145, "l_a": 2.778425931930542, "l_c": 2.730375289916992, "z_loss": 32.79493260383606, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12432412058115005, "tokens_per_second": 1351335.5807946948, "eta_seconds": 12857.19126982689, "data_wait_seconds": 4.589557647705078e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 39350, "total_steps": 72479, "tokens": 20630732800, "loss": 4.532122313976288, "l_mix": 2.8087616562843323, "l_a": 2.898203730583191, "l_c": 2.8463315069675446, "z_loss": 33.277865409851074, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13299019634723663, "tokens_per_second": 1411417.3635438923, "eta_seconds": 12306.166553306579, "data_wait_seconds": 1.570582389831543e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 39360, "total_steps": 72479, "tokens": 20635975680, "loss": 4.505951404571533, "l_mix": 2.792496234178543, "l_a": 2.8811440765857697, "l_c": 2.8303728997707367, "z_loss": 33.561484813690186, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17263263463974, "tokens_per_second": 1389409.8365966429, "eta_seconds": 12497.316353058815, "data_wait_seconds": 1.990795135498047e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 39370, "total_steps": 72479, "tokens": 20641218560, "loss": 4.660278737545013, "l_mix": 2.8890574872493744, "l_a": 2.974228858947754, "l_c": 2.9298418164253235, "z_loss": 33.867494106292725, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14890897274017334, "tokens_per_second": 1425277.0935906505, "eta_seconds": 12179.141494703294, "data_wait_seconds": 1.5854835510253906e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 39380, "total_steps": 72479, "tokens": 20646461440, "loss": 4.54218864440918, "l_mix": 2.8138233721256256, "l_a": 2.9072238504886627, "l_c": 2.8539936244487762, "z_loss": 33.16640901565552, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19682522118091583, "tokens_per_second": 1467684.456619771, "eta_seconds": 11823.664435315131, "data_wait_seconds": 3.358721733093262e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20764, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 39390, "total_steps": 72479, "tokens": 20651704320, "loss": 4.3962584137916565, "l_mix": 2.7239211201667786, "l_a": 2.8154572248458862, "l_c": 2.759000301361084, "z_loss": 34.60387325286865, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1884663701057434, "tokens_per_second": 1416520.30124733, "eta_seconds": 12247.029298996926, "data_wait_seconds": 3.7103891372680664e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 39400, "total_steps": 72479, "tokens": 20656947200, "loss": 4.505859851837158, "l_mix": 2.7916087210178375, "l_a": 2.8798664808273315, "l_c": 2.8343037962913513, "z_loss": 33.52369499206543, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15617182850837708, "tokens_per_second": 1418427.8177857485, "eta_seconds": 12226.863104724885, "data_wait_seconds": 2.1755695343017578e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.21854989, "gate_a2b/L1_absmax": 0.72613841, "gate_a2b/L1_rms": 0.2315778, "gate_a2b/L3_absmean": 0.22042127, "gate_a2b/L3_absmax": 0.60681748, "gate_a2b/L3_rms": 0.22677806, "gate_a2b/L5_absmean": 0.3441599, "gate_a2b/L5_absmax": 0.70893317, "gate_a2b/L5_rms": 0.35003698, "gate_c2b/L1_absmean": 0.0176023, "gate_c2b/L1_absmax": 0.62113041, "gate_c2b/L1_rms": 0.06765854, "gate_c2b/L3_absmean": 0.20509827, "gate_c2b/L3_absmax": 0.46318334, "gate_c2b/L3_rms": 0.20934406, "gate_c2b/L5_absmean": 0.57785171, "gate_c2b/L5_absmax": 0.81083637, "gate_c2b/L5_rms": 0.5905562, "gate_b2a/L7_absmean": 0.05412075, "gate_b2a/L7_absmax": 0.3258509, "gate_b2a/L7_rms": 0.05967346, "gate_b2a/L11_absmean": 0.08614565, "gate_b2a/L11_absmax": 0.59326631, "gate_b2a/L11_rms": 0.09469848, "gate_b2a/L14_absmean": 0.22071311, "gate_b2a/L14_absmax": 0.77109993, "gate_b2a/L14_rms": 0.22865085, "gate_b2c/L3_absmean": 0.04117536, "gate_b2c/L3_absmax": 0.60659534, "gate_b2c/L3_rms": 0.05330379, "gate_b2c/L5_absmean": 0.04514718, "gate_b2c/L5_absmax": 0.63857132, "gate_b2c/L5_rms": 0.05645057, "gate_b2c/L6_absmean": 0.06451715, "gate_b2c/L6_absmax": 0.43181065, "gate_b2c/L6_rms": 0.07242839}, "streams": {"cos_a2b/round0": 0.373003, "cos_c2b/round0": 0.305678, "cos_b2a/round0": 0.143213, "cos_b2c/round0": 0.125634, "rms_a/round0": 4.30184, "rms_b/round0": 8.26573, "rms_c/round0": 3.359074, "cos_a2b/round1": 0.422202, "cos_c2b/round1": 0.299688, "cos_b2a/round1": 0.244936, "cos_b2c/round1": 0.117473, "rms_a/round1": 8.206174, "rms_b/round1": 18.371735, "rms_c/round1": 6.699813, "cos_a2b/round2": 0.558192, "cos_c2b/round2": 0.522854, "cos_b2a/round2": 0.101287, "cos_b2c/round2": 0.174304, "rms_a/round2": 21.611479, "rms_b/round2": 46.901402, "rms_c/round2": 12.364997}, "alpha_mean": 0.442541587463424, "alpha_hist": [0.00764, 0.06624, 0.172, 0.19409, 0.17809, 0.16779, 0.12968, 0.07141, 0.01302, 5e-05], "alpha_by_type": {"special": {"share": 0.00107, "mean": 0.54438, "std": 0.10212}, "whitespace": {"share": 0.02673, "mean": 0.52513, "std": 0.11669}, "digit": {"share": 0.02454, "mean": 0.42344, "std": 0.1072}, "word_start": {"share": 0.56714, "mean": 0.46801, "std": 0.16852}, "word_cont": {"share": 0.26816, "mean": 0.32609, "std": 0.12761}, "punct": {"share": 0.11237, "mean": 0.57548, "std": 0.16855}}, "alpha_p05": 0.19163, "alpha_p50": 0.43182, "alpha_p95": 0.72467}
{"step": 39410, "total_steps": 72479, "tokens": 20662190080, "loss": 4.392786860466003, "l_mix": 2.7220334708690643, "l_a": 2.8097490668296814, "l_c": 2.7594285011291504, "z_loss": 33.01067352294922, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15002407133579254, "tokens_per_second": 1457913.1030341177, "eta_seconds": 11892.121578383447, "data_wait_seconds": 1.615285873413086e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 39420, "total_steps": 72479, "tokens": 20667432960, "loss": 4.50080281496048, "l_mix": 2.788778305053711, "l_a": 2.8792874217033386, "l_c": 2.8274609446525574, "z_loss": 34.33172559738159, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1913740485906601, "tokens_per_second": 1292341.6698687878, "eta_seconds": 13411.652193927766, "data_wait_seconds": 2.1964311599731445e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 39430, "total_steps": 72479, "tokens": 20672675840, "loss": 3.195507675409317, "l_mix": 1.9663788676261902, "l_a": 2.080107241868973, "l_c": 2.0169885754585266, "z_loss": 34.68865394592285, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14859898388385773, "tokens_per_second": 1312907.1591921009, "eta_seconds": 13197.577597689628, "data_wait_seconds": 2.4110078811645508e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 39440, "total_steps": 72479, "tokens": 20677918720, "loss": 3.1257465481758118, "l_mix": 1.9319354742765427, "l_a": 2.026473790407181, "l_c": 1.9528964012861252, "z_loss": 38.51674461364746, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.3464598059654236, "tokens_per_second": 1447945.148251159, "eta_seconds": 11963.126678466797, "data_wait_seconds": 1.728534698486328e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 39450, "total_steps": 72479, "tokens": 20683161600, "loss": 4.46051812171936, "l_mix": 2.7640455067157745, "l_a": 2.852798879146576, "l_c": 2.8021095991134644, "z_loss": 33.57865333557129, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.142454132437706, "tokens_per_second": 1477051.143615744, "eta_seconds": 11723.838017964363, "data_wait_seconds": 1.5169382095336914e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 39460, "total_steps": 72479, "tokens": 20688404480, "loss": 4.397413849830627, "l_mix": 2.724568098783493, "l_a": 2.814999520778656, "l_c": 2.7611527740955353, "z_loss": 33.99376726150513, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18368130922317505, "tokens_per_second": 1432844.9520161655, "eta_seconds": 12081.883282375336, "data_wait_seconds": 1.6868114471435547e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 39470, "total_steps": 72479, "tokens": 20693647360, "loss": 4.405519723892212, "l_mix": 2.730699360370636, "l_a": 2.8132737278938293, "l_c": 2.7694603502750397, "z_loss": 32.641066551208496, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17413650453090668, "tokens_per_second": 1325678.3136344482, "eta_seconds": 13054.616956472397, "data_wait_seconds": 1.8268823623657227e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 39480, "total_steps": 72479, "tokens": 20698890240, "loss": 4.410062491893768, "l_mix": 2.731837123632431, "l_a": 2.8245757818222046, "l_c": 2.769508421421051, "z_loss": 34.16468954086304, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13037630915641785, "tokens_per_second": 1422043.7310696067, "eta_seconds": 12166.27824728489, "data_wait_seconds": 4.45246696472168e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 39490, "total_steps": 72479, "tokens": 20704133120, "loss": 4.506801605224609, "l_mix": 2.79311802983284, "l_a": 2.8812011182308197, "l_c": 2.8310771584510803, "z_loss": 33.09189701080322, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1857900321483612, "tokens_per_second": 1332273.6178455648, "eta_seconds": 12982.1206397295, "data_wait_seconds": 2.0265579223632812e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 39500, "total_steps": 72479, "tokens": 20709376000, "loss": 4.732334673404694, "l_mix": 2.9356404542922974, "l_a": 3.0160567462444305, "l_c": 2.9729237854480743, "z_loss": 33.00657558441162, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15581701695919037, "tokens_per_second": 1425057.7294903537, "eta_seconds": 12133.188427519799, "data_wait_seconds": 1.7881393432617188e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.21814762, "gate_a2b/L1_absmax": 0.72737706, "gate_a2b/L1_rms": 0.23130767, "gate_a2b/L3_absmean": 0.21934398, "gate_a2b/L3_absmax": 0.60926038, "gate_a2b/L3_rms": 0.22564746, "gate_a2b/L5_absmean": 0.34268448, "gate_a2b/L5_absmax": 0.70921397, "gate_a2b/L5_rms": 0.34848085, "gate_c2b/L1_absmean": 0.01743276, "gate_c2b/L1_absmax": 0.62176591, "gate_c2b/L1_rms": 0.06784075, "gate_c2b/L3_absmean": 0.20498656, "gate_c2b/L3_absmax": 0.45923823, "gate_c2b/L3_rms": 0.20929745, "gate_c2b/L5_absmean": 0.57810324, "gate_c2b/L5_absmax": 0.81480205, "gate_c2b/L5_rms": 0.59077275, "gate_b2a/L7_absmean": 0.05371144, "gate_b2a/L7_absmax": 0.32666385, "gate_b2a/L7_rms": 0.05923678, "gate_b2a/L11_absmean": 0.08623566, "gate_b2a/L11_absmax": 0.59238726, "gate_b2a/L11_rms": 0.09467561, "gate_b2a/L14_absmean": 0.22120008, "gate_b2a/L14_absmax": 0.77211708, "gate_b2a/L14_rms": 0.22909054, "gate_b2c/L3_absmean": 0.04085011, "gate_b2c/L3_absmax": 0.60622245, "gate_b2c/L3_rms": 0.05289852, "gate_b2c/L5_absmean": 0.04485977, "gate_b2c/L5_absmax": 0.63681757, "gate_b2c/L5_rms": 0.05606718, "gate_b2c/L6_absmean": 0.06466495, "gate_b2c/L6_absmax": 0.43069163, "gate_b2c/L6_rms": 0.07244866}, "streams": {"cos_a2b/round0": 0.378967, "cos_c2b/round0": 0.314784, "cos_b2a/round0": 0.146978, "cos_b2c/round0": 0.131605, "rms_a/round0": 4.303809, "rms_b/round0": 8.148878, "rms_c/round0": 3.347064, "cos_a2b/round1": 0.422383, "cos_c2b/round1": 0.303399, "cos_b2a/round1": 0.251659, "cos_b2c/round1": 0.118376, "rms_a/round1": 8.028552, "rms_b/round1": 18.067169, "rms_c/round1": 6.696192, "cos_a2b/round2": 0.562421, "cos_c2b/round2": 0.527005, "cos_b2a/round2": 0.111677, "cos_b2c/round2": 0.177958, "rms_a/round2": 21.314062, "rms_b/round2": 46.405193, "rms_c/round2": 12.312018}, "alpha_mean": 0.45755054302006215, "alpha_hist": [0.004, 0.04187, 0.14752, 0.19354, 0.20924, 0.18922, 0.1356, 0.06981, 0.00911, 8e-05], "alpha_by_type": {"special": {"share": 0.00044, "mean": 0.52482, "std": 0.19015}, "whitespace": {"share": 0.06239, "mean": 0.52914, "std": 0.10682}, "digit": {"share": 0.04129, "mean": 0.45182, "std": 0.09474}, "word_start": {"share": 0.55919, "mean": 0.47717, "std": 0.15881}, "word_cont": {"share": 0.24614, "mean": 0.34637, "std": 0.1236}, "punct": {"share": 0.09055, "mean": 0.59158, "std": 0.15663}}, "alpha_p05": 0.20947, "alpha_p50": 0.44662, "alpha_p95": 0.7272}
{"step": 39510, "total_steps": 72479, "tokens": 20714618880, "loss": 4.898915469646454, "l_mix": 3.038771390914917, "l_a": 3.115980088710785, "l_c": 3.0844999253749847, "z_loss": 33.562355518341064, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18710556626319885, "tokens_per_second": 1467375.5652001984, "eta_seconds": 11779.704856705666, "data_wait_seconds": 2.8699636459350586e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 39520, "total_steps": 72479, "tokens": 20719861760, "loss": 4.790869116783142, "l_mix": 2.9721234142780304, "l_a": 3.050706595182419, "l_c": 3.0117785930633545, "z_loss": 32.66152548789978, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12916263937950134, "tokens_per_second": 1483384.7524446584, "eta_seconds": 11649.039915990828, "data_wait_seconds": 1.5556812286376953e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 39530, "total_steps": 72479, "tokens": 20725104640, "loss": 4.7625585198402405, "l_mix": 2.9548139572143555, "l_a": 3.031136155128479, "l_c": 2.9946794509887695, "z_loss": 32.24699020385742, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14375723898410797, "tokens_per_second": 1356408.893650217, "eta_seconds": 12735.66208012104, "data_wait_seconds": 1.6510486602783203e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 39540, "total_steps": 72479, "tokens": 20730347520, "loss": 4.766268849372864, "l_mix": 2.956582635641098, "l_a": 3.035316050052643, "l_c": 2.9969708621501923, "z_loss": 33.43792724609375, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16607429087162018, "tokens_per_second": 1374088.9527615835, "eta_seconds": 12567.979967594147, "data_wait_seconds": 2.1755695343017578e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 39550, "total_steps": 72479, "tokens": 20735590400, "loss": 2.9954817295074463, "l_mix": 1.8530605286359787, "l_a": 1.9236674159765244, "l_c": 1.8844029903411865, "z_loss": 40.241344928741455, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20884528756141663, "tokens_per_second": 1461606.907506452, "eta_seconds": 11811.848632717132, "data_wait_seconds": 2.2470951080322266e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 39560, "total_steps": 72479, "tokens": 20740833280, "loss": 4.364309847354889, "l_mix": 2.7048390805721283, "l_a": 2.7891527712345123, "l_c": 2.742416173219681, "z_loss": 33.98850202560425, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11211210489273071, "tokens_per_second": 1477130.2192739341, "eta_seconds": 11684.167344760894, "data_wait_seconds": 1.9103288650512695e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 39570, "total_steps": 72479, "tokens": 20746076160, "loss": 4.487847089767456, "l_mix": 2.7808068692684174, "l_a": 2.869985908269882, "l_c": 2.8201480209827423, "z_loss": 34.518866539001465, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13848143815994263, "tokens_per_second": 1269211.5593140905, "eta_seconds": 13594.103887081146, "data_wait_seconds": 1.8388032913208008e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 39580, "total_steps": 72479, "tokens": 20751319040, "loss": 4.410919487476349, "l_mix": 2.7338419556617737, "l_a": 2.8170546293258667, "l_c": 2.7732038497924805, "z_loss": 33.23825788497925, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1974603831768036, "tokens_per_second": 1449728.863059361, "eta_seconds": 11897.777130270004, "data_wait_seconds": 1.7881393432617188e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 39590, "total_steps": 72479, "tokens": 20756561920, "loss": 4.413139283657074, "l_mix": 2.734216809272766, "l_a": 2.825717478990555, "l_c": 2.7706905007362366, "z_loss": 34.72585153579712, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2117023766040802, "tokens_per_second": 1486615.9371736655, "eta_seconds": 11599.033483242987, "data_wait_seconds": 1.58846378326416e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 39600, "total_steps": 72479, "tokens": 20761804800, "loss": 4.486093699932098, "l_mix": 2.779826134443283, "l_a": 2.8705004155635834, "l_c": 2.8170582056045532, "z_loss": 34.35518026351929, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20444336533546448, "tokens_per_second": 1272068.2596248242, "eta_seconds": 13551.210810875893, "data_wait_seconds": 1.862645149230957e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.21740623, "gate_a2b/L1_absmax": 0.72662646, "gate_a2b/L1_rms": 0.23048878, "gate_a2b/L3_absmean": 0.21842848, "gate_a2b/L3_absmax": 0.60690659, "gate_a2b/L3_rms": 0.2247602, "gate_a2b/L5_absmean": 0.34135628, "gate_a2b/L5_absmax": 0.70887256, "gate_a2b/L5_rms": 0.34727618, "gate_c2b/L1_absmean": 0.01755993, "gate_c2b/L1_absmax": 0.62294662, "gate_c2b/L1_rms": 0.06766617, "gate_c2b/L3_absmean": 0.20522343, "gate_c2b/L3_absmax": 0.46289149, "gate_c2b/L3_rms": 0.20942995, "gate_c2b/L5_absmean": 0.57737094, "gate_c2b/L5_absmax": 0.81093413, "gate_c2b/L5_rms": 0.59004587, "gate_b2a/L7_absmean": 0.05373016, "gate_b2a/L7_absmax": 0.32462487, "gate_b2a/L7_rms": 0.059296, "gate_b2a/L11_absmean": 0.08602348, "gate_b2a/L11_absmax": 0.59062034, "gate_b2a/L11_rms": 0.09443296, "gate_b2a/L14_absmean": 0.22121009, "gate_b2a/L14_absmax": 0.76975006, "gate_b2a/L14_rms": 0.22904775, "gate_b2c/L3_absmean": 0.04089587, "gate_b2c/L3_absmax": 0.60447383, "gate_b2c/L3_rms": 0.052854, "gate_b2c/L5_absmean": 0.04490317, "gate_b2c/L5_absmax": 0.63487285, "gate_b2c/L5_rms": 0.05606119, "gate_b2c/L6_absmean": 0.06478506, "gate_b2c/L6_absmax": 0.42858109, "gate_b2c/L6_rms": 0.07253771}, "streams": {"cos_a2b/round0": 0.377551, "cos_c2b/round0": 0.313032, "cos_b2a/round0": 0.142421, "cos_b2c/round0": 0.129203, "rms_a/round0": 4.341657, "rms_b/round0": 8.258412, "rms_c/round0": 3.372698, "cos_a2b/round1": 0.419376, "cos_c2b/round1": 0.296443, "cos_b2a/round1": 0.244377, "cos_b2c/round1": 0.113525, "rms_a/round1": 8.260792, "rms_b/round1": 18.464783, "rms_c/round1": 6.728773, "cos_a2b/round2": 0.556585, "cos_c2b/round2": 0.517997, "cos_b2a/round2": 0.101198, "cos_b2c/round2": 0.173497, "rms_a/round2": 21.848438, "rms_b/round2": 47.688316, "rms_c/round2": 12.410982}, "alpha_mean": 0.4425531393947466, "alpha_hist": [0.0067, 0.06079, 0.17599, 0.19618, 0.18332, 0.16431, 0.1317, 0.07146, 0.00929, 0.00026], "alpha_by_type": {"special": {"share": 0.00092, "mean": 0.5747, "std": 0.1121}, "whitespace": {"share": 0.02727, "mean": 0.53886, "std": 0.13149}, "digit": {"share": 0.04916, "mean": 0.47124, "std": 0.1197}, "word_start": {"share": 0.56108, "mean": 0.45652, "std": 0.16549}, "word_cont": {"share": 0.24619, "mean": 0.32616, "std": 0.12602}, "punct": {"share": 0.11539, "mean": 0.58694, "std": 0.15884}}, "alpha_p05": 0.18126, "alpha_p50": 0.43891, "alpha_p95": 0.72681}
{"step": 39610, "total_steps": 72479, "tokens": 20767047680, "loss": 3.069084197282791, "l_mix": 1.899526685476303, "l_a": 1.969839096069336, "l_c": 1.92868572473526, "z_loss": 40.16147708892822, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1933734118938446, "tokens_per_second": 1389008.5896873488, "eta_seconds": 12406.562781500816, "data_wait_seconds": 1.9222497940063477e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 39620, "total_steps": 72479, "tokens": 20772290560, "loss": 4.865544378757477, "l_mix": 3.019478976726532, "l_a": 3.0975412130355835, "l_c": 3.0560100078582764, "z_loss": 32.36815881729126, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16516050696372986, "tokens_per_second": 1436710.7235124207, "eta_seconds": 11990.98684937954, "data_wait_seconds": 2.5540590286254883e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 39630, "total_steps": 72479, "tokens": 20777533440, "loss": 4.820887923240662, "l_mix": 2.9907907843589783, "l_a": 3.0694902539253235, "l_c": 3.0308333039283752, "z_loss": 32.806504249572754, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13499903678894043, "tokens_per_second": 1298469.408054783, "eta_seconds": 13263.567401099204, "data_wait_seconds": 2.1159648895263672e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 39640, "total_steps": 72479, "tokens": 20782776320, "loss": 4.805830240249634, "l_mix": 2.982536107301712, "l_a": 3.0570534765720367, "l_c": 3.0205934047698975, "z_loss": 31.980079174041748, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1999441534280777, "tokens_per_second": 1453236.4207938684, "eta_seconds": 11847.414079117776, "data_wait_seconds": 2.1278858184814453e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 39650, "total_steps": 72479, "tokens": 20788019200, "loss": 4.744488954544067, "l_mix": 2.943515807390213, "l_a": 3.0188752710819244, "l_c": 2.984368234872818, "z_loss": 34.2678427696228, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16876865923404694, "tokens_per_second": 1458441.1382644614, "eta_seconds": 11801.539534521104, "data_wait_seconds": 2.7567148208618164e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 39660, "total_steps": 72479, "tokens": 20793262080, "loss": 2.9905992448329926, "l_mix": 1.8512910604476929, "l_a": 1.9174112230539322, "l_c": 1.8802824765443802, "z_loss": 39.67613458633423, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18258583545684814, "tokens_per_second": 1463581.177914148, "eta_seconds": 11756.510763907432, "data_wait_seconds": 2.002716064453125e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 39670, "total_steps": 72479, "tokens": 20798504960, "loss": 4.491117298603058, "l_mix": 2.783719837665558, "l_a": 2.8692264556884766, "l_c": 2.8220982253551483, "z_loss": 33.21138668060303, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13240067660808563, "tokens_per_second": 1476168.8885505619, "eta_seconds": 11652.707983088492, "data_wait_seconds": 2.142786979675293e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 39680, "total_steps": 72479, "tokens": 20803747840, "loss": 4.469752252101898, "l_mix": 2.7699115872383118, "l_a": 2.8579463064670563, "l_c": 2.808189034461975, "z_loss": 33.822691917419434, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1317325234413147, "tokens_per_second": 1454739.7191555111, "eta_seconds": 11820.755208349226, "data_wait_seconds": 2.4020671844482422e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 39690, "total_steps": 72479, "tokens": 20808990720, "loss": 4.612304210662842, "l_mix": 2.8592034578323364, "l_a": 2.9465901255607605, "l_c": 2.897078812122345, "z_loss": 33.34574770927429, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.10506612062454224, "tokens_per_second": 1409046.8021268116, "eta_seconds": 12200.360702037811, "data_wait_seconds": 2.2113323211669922e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 39700, "total_steps": 72479, "tokens": 20814233600, "loss": 4.473814249038696, "l_mix": 2.7736507058143616, "l_a": 2.8596503138542175, "l_c": 2.807560831308365, "z_loss": 32.9655225276947, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16508479416370392, "tokens_per_second": 1379390.5150322495, "eta_seconds": 12458.862203788756, "data_wait_seconds": 1.582503318786621e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.21732795, "gate_a2b/L1_absmax": 0.72535461, "gate_a2b/L1_rms": 0.23027767, "gate_a2b/L3_absmean": 0.2180907, "gate_a2b/L3_absmax": 0.60679471, "gate_a2b/L3_rms": 0.22445707, "gate_a2b/L5_absmean": 0.34021941, "gate_a2b/L5_absmax": 0.71093839, "gate_a2b/L5_rms": 0.3461042, "gate_c2b/L1_absmean": 0.01764082, "gate_c2b/L1_absmax": 0.62185746, "gate_c2b/L1_rms": 0.06756591, "gate_c2b/L3_absmean": 0.20434146, "gate_c2b/L3_absmax": 0.46312919, "gate_c2b/L3_rms": 0.20857029, "gate_c2b/L5_absmean": 0.57693303, "gate_c2b/L5_absmax": 0.81163746, "gate_c2b/L5_rms": 0.58958203, "gate_b2a/L7_absmean": 0.05354986, "gate_b2a/L7_absmax": 0.32650977, "gate_b2a/L7_rms": 0.05910125, "gate_b2a/L11_absmean": 0.08584809, "gate_b2a/L11_absmax": 0.5914579, "gate_b2a/L11_rms": 0.09430967, "gate_b2a/L14_absmean": 0.22076924, "gate_b2a/L14_absmax": 0.77115756, "gate_b2a/L14_rms": 0.22865391, "gate_b2c/L3_absmean": 0.04084719, "gate_b2c/L3_absmax": 0.60438144, "gate_b2c/L3_rms": 0.05283041, "gate_b2c/L5_absmean": 0.04471609, "gate_b2c/L5_absmax": 0.63306767, "gate_b2c/L5_rms": 0.05585118, "gate_b2c/L6_absmean": 0.06446408, "gate_b2c/L6_absmax": 0.42850739, "gate_b2c/L6_rms": 0.07220038}, "streams": {"cos_a2b/round0": 0.379389, "cos_c2b/round0": 0.31381, "cos_b2a/round0": 0.142201, "cos_b2c/round0": 0.129187, "rms_a/round0": 4.397331, "rms_b/round0": 8.263976, "rms_c/round0": 3.374135, "cos_a2b/round1": 0.422144, "cos_c2b/round1": 0.292373, "cos_b2a/round1": 0.248119, "cos_b2c/round1": 0.113627, "rms_a/round1": 8.263826, "rms_b/round1": 18.524853, "rms_c/round1": 6.733704, "cos_a2b/round2": 0.554563, "cos_c2b/round2": 0.520146, "cos_b2a/round2": 0.100544, "cos_b2c/round2": 0.174466, "rms_a/round2": 21.828259, "rms_b/round2": 47.844624, "rms_c/round2": 12.408124}, "alpha_mean": 0.44588171174720515, "alpha_hist": [0.00873, 0.06181, 0.16463, 0.19809, 0.18217, 0.16635, 0.12808, 0.07692, 0.01292, 0.00029], "alpha_by_type": {"special": {"share": 0.00084, "mean": 0.5599, "std": 0.11527}, "whitespace": {"share": 0.02797, "mean": 0.54918, "std": 0.11684}, "digit": {"share": 0.05101, "mean": 0.48327, "std": 0.11924}, "word_start": {"share": 0.55443, "mean": 0.45884, "std": 0.1689}, "word_cont": {"share": 0.25217, "mean": 0.32804, "std": 0.12678}, "punct": {"share": 0.11359, "mean": 0.60117, "std": 0.15708}}, "alpha_p05": 0.1686, "alpha_p50": 0.42155, "alpha_p95": 0.74205}
{"step": 39710, "total_steps": 72479, "tokens": 20819476480, "loss": 4.404197871685028, "l_mix": 2.729076385498047, "l_a": 2.8144231736660004, "l_c": 2.7693148255348206, "z_loss": 33.838359355926514, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1654120683670044, "tokens_per_second": 1457949.1569987119, "eta_seconds": 11783.94554400444, "data_wait_seconds": 2.136826515197754e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 39720, "total_steps": 72479, "tokens": 20824719360, "loss": 2.9840374141931534, "l_mix": 1.8473056852817535, "l_a": 1.9135198295116425, "l_c": 1.8755860030651093, "z_loss": 41.296557903289795, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2359747588634491, "tokens_per_second": 1430674.9705781129, "eta_seconds": 12004.928404569626, "data_wait_seconds": 1.749396324157715e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 39730, "total_steps": 72479, "tokens": 20829962240, "loss": 3.0091311931610107, "l_mix": 1.862429916858673, "l_a": 1.9298444837331772, "l_c": 1.8924928605556488, "z_loss": 39.150877952575684, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.22738604247570038, "tokens_per_second": 1393654.544103598, "eta_seconds": 12320.06007847786, "data_wait_seconds": 2.339482307434082e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08236}}
{"step": 39740, "total_steps": 72479, "tokens": 20835205120, "loss": 4.694465458393097, "l_mix": 2.9126459658145905, "l_a": 2.9878391325473785, "l_c": 2.951558530330658, "z_loss": 32.09475255012512, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16766631603240967, "tokens_per_second": 1381710.6147692683, "eta_seconds": 12422.763962674142, "data_wait_seconds": 2.473592758178711e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 39750, "total_steps": 72479, "tokens": 20840448000, "loss": 4.835112512111664, "l_mix": 3.000076323747635, "l_a": 3.0792896151542664, "l_c": 3.037497788667679, "z_loss": 33.49898433685303, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15404298901557922, "tokens_per_second": 1469080.00459091, "eta_seconds": 11680.386295080185, "data_wait_seconds": 1.6868114471435547e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 39760, "total_steps": 72479, "tokens": 20845690880, "loss": 4.820817530155182, "l_mix": 2.990937501192093, "l_a": 3.0667837858200073, "l_c": 3.032815992832184, "z_loss": 33.21917676925659, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17014694213867188, "tokens_per_second": 1376262.4558896588, "eta_seconds": 12464.32248339653, "data_wait_seconds": 1.621246337890625e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 39770, "total_steps": 72479, "tokens": 20850933760, "loss": 4.7727484703063965, "l_mix": 2.961087614297867, "l_a": 3.039283871650696, "l_c": 2.999585509300232, "z_loss": 33.201499462127686, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14001551270484924, "tokens_per_second": 1405752.937060523, "eta_seconds": 12199.111052799224, "data_wait_seconds": 2.041459083557129e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 39780, "total_steps": 72479, "tokens": 20856176640, "loss": 4.436765968799591, "l_mix": 2.74844890832901, "l_a": 2.839883953332901, "l_c": 2.787839323282242, "z_loss": 34.34249258041382, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13920378684997559, "tokens_per_second": 1405321.3605258544, "eta_seconds": 12199.126686286925, "data_wait_seconds": 1.7702579498291016e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 39790, "total_steps": 72479, "tokens": 20861419520, "loss": 4.52267462015152, "l_mix": 2.8028738498687744, "l_a": 2.8919757902622223, "l_c": 2.8406931161880493, "z_loss": 33.59607744216919, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12961985170841217, "tokens_per_second": 1380190.821791696, "eta_seconds": 12417.44993619919, "data_wait_seconds": 1.9162893295288086e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 39800, "total_steps": 72479, "tokens": 20866662400, "loss": 4.456984996795654, "l_mix": 2.7617921233177185, "l_a": 2.8497779071331024, "l_c": 2.800864666700363, "z_loss": 33.57454013824463, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13828931748867035, "tokens_per_second": 1204715.1887022029, "eta_seconds": 14221.79093670845, "data_wait_seconds": 1.6003847122192383e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.21641694, "gate_a2b/L1_absmax": 0.72517568, "gate_a2b/L1_rms": 0.22941574, "gate_a2b/L3_absmean": 0.21767564, "gate_a2b/L3_absmax": 0.60439765, "gate_a2b/L3_rms": 0.22394973, "gate_a2b/L5_absmean": 0.33921054, "gate_a2b/L5_absmax": 0.71039909, "gate_a2b/L5_rms": 0.34510306, "gate_c2b/L1_absmean": 0.01785527, "gate_c2b/L1_absmax": 0.62212837, "gate_c2b/L1_rms": 0.06748313, "gate_c2b/L3_absmean": 0.2043827, "gate_c2b/L3_absmax": 0.46086726, "gate_c2b/L3_rms": 0.20861562, "gate_c2b/L5_absmean": 0.57618374, "gate_c2b/L5_absmax": 0.80729932, "gate_c2b/L5_rms": 0.5887357, "gate_b2a/L7_absmean": 0.05353559, "gate_b2a/L7_absmax": 0.32558584, "gate_b2a/L7_rms": 0.0590593, "gate_b2a/L11_absmean": 0.08605048, "gate_b2a/L11_absmax": 0.59149027, "gate_b2a/L11_rms": 0.09443361, "gate_b2a/L14_absmean": 0.22052562, "gate_b2a/L14_absmax": 0.76924133, "gate_b2a/L14_rms": 0.22844331, "gate_b2c/L3_absmean": 0.04059443, "gate_b2c/L3_absmax": 0.60379273, "gate_b2c/L3_rms": 0.05262354, "gate_b2c/L5_absmean": 0.04446743, "gate_b2c/L5_absmax": 0.63509327, "gate_b2c/L5_rms": 0.05565787, "gate_b2c/L6_absmean": 0.06431836, "gate_b2c/L6_absmax": 0.42390934, "gate_b2c/L6_rms": 0.07193364}, "streams": {"cos_a2b/round0": 0.373691, "cos_c2b/round0": 0.303507, "cos_b2a/round0": 0.140639, "cos_b2c/round0": 0.124691, "rms_a/round0": 4.383931, "rms_b/round0": 8.346703, "rms_c/round0": 3.407604, "cos_a2b/round1": 0.420541, "cos_c2b/round1": 0.295093, "cos_b2a/round1": 0.244859, "cos_b2c/round1": 0.117463, "rms_a/round1": 8.366092, "rms_b/round1": 18.79883, "rms_c/round1": 6.795345, "cos_a2b/round2": 0.545524, "cos_c2b/round2": 0.524362, "cos_b2a/round2": 0.100823, "cos_b2c/round2": 0.174746, "rms_a/round2": 21.880795, "rms_b/round2": 48.167969, "rms_c/round2": 12.641601}, "alpha_mean": 0.42673159326920995, "alpha_hist": [0.00954, 0.06982, 0.17601, 0.21252, 0.19824, 0.15819, 0.11424, 0.05457, 0.00682, 5e-05], "alpha_by_type": {"special": {"share": 0.00154, "mean": 0.5326, "std": 0.12083}, "whitespace": {"share": 0.02644, "mean": 0.52695, "std": 0.12713}, "digit": {"share": 0.03345, "mean": 0.45579, "std": 0.12737}, "word_start": {"share": 0.56493, "mean": 0.44468, "std": 0.16103}, "word_cont": {"share": 0.27313, "mean": 0.32375, "std": 0.12401}, "punct": {"share": 0.10051, "mean": 0.56803, "std": 0.15692}}, "alpha_p05": 0.17273, "alpha_p50": 0.41846, "alpha_p95": 0.707}
{"step": 39810, "total_steps": 72479, "tokens": 20871905280, "loss": 4.4588324427604675, "l_mix": 2.763250380754471, "l_a": 2.8508503437042236, "l_c": 2.8010899424552917, "z_loss": 33.79424238204956, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11635647714138031, "tokens_per_second": 1366437.0419405364, "eta_seconds": 12534.763144063949, "data_wait_seconds": 1.6927719116210938e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 39820, "total_steps": 72479, "tokens": 20877148160, "loss": 4.495640277862549, "l_mix": 2.7854220867156982, "l_a": 2.877381205558777, "l_c": 2.8233457505702972, "z_loss": 34.05686902999878, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17635929584503174, "tokens_per_second": 1397820.5008657624, "eta_seconds": 12249.585537910461, "data_wait_seconds": 1.621246337890625e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 39830, "total_steps": 72479, "tokens": 20882391040, "loss": 3.200254499912262, "l_mix": 1.9830427765846252, "l_a": 2.047073259949684, "l_c": 2.010299116373062, "z_loss": 39.42559623718262, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15443885326385498, "tokens_per_second": 1455947.625868453, "eta_seconds": 11756.933153271675, "data_wait_seconds": 2.8342008590698242e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 39840, "total_steps": 72479, "tokens": 20887633920, "loss": 2.9976636171340942, "l_mix": 1.855607584118843, "l_a": 1.9225263744592667, "l_c": 1.8843267112970352, "z_loss": 41.77889537811279, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19645124673843384, "tokens_per_second": 1450664.475510516, "eta_seconds": 11796.136405682564, "data_wait_seconds": 2.1249055862426758e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 39850, "total_steps": 72479, "tokens": 20892876800, "loss": 4.554773271083832, "l_mix": 2.8231808245182037, "l_a": 2.912221670150757, "l_c": 2.8597530126571655, "z_loss": 34.246291160583496, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.161071315407753, "tokens_per_second": 1462359.410194592, "eta_seconds": 11698.213881444932, "data_wait_seconds": 1.7136335372924805e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 39860, "total_steps": 72479, "tokens": 20898119680, "loss": 4.389232635498047, "l_mix": 2.718989461660385, "l_a": 2.8067036867141724, "l_c": 2.760773152112961, "z_loss": 34.30760478973389, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16752299666404724, "tokens_per_second": 1437687.9708448593, "eta_seconds": 11895.314295458795, "data_wait_seconds": 1.990795135498047e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 39870, "total_steps": 72479, "tokens": 20903362560, "loss": 4.377872705459595, "l_mix": 2.7134237587451935, "l_a": 2.8007395267486572, "l_c": 2.7474232614040375, "z_loss": 33.65800189971924, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2192700356245041, "tokens_per_second": 1483570.7613617224, "eta_seconds": 11523.890762249628, "data_wait_seconds": 1.6450881958007812e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 39880, "total_steps": 72479, "tokens": 20908605440, "loss": 4.391231149435043, "l_mix": 2.7204785645008087, "l_a": 2.8106476962566376, "l_c": 2.7585271894931793, "z_loss": 34.18738031387329, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12008509039878845, "tokens_per_second": 1363964.1599114765, "eta_seconds": 12530.58182489872, "data_wait_seconds": 2.536177635192871e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 39890, "total_steps": 72479, "tokens": 20913848320, "loss": 4.455601513385773, "l_mix": 2.761217415332794, "l_a": 2.8494276106357574, "l_c": 2.7985191643238068, "z_loss": 33.10068130493164, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1339874416589737, "tokens_per_second": 1351260.2661063573, "eta_seconds": 12644.508286499977, "data_wait_seconds": 2.1725893020629883e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 39900, "total_steps": 72479, "tokens": 20919091200, "loss": 4.4123300313949585, "l_mix": 2.7336035072803497, "l_a": 2.823899507522583, "l_c": 2.771855443716049, "z_loss": 33.37929940223694, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12471997737884521, "tokens_per_second": 1391922.3627920237, "eta_seconds": 12271.358811807633, "data_wait_seconds": 1.5348196029663086e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.21617889, "gate_a2b/L1_absmax": 0.72613257, "gate_a2b/L1_rms": 0.22913329, "gate_a2b/L3_absmean": 0.21727677, "gate_a2b/L3_absmax": 0.60597372, "gate_a2b/L3_rms": 0.22352634, "gate_a2b/L5_absmean": 0.33859298, "gate_a2b/L5_absmax": 0.71228063, "gate_a2b/L5_rms": 0.34446654, "gate_c2b/L1_absmean": 0.01721138, "gate_c2b/L1_absmax": 0.62294471, "gate_c2b/L1_rms": 0.06732146, "gate_c2b/L3_absmean": 0.20353737, "gate_c2b/L3_absmax": 0.45886493, "gate_c2b/L3_rms": 0.20778212, "gate_c2b/L5_absmean": 0.57540411, "gate_c2b/L5_absmax": 0.80588657, "gate_c2b/L5_rms": 0.58795166, "gate_b2a/L7_absmean": 0.05337685, "gate_b2a/L7_absmax": 0.32514843, "gate_b2a/L7_rms": 0.05890944, "gate_b2a/L11_absmean": 0.08574023, "gate_b2a/L11_absmax": 0.59292614, "gate_b2a/L11_rms": 0.09420796, "gate_b2a/L14_absmean": 0.22039029, "gate_b2a/L14_absmax": 0.77066749, "gate_b2a/L14_rms": 0.22832711, "gate_b2c/L3_absmean": 0.0404939, "gate_b2c/L3_absmax": 0.60365933, "gate_b2c/L3_rms": 0.0525021, "gate_b2c/L5_absmean": 0.0447292, "gate_b2c/L5_absmax": 0.63351357, "gate_b2c/L5_rms": 0.05581915, "gate_b2c/L6_absmean": 0.06430335, "gate_b2c/L6_absmax": 0.42403501, "gate_b2c/L6_rms": 0.07192712}, "streams": {"cos_a2b/round0": 0.375387, "cos_c2b/round0": 0.309532, "cos_b2a/round0": 0.14431, "cos_b2c/round0": 0.127548, "rms_a/round0": 4.434222, "rms_b/round0": 8.356483, "rms_c/round0": 3.419444, "cos_a2b/round1": 0.421417, "cos_c2b/round1": 0.297927, "cos_b2a/round1": 0.246663, "cos_b2c/round1": 0.117419, "rms_a/round1": 8.409222, "rms_b/round1": 18.896015, "rms_c/round1": 6.851044, "cos_a2b/round2": 0.545846, "cos_c2b/round2": 0.526106, "cos_b2a/round2": 0.099709, "cos_b2c/round2": 0.174557, "rms_a/round2": 22.174728, "rms_b/round2": 48.723858, "rms_c/round2": 12.79115}, "alpha_mean": 0.4327601644696273, "alpha_hist": [0.01244, 0.08537, 0.17421, 0.18779, 0.17522, 0.15665, 0.12297, 0.07419, 0.01112, 5e-05], "alpha_by_type": {"special": {"share": 0.0011, "mean": 0.55647, "std": 0.10174}, "whitespace": {"share": 0.02818, "mean": 0.51928, "std": 0.12608}, "digit": {"share": 0.0327, "mean": 0.49414, "std": 0.1214}, "word_start": {"share": 0.58492, "mean": 0.44756, "std": 0.17293}, "word_cont": {"share": 0.24973, "mean": 0.3163, "std": 0.13608}, "punct": {"share": 0.10338, "mean": 0.58601, "std": 0.16808}}, "alpha_p05": 0.15305, "alpha_p50": 0.43638, "alpha_p95": 0.7235}
{"step": 39910, "total_steps": 72479, "tokens": 20924334080, "loss": 4.457611501216888, "l_mix": 2.761506199836731, "l_a": 2.854009687900543, "l_c": 2.7996743619441986, "z_loss": 33.67536020278931, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11169420927762985, "tokens_per_second": 1456681.863218821, "eta_seconds": 11722.213547897338, "data_wait_seconds": 1.6391277313232422e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 39920, "total_steps": 72479, "tokens": 20929576960, "loss": 4.421172857284546, "l_mix": 2.739539295434952, "l_a": 2.8294205367565155, "l_c": 2.7760245501995087, "z_loss": 33.964054107666016, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12474632263183594, "tokens_per_second": 1449449.264903002, "eta_seconds": 11777.088998794556, "data_wait_seconds": 1.77919864654541e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 39930, "total_steps": 72479, "tokens": 20934819840, "loss": 4.4942909479141235, "l_mix": 2.7845636904239655, "l_a": 2.8729811906814575, "l_c": 2.82610946893692, "z_loss": 34.12019491195679, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12514491379261017, "tokens_per_second": 1430269.7259537226, "eta_seconds": 11931.350990891457, "data_wait_seconds": 1.5556812286376953e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 39940, "total_steps": 72479, "tokens": 20940062720, "loss": 3.076399564743042, "l_mix": 1.9042048156261444, "l_a": 1.9736961424350739, "l_c": 1.933619424700737, "z_loss": 40.15019512176514, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19495676457881927, "tokens_per_second": 1428462.3707443606, "eta_seconds": 11942.776779699327, "data_wait_seconds": 2.333521842956543e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 39950, "total_steps": 72479, "tokens": 20945305600, "loss": 2.951574206352234, "l_mix": 1.8265555053949356, "l_a": 1.8942268639802933, "l_c": 1.8558354824781418, "z_loss": 40.06522989273071, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.206268772482872, "tokens_per_second": 1305873.068195011, "eta_seconds": 13059.894385886191, "data_wait_seconds": 3.740191459655762e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 39960, "total_steps": 72479, "tokens": 20950548480, "loss": 4.448607265949249, "l_mix": 2.756735324859619, "l_a": 2.844179332256317, "l_c": 2.7953937351703644, "z_loss": 33.299399852752686, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11800426244735718, "tokens_per_second": 1475851.067163265, "eta_seconds": 11552.196458935738, "data_wait_seconds": 3.331899642944336e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 39970, "total_steps": 72479, "tokens": 20955791360, "loss": 4.511460900306702, "l_mix": 2.7960799038410187, "l_a": 2.884114623069763, "l_c": 2.8338217437267303, "z_loss": 33.8236289024353, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12631957232952118, "tokens_per_second": 1470199.4954387527, "eta_seconds": 11593.037982177733, "data_wait_seconds": 2.73287296295166e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 39980, "total_steps": 72479, "tokens": 20961034240, "loss": 4.425815582275391, "l_mix": 2.741654396057129, "l_a": 2.835007607936859, "l_c": 2.7788628339767456, "z_loss": 34.89115047454834, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14110150933265686, "tokens_per_second": 1463700.6123928137, "eta_seconds": 11640.92955057621, "data_wait_seconds": 1.963973045349121e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 39990, "total_steps": 72479, "tokens": 20966277120, "loss": 4.427314043045044, "l_mix": 2.7438286542892456, "l_a": 2.8312885761260986, "l_c": 2.7803293466567993, "z_loss": 33.52285194396973, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1531822681427002, "tokens_per_second": 1467420.3140820146, "eta_seconds": 11607.848595619202, "data_wait_seconds": 2.2143125534057617e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 40000, "total_steps": 72479, "tokens": 20971520000, "loss": 4.421319246292114, "l_mix": 2.7396817803382874, "l_a": 2.828273683786392, "l_c": 2.7771845161914825, "z_loss": 33.69119739532471, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13879185914993286, "tokens_per_second": 1434393.9075053362, "eta_seconds": 11871.460038208961, "data_wait_seconds": 1.5974044799804688e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.21612631, "gate_a2b/L1_absmax": 0.72573781, "gate_a2b/L1_rms": 0.22891441, "gate_a2b/L3_absmean": 0.21628539, "gate_a2b/L3_absmax": 0.60673606, "gate_a2b/L3_rms": 0.22254486, "gate_a2b/L5_absmean": 0.33771119, "gate_a2b/L5_absmax": 0.7131815, "gate_a2b/L5_rms": 0.34358779, "gate_c2b/L1_absmean": 0.0171618, "gate_c2b/L1_absmax": 0.62253737, "gate_c2b/L1_rms": 0.06741703, "gate_c2b/L3_absmean": 0.20368679, "gate_c2b/L3_absmax": 0.45720381, "gate_c2b/L3_rms": 0.20795582, "gate_c2b/L5_absmean": 0.5748297, "gate_c2b/L5_absmax": 0.80422574, "gate_c2b/L5_rms": 0.58738321, "gate_b2a/L7_absmean": 0.05367093, "gate_b2a/L7_absmax": 0.32695621, "gate_b2a/L7_rms": 0.05916708, "gate_b2a/L11_absmean": 0.08547007, "gate_b2a/L11_absmax": 0.59243381, "gate_b2a/L11_rms": 0.0939656, "gate_b2a/L14_absmean": 0.21980895, "gate_b2a/L14_absmax": 0.77143621, "gate_b2a/L14_rms": 0.22781344, "gate_b2c/L3_absmean": 0.04037122, "gate_b2c/L3_absmax": 0.60294867, "gate_b2c/L3_rms": 0.05233355, "gate_b2c/L5_absmean": 0.04419543, "gate_b2c/L5_absmax": 0.63499373, "gate_b2c/L5_rms": 0.05541429, "gate_b2c/L6_absmean": 0.06390342, "gate_b2c/L6_absmax": 0.42338479, "gate_b2c/L6_rms": 0.07157777}, "streams": {"cos_a2b/round0": 0.378676, "cos_c2b/round0": 0.307036, "cos_b2a/round0": 0.143746, "cos_b2c/round0": 0.128302, "rms_a/round0": 4.462472, "rms_b/round0": 8.406746, "rms_c/round0": 3.43821, "cos_a2b/round1": 0.418357, "cos_c2b/round1": 0.298817, "cos_b2a/round1": 0.245102, "cos_b2c/round1": 0.115295, "rms_a/round1": 8.41589, "rms_b/round1": 18.828007, "rms_c/round1": 6.826349, "cos_a2b/round2": 0.553636, "cos_c2b/round2": 0.517148, "cos_b2a/round2": 0.10085, "cos_b2c/round2": 0.177752, "rms_a/round2": 22.267809, "rms_b/round2": 48.499046, "rms_c/round2": 12.583949}, "alpha_mean": 0.43820943383379785, "alpha_hist": [0.01155, 0.06696, 0.16739, 0.19438, 0.18803, 0.16762, 0.12836, 0.06709, 0.00838, 0.00024], "alpha_by_type": {"special": {"share": 0.0016, "mean": 0.54763, "std": 0.11974}, "whitespace": {"share": 0.02568, "mean": 0.52721, "std": 0.12325}, "digit": {"share": 0.03897, "mean": 0.46317, "std": 0.12529}, "word_start": {"share": 0.58125, "mean": 0.45525, "std": 0.16561}, "word_cont": {"share": 0.25145, "mean": 0.32799, "std": 0.13596}, "punct": {"share": 0.10104, "mean": 0.58051, "std": 0.15927}}, "alpha_p05": 0.17441, "alpha_p50": 0.41822, "alpha_p95": 0.71721}
{"step": 40010, "total_steps": 72479, "tokens": 20976762880, "loss": 4.662509202957153, "l_mix": 2.892394483089447, "l_a": 2.9683018028736115, "l_c": 2.932080626487732, "z_loss": 33.87249135971069, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13095451891422272, "tokens_per_second": 1389816.5856846361, "eta_seconds": 12248.455837512016, "data_wait_seconds": 2.327561378479004e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 40020, "total_steps": 72479, "tokens": 20982005760, "loss": 4.787952780723572, "l_mix": 2.9703964293003082, "l_a": 3.046725422143936, "l_c": 3.0117955803871155, "z_loss": 33.353915214538574, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11918371915817261, "tokens_per_second": 1340129.5417750038, "eta_seconds": 12698.67103254795, "data_wait_seconds": 1.6063451766967773e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 40030, "total_steps": 72479, "tokens": 20987248640, "loss": 4.71696925163269, "l_mix": 2.926448106765747, "l_a": 3.003444641828537, "l_c": 2.964958906173706, "z_loss": 33.43466520309448, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1300959438085556, "tokens_per_second": 1435209.0289370064, "eta_seconds": 11853.75855989456, "data_wait_seconds": 2.8401613235473633e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 40040, "total_steps": 72479, "tokens": 20992491520, "loss": 4.7873382568359375, "l_mix": 2.969969779253006, "l_a": 3.0459176003932953, "l_c": 3.011976808309555, "z_loss": 33.44496536254883, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1272459626197815, "tokens_per_second": 1395716.1649527382, "eta_seconds": 12185.413380646705, "data_wait_seconds": 2.205371856689453e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 40050, "total_steps": 72479, "tokens": 20997734400, "loss": 4.775865316390991, "l_mix": 2.9630440771579742, "l_a": 3.0385898053646088, "l_c": 3.0041471123695374, "z_loss": 34.05138349533081, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12472615391016006, "tokens_per_second": 1442806.139394726, "eta_seconds": 11784.074857854843, "data_wait_seconds": 3.3795833587646484e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 40060, "total_steps": 72479, "tokens": 21002977280, "loss": 2.923945128917694, "l_mix": 1.8094518184661865, "l_a": 1.8764394223690033, "l_c": 1.8385380506515503, "z_loss": 41.6234564781189, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1473776251077652, "tokens_per_second": 1403572.6165635902, "eta_seconds": 12109.735165405273, "data_wait_seconds": 2.8192996978759766e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 40070, "total_steps": 72479, "tokens": 21008220160, "loss": 4.35272604227066, "l_mix": 2.6964582800865173, "l_a": 2.7861408591270447, "l_c": 2.7347511053085327, "z_loss": 34.75450658798218, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13407693803310394, "tokens_per_second": 1249797.3181826991, "eta_seconds": 13595.524286055565, "data_wait_seconds": 1.8984079360961914e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 40080, "total_steps": 72479, "tokens": 21013463040, "loss": 4.42097932100296, "l_mix": 2.7392320036888123, "l_a": 2.825955241918564, "l_c": 2.7798690497875214, "z_loss": 33.54978799819946, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18792983889579773, "tokens_per_second": 1287860.369741446, "eta_seconds": 13189.633993792535, "data_wait_seconds": 1.767277717590332e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 40090, "total_steps": 72479, "tokens": 21018705920, "loss": 4.439699113368988, "l_mix": 2.7502134144306183, "l_a": 2.8417660295963287, "l_c": 2.789852887392044, "z_loss": 34.56331825256348, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14249791204929352, "tokens_per_second": 1430911.058106868, "eta_seconds": 11867.379132890701, "data_wait_seconds": 1.7344951629638672e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 40100, "total_steps": 72479, "tokens": 21023948800, "loss": 4.462037682533264, "l_mix": 2.7651631832122803, "l_a": 2.8544550240039825, "l_c": 2.8017930388450623, "z_loss": 34.28306436538696, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1243501603603363, "tokens_per_second": 1408009.005753183, "eta_seconds": 12056.685065674781, "data_wait_seconds": 1.9818544387817383e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.21579848, "gate_a2b/L1_absmax": 0.72366518, "gate_a2b/L1_rms": 0.2286678, "gate_a2b/L3_absmean": 0.21570288, "gate_a2b/L3_absmax": 0.60588253, "gate_a2b/L3_rms": 0.22200266, "gate_a2b/L5_absmean": 0.33654994, "gate_a2b/L5_absmax": 0.71261901, "gate_a2b/L5_rms": 0.3424584, "gate_c2b/L1_absmean": 0.0174035, "gate_c2b/L1_absmax": 0.62174398, "gate_c2b/L1_rms": 0.06730034, "gate_c2b/L3_absmean": 0.20321964, "gate_c2b/L3_absmax": 0.4606069, "gate_c2b/L3_rms": 0.20740329, "gate_c2b/L5_absmean": 0.57447159, "gate_c2b/L5_absmax": 0.81001168, "gate_c2b/L5_rms": 0.58701777, "gate_b2a/L7_absmean": 0.05337501, "gate_b2a/L7_absmax": 0.32547429, "gate_b2a/L7_rms": 0.05890333, "gate_b2a/L11_absmean": 0.08538561, "gate_b2a/L11_absmax": 0.5952667, "gate_b2a/L11_rms": 0.09397501, "gate_b2a/L14_absmean": 0.2194223, "gate_b2a/L14_absmax": 0.76900965, "gate_b2a/L14_rms": 0.22739471, "gate_b2c/L3_absmean": 0.04045248, "gate_b2c/L3_absmax": 0.60180938, "gate_b2c/L3_rms": 0.05236498, "gate_b2c/L5_absmean": 0.04433976, "gate_b2c/L5_absmax": 0.63613951, "gate_b2c/L5_rms": 0.05557397, "gate_b2c/L6_absmean": 0.0638264, "gate_b2c/L6_absmax": 0.42217398, "gate_b2c/L6_rms": 0.07143718}, "streams": {"cos_a2b/round0": 0.376274, "cos_c2b/round0": 0.310696, "cos_b2a/round0": 0.143097, "cos_b2c/round0": 0.126975, "rms_a/round0": 4.457964, "rms_b/round0": 8.432872, "rms_c/round0": 3.451139, "cos_a2b/round1": 0.420448, "cos_c2b/round1": 0.299816, "cos_b2a/round1": 0.247044, "cos_b2c/round1": 0.112411, "rms_a/round1": 8.461842, "rms_b/round1": 18.970287, "rms_c/round1": 6.915945, "cos_a2b/round2": 0.548004, "cos_c2b/round2": 0.526467, "cos_b2a/round2": 0.103966, "cos_b2c/round2": 0.178521, "rms_a/round2": 22.398289, "rms_b/round2": 49.437149, "rms_c/round2": 12.843779}, "alpha_mean": 0.4192772969731422, "alpha_hist": [0.01076, 0.08008, 0.20334, 0.20193, 0.17679, 0.14758, 0.10941, 0.06158, 0.00832, 0.00021], "alpha_by_type": {"special": {"share": 0.00142, "mean": 0.57262, "std": 0.11122}, "whitespace": {"share": 0.02734, "mean": 0.50532, "std": 0.11469}, "digit": {"share": 0.03938, "mean": 0.43369, "std": 0.12735}, "word_start": {"share": 0.56934, "mean": 0.43787, "std": 0.16898}, "word_cont": {"share": 0.25824, "mean": 0.30953, "std": 0.12629}, "punct": {"share": 0.10428, "mean": 0.55944, "std": 0.17106}}, "alpha_p05": 0.15948, "alpha_p50": 0.40098, "alpha_p95": 0.72506}
{"step": 40110, "total_steps": 72479, "tokens": 21029191680, "loss": 4.348227441310883, "l_mix": 2.693481683731079, "l_a": 2.7823805809020996, "l_c": 2.7334382832050323, "z_loss": 33.8587703704834, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19165046513080597, "tokens_per_second": 1413444.4174176676, "eta_seconds": 12006.611694717407, "data_wait_seconds": 2.1159648895263672e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 40120, "total_steps": 72479, "tokens": 21034434560, "loss": 3.112647145986557, "l_mix": 1.9273008108139038, "l_a": 1.9948891699314117, "l_c": 1.956265076994896, "z_loss": 39.90995502471924, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.24335919320583344, "tokens_per_second": 1204339.0445009798, "eta_seconds": 14086.926326489449, "data_wait_seconds": 1.8984079360961914e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 40130, "total_steps": 72479, "tokens": 21039677440, "loss": 4.783201813697815, "l_mix": 2.967221438884735, "l_a": 3.0476760864257812, "l_c": 3.005591720342636, "z_loss": 34.00367975234985, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15084144473075867, "tokens_per_second": 1342586.677158147, "eta_seconds": 12632.474908733368, "data_wait_seconds": 2.2470951080322266e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20768, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 40140, "total_steps": 72479, "tokens": 21044920320, "loss": 4.733660161495209, "l_mix": 2.9363150894641876, "l_a": 3.01415479183197, "l_c": 2.976995050907135, "z_loss": 33.95985651016235, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1161455512046814, "tokens_per_second": 1351410.5716750412, "eta_seconds": 12546.112918877601, "data_wait_seconds": 1.6957521438598633e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 40150, "total_steps": 72479, "tokens": 21050163200, "loss": 4.739927351474762, "l_mix": 2.9396854639053345, "l_a": 3.0190733671188354, "l_c": 2.98173251748085, "z_loss": 34.92504835128784, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16899000108242035, "tokens_per_second": 1448643.9449708213, "eta_seconds": 11700.395263338089, "data_wait_seconds": 2.130866050720215e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 40160, "total_steps": 72479, "tokens": 21055406080, "loss": 4.730119943618774, "l_mix": 2.9345103800296783, "l_a": 3.0129720866680145, "l_c": 2.972392737865448, "z_loss": 33.98743438720703, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11564387381076813, "tokens_per_second": 1280644.5898562141, "eta_seconds": 13231.19935555458, "data_wait_seconds": 1.6868114471435547e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 40170, "total_steps": 72479, "tokens": 21060648960, "loss": 4.613958835601807, "l_mix": 2.861448645591736, "l_a": 2.9392974972724915, "l_c": 2.902403086423874, "z_loss": 35.125072956085205, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11629854142665863, "tokens_per_second": 1251880.9357981444, "eta_seconds": 13531.016015672685, "data_wait_seconds": 3.039836883544922e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 40180, "total_steps": 72479, "tokens": 21065891840, "loss": 4.336115002632141, "l_mix": 2.685524672269821, "l_a": 2.779060482978821, "l_c": 2.722907155752182, "z_loss": 34.656649112701416, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.10980468988418579, "tokens_per_second": 1434885.9402057575, "eta_seconds": 11801.61965317726, "data_wait_seconds": 2.1517276763916016e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 40190, "total_steps": 72479, "tokens": 21071134720, "loss": 4.404064893722534, "l_mix": 2.728574961423874, "l_a": 2.8181832432746887, "l_c": 2.766782909631729, "z_loss": 34.31657361984253, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11594488471746445, "tokens_per_second": 1285416.9126178848, "eta_seconds": 13169.840124106408, "data_wait_seconds": 1.609325408935547e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 40200, "total_steps": 72479, "tokens": 21076377600, "loss": 4.501164793968201, "l_mix": 2.788390636444092, "l_a": 2.882705330848694, "l_c": 2.8265416026115417, "z_loss": 33.92643451690674, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11186184734106064, "tokens_per_second": 1290885.234737278, "eta_seconds": 13109.989870977402, "data_wait_seconds": 0.04766783118247986, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.21299763, "gate_a2b/L1_absmax": 0.7324903, "gate_a2b/L1_rms": 0.22636953, "gate_a2b/L3_absmean": 0.21302496, "gate_a2b/L3_absmax": 0.58948863, "gate_a2b/L3_rms": 0.21944451, "gate_a2b/L5_absmean": 0.33592364, "gate_a2b/L5_absmax": 0.71987915, "gate_a2b/L5_rms": 0.34196532, "gate_c2b/L1_absmean": 0.02107693, "gate_c2b/L1_absmax": 0.63164699, "gate_c2b/L1_rms": 0.06883319, "gate_c2b/L3_absmean": 0.20319282, "gate_c2b/L3_absmax": 0.45397663, "gate_c2b/L3_rms": 0.20744567, "gate_c2b/L5_absmean": 0.57268918, "gate_c2b/L5_absmax": 0.82087904, "gate_c2b/L5_rms": 0.58525205, "gate_b2a/L7_absmean": 0.0536194, "gate_b2a/L7_absmax": 0.33080763, "gate_b2a/L7_rms": 0.05950874, "gate_b2a/L11_absmean": 0.08708138, "gate_b2a/L11_absmax": 0.59169173, "gate_b2a/L11_rms": 0.09542545, "gate_b2a/L14_absmean": 0.21885005, "gate_b2a/L14_absmax": 0.7726993, "gate_b2a/L14_rms": 0.22700727, "gate_b2c/L3_absmean": 0.04043766, "gate_b2c/L3_absmax": 0.60479158, "gate_b2c/L3_rms": 0.05239805, "gate_b2c/L5_absmean": 0.04390672, "gate_b2c/L5_absmax": 0.63001758, "gate_b2c/L5_rms": 0.05523964, "gate_b2c/L6_absmean": 0.06360154, "gate_b2c/L6_absmax": 0.42304817, "gate_b2c/L6_rms": 0.07141994}, "streams": {"cos_a2b/round0": 0.37445, "cos_c2b/round0": 0.293113, "cos_b2a/round0": 0.141966, "cos_b2c/round0": 0.119584, "rms_a/round0": 4.545629, "rms_b/round0": 8.543097, "rms_c/round0": 3.471385, "cos_a2b/round1": 0.420749, "cos_c2b/round1": 0.29799, "cos_b2a/round1": 0.248173, "cos_b2c/round1": 0.11796, "rms_a/round1": 8.602252, "rms_b/round1": 19.282026, "rms_c/round1": 6.952742, "cos_a2b/round2": 0.553391, "cos_c2b/round2": 0.518716, "cos_b2a/round2": 0.097485, "cos_b2c/round2": 0.176077, "rms_a/round2": 22.881649, "rms_b/round2": 49.776005, "rms_c/round2": 12.958945}, "alpha_mean": 0.44290169010068325, "alpha_hist": [0.01027, 0.0737, 0.15895, 0.19104, 0.18459, 0.16281, 0.13139, 0.07439, 0.01263, 0.00023], "alpha_by_type": {"special": {"share": 0.00137, "mean": 0.57749, "std": 0.10948}, "whitespace": {"share": 0.02478, "mean": 0.53498, "std": 0.11822}, "digit": {"share": 0.02715, "mean": 0.43692, "std": 0.10244}, "word_start": {"share": 0.57727, "mean": 0.46048, "std": 0.17167}, "word_cont": {"share": 0.26535, "mean": 0.33693, "std": 0.14241}, "punct": {"share": 0.10408, "mean": 0.59345, "std": 0.15404}}, "alpha_p05": 0.17301, "alpha_p50": 0.42274, "alpha_p95": 0.75236}
{"step": 40210, "total_steps": 72479, "tokens": 21081620480, "loss": 4.341108918190002, "l_mix": 2.6888291239738464, "l_a": 2.780296713113785, "l_c": 2.7273024022579193, "z_loss": 34.15306901931763, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.09986850619316101, "tokens_per_second": 1415530.4224473587, "eta_seconds": 11951.879806828498, "data_wait_seconds": 1.806020736694336e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 40220, "total_steps": 72479, "tokens": 21086863360, "loss": 4.26863032579422, "l_mix": 2.6431125700473785, "l_a": 2.7347218096256256, "l_c": 2.683670699596405, "z_loss": 35.06366205215454, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11364755034446716, "tokens_per_second": 1418871.7859664548, "eta_seconds": 11920.038695025443, "data_wait_seconds": 1.7195940017700195e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 40230, "total_steps": 72479, "tokens": 21092106240, "loss": 2.926788955926895, "l_mix": 1.809731975197792, "l_a": 1.8723472207784653, "l_c": 1.8511761277914047, "z_loss": 41.21315288543701, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.28805622458457947, "tokens_per_second": 1338427.2493066757, "eta_seconds": 12632.560881257059, "data_wait_seconds": 1.6510486602783203e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 40240, "total_steps": 72479, "tokens": 21097349120, "loss": 3.0083818435668945, "l_mix": 1.8602113872766495, "l_a": 1.9335794150829315, "l_c": 1.8936554044485092, "z_loss": 41.96492147445679, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2311624139547348, "tokens_per_second": 1432532.07206179, "eta_seconds": 11799.052294635772, "data_wait_seconds": 2.6404857635498047e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08236}}
{"step": 40250, "total_steps": 72479, "tokens": 21102592000, "loss": 4.713216006755829, "l_mix": 2.9239651262760162, "l_a": 3.002206265926361, "l_c": 2.9619630575180054, "z_loss": 33.42582941055298, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12641671299934387, "tokens_per_second": 1465809.1177775504, "eta_seconds": 11527.61143798828, "data_wait_seconds": 1.946091651916504e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20768, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 40260, "total_steps": 72479, "tokens": 21107834880, "loss": 4.649764835834503, "l_mix": 2.884585440158844, "l_a": 2.961943179368973, "l_c": 2.9219874441623688, "z_loss": 33.483625411987305, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1527097523212433, "tokens_per_second": 1416454.241927697, "eta_seconds": 11925.577665686607, "data_wait_seconds": 3.0040740966796875e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20768, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 40270, "total_steps": 72479, "tokens": 21113077760, "loss": 4.830830454826355, "l_mix": 2.996930569410324, "l_a": 3.0735355615615845, "l_c": 3.039463460445404, "z_loss": 33.67893195152283, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18025702238082886, "tokens_per_second": 1343400.3942383332, "eta_seconds": 12570.185526537896, "data_wait_seconds": 2.238154411315918e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 40280, "total_steps": 72479, "tokens": 21118320640, "loss": 4.812016427516937, "l_mix": 2.9851711690425873, "l_a": 3.0624583065509796, "l_c": 3.027025580406189, "z_loss": 33.59442472457886, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12043243646621704, "tokens_per_second": 1391734.9888361758, "eta_seconds": 12129.859094882011, "data_wait_seconds": 2.1129846572875977e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 40290, "total_steps": 72479, "tokens": 21123563520, "loss": 4.239335715770721, "l_mix": 2.625481367111206, "l_a": 2.71677765250206, "l_c": 2.6627364456653595, "z_loss": 35.57925081253052, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1302720308303833, "tokens_per_second": 1404757.763649291, "eta_seconds": 12013.677282094955, "data_wait_seconds": 2.0772218704223633e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 40300, "total_steps": 72479, "tokens": 21128806400, "loss": 4.268631219863892, "l_mix": 2.642892301082611, "l_a": 2.736772507429123, "l_c": 2.6823568046092987, "z_loss": 35.29106903076172, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1407414823770523, "tokens_per_second": 1369616.165579923, "eta_seconds": 12318.09610311985, "data_wait_seconds": 2.9146671295166016e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.21236296, "gate_a2b/L1_absmax": 0.73351669, "gate_a2b/L1_rms": 0.22540139, "gate_a2b/L3_absmean": 0.2125798, "gate_a2b/L3_absmax": 0.59180164, "gate_a2b/L3_rms": 0.21883132, "gate_a2b/L5_absmean": 0.33401534, "gate_a2b/L5_absmax": 0.71859771, "gate_a2b/L5_rms": 0.33995727, "gate_c2b/L1_absmean": 0.01750777, "gate_c2b/L1_absmax": 0.6307252, "gate_c2b/L1_rms": 0.06796508, "gate_c2b/L3_absmean": 0.20328894, "gate_c2b/L3_absmax": 0.46319994, "gate_c2b/L3_rms": 0.20750713, "gate_c2b/L5_absmean": 0.57403928, "gate_c2b/L5_absmax": 0.81176722, "gate_c2b/L5_rms": 0.58667129, "gate_b2a/L7_absmean": 0.05361066, "gate_b2a/L7_absmax": 0.32567185, "gate_b2a/L7_rms": 0.05914055, "gate_b2a/L11_absmean": 0.08580324, "gate_b2a/L11_absmax": 0.5884788, "gate_b2a/L11_rms": 0.09408575, "gate_b2a/L14_absmean": 0.2204954, "gate_b2a/L14_absmax": 0.77742177, "gate_b2a/L14_rms": 0.22848214, "gate_b2c/L3_absmean": 0.04021685, "gate_b2c/L3_absmax": 0.6002835, "gate_b2c/L3_rms": 0.0520619, "gate_b2c/L5_absmean": 0.04394139, "gate_b2c/L5_absmax": 0.62006861, "gate_b2c/L5_rms": 0.05479116, "gate_b2c/L6_absmean": 0.06319849, "gate_b2c/L6_absmax": 0.41793543, "gate_b2c/L6_rms": 0.07075059}, "streams": {"cos_a2b/round0": 0.373561, "cos_c2b/round0": 0.301455, "cos_b2a/round0": 0.14784, "cos_b2c/round0": 0.124724, "rms_a/round0": 4.541581, "rms_b/round0": 8.561419, "rms_c/round0": 3.486046, "cos_a2b/round1": 0.425794, "cos_c2b/round1": 0.302642, "cos_b2a/round1": 0.246772, "cos_b2c/round1": 0.117997, "rms_a/round1": 8.741684, "rms_b/round1": 19.36775, "rms_c/round1": 6.989402, "cos_a2b/round2": 0.547053, "cos_c2b/round2": 0.532709, "cos_b2a/round2": 0.098737, "cos_b2c/round2": 0.179216, "rms_a/round2": 22.823603, "rms_b/round2": 50.357311, "rms_c/round2": 13.02049}, "alpha_mean": 0.4262728160907443, "alpha_hist": [0.01059, 0.08286, 0.18277, 0.19672, 0.17503, 0.16515, 0.11792, 0.05913, 0.00977, 8e-05], "alpha_by_type": {"special": {"share": 0.00099, "mean": 0.55724, "std": 0.09471}, "whitespace": {"share": 0.02458, "mean": 0.53523, "std": 0.1167}, "digit": {"share": 0.02586, "mean": 0.43001, "std": 0.10714}, "word_start": {"share": 0.59941, "mean": 0.44334, "std": 0.1685}, "word_cont": {"share": 0.24931, "mean": 0.3143, "std": 0.12723}, "punct": {"share": 0.09984, "mean": 0.57432, "std": 0.16697}}, "alpha_p05": 0.16, "alpha_p50": 0.41087, "alpha_p95": 0.70355}
{"step": 40310, "total_steps": 72479, "tokens": 21134049280, "loss": 4.203080415725708, "l_mix": 2.603160083293915, "l_a": 2.6914665699005127, "l_c": 2.6416011452674866, "z_loss": 34.58403730392456, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1514054387807846, "tokens_per_second": 1422822.225420537, "eta_seconds": 11853.779320192336, "data_wait_seconds": 6.458163261413574e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 40320, "total_steps": 72479, "tokens": 21139292160, "loss": 4.320742130279541, "l_mix": 2.676444798707962, "l_a": 2.7698222994804382, "l_c": 2.711168557405472, "z_loss": 34.882872581481934, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16399811208248138, "tokens_per_second": 1411760.6932050474, "eta_seconds": 11942.943214917183, "data_wait_seconds": 1.856684684753418e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 40330, "total_steps": 72479, "tokens": 21144535040, "loss": 4.230716943740845, "l_mix": 2.620461404323578, "l_a": 2.7091812789440155, "l_c": 2.658336818218231, "z_loss": 34.52134370803833, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11844086647033691, "tokens_per_second": 1373629.1451363454, "eta_seconds": 12270.659057927132, "data_wait_seconds": 1.9341707229614258e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 40340, "total_steps": 72479, "tokens": 21149777920, "loss": 2.7388710379600525, "l_mix": 1.6950296461582184, "l_a": 1.7579920440912247, "l_c": 1.7214793413877487, "z_loss": 37.83919858932495, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.25684136152267456, "tokens_per_second": 1381526.5006222143, "eta_seconds": 12196.720095062257, "data_wait_seconds": 2.0951032638549805e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 40350, "total_steps": 72479, "tokens": 21155020800, "loss": 2.871914714574814, "l_mix": 1.7768064737319946, "l_a": 1.8473051190376282, "l_c": 1.8030557036399841, "z_loss": 41.04084396362305, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19283775985240936, "tokens_per_second": 1318608.5208744796, "eta_seconds": 12774.715835165976, "data_wait_seconds": 3.2842159271240234e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 40360, "total_steps": 72479, "tokens": 21160263680, "loss": 3.160935014486313, "l_mix": 1.943851962685585, "l_a": 2.060718387365341, "l_c": 1.996225044131279, "z_loss": 36.47135019302368, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16800716519355774, "tokens_per_second": 1433003.404277004, "eta_seconds": 11751.267458081245, "data_wait_seconds": 2.4437904357910156e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20045, "finemath": 0.08233}}
{"step": 40370, "total_steps": 72479, "tokens": 21165506560, "loss": 3.2001828849315643, "l_mix": 1.9688575118780136, "l_a": 2.0845741629600525, "l_c": 2.0198435336351395, "z_loss": 36.29093933105469, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13011914491653442, "tokens_per_second": 1410978.5906006463, "eta_seconds": 11930.984285759927, "data_wait_seconds": 2.3633241653442383e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20045, "finemath": 0.08234}}
{"step": 40380, "total_steps": 72479, "tokens": 21170749440, "loss": 4.203513085842133, "l_mix": 2.6027985513210297, "l_a": 2.696331709623337, "l_c": 2.6393831074237823, "z_loss": 35.312214374542236, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14122022688388824, "tokens_per_second": 1429167.0978615559, "eta_seconds": 11775.474356484412, "data_wait_seconds": 2.4616718292236328e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 40390, "total_steps": 72479, "tokens": 21175992320, "loss": 4.237950444221497, "l_mix": 2.6244564056396484, "l_a": 2.716160535812378, "l_c": 2.6621527671813965, "z_loss": 34.98807668685913, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11707461625337601, "tokens_per_second": 1408339.4045112112, "eta_seconds": 11945.897116923332, "data_wait_seconds": 2.771615982055664e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 40400, "total_steps": 72479, "tokens": 21181235200, "loss": 4.536643385887146, "l_mix": 2.810863733291626, "l_a": 2.9015019237995148, "l_c": 2.8510969281196594, "z_loss": 34.22379684448242, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14490388333797455, "tokens_per_second": 1335015.0189434746, "eta_seconds": 12598.086548352241, "data_wait_seconds": 2.288818359375e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.21137057, "gate_a2b/L1_absmax": 0.73251337, "gate_a2b/L1_rms": 0.22423482, "gate_a2b/L3_absmean": 0.21215622, "gate_a2b/L3_absmax": 0.59705734, "gate_a2b/L3_rms": 0.21840231, "gate_a2b/L5_absmean": 0.33279869, "gate_a2b/L5_absmax": 0.71883553, "gate_a2b/L5_rms": 0.33873129, "gate_c2b/L1_absmean": 0.01860384, "gate_c2b/L1_absmax": 0.62981176, "gate_c2b/L1_rms": 0.06831691, "gate_c2b/L3_absmean": 0.20273195, "gate_c2b/L3_absmax": 0.46682683, "gate_c2b/L3_rms": 0.20705549, "gate_c2b/L5_absmean": 0.57362157, "gate_c2b/L5_absmax": 0.81068325, "gate_c2b/L5_rms": 0.58624512, "gate_b2a/L7_absmean": 0.0533438, "gate_b2a/L7_absmax": 0.32596081, "gate_b2a/L7_rms": 0.0589013, "gate_b2a/L11_absmean": 0.08554748, "gate_b2a/L11_absmax": 0.58862704, "gate_b2a/L11_rms": 0.09386432, "gate_b2a/L14_absmean": 0.22039255, "gate_b2a/L14_absmax": 0.77601784, "gate_b2a/L14_rms": 0.22833411, "gate_b2c/L3_absmean": 0.03992565, "gate_b2c/L3_absmax": 0.59697467, "gate_b2c/L3_rms": 0.05173661, "gate_b2c/L5_absmean": 0.04380782, "gate_b2c/L5_absmax": 0.61947131, "gate_b2c/L5_rms": 0.0546317, "gate_b2c/L6_absmean": 0.06299406, "gate_b2c/L6_absmax": 0.41689086, "gate_b2c/L6_rms": 0.07048308}, "streams": {"cos_a2b/round0": 0.371435, "cos_c2b/round0": 0.29835, "cos_b2a/round0": 0.141982, "cos_b2c/round0": 0.125203, "rms_a/round0": 4.586431, "rms_b/round0": 8.670585, "rms_c/round0": 3.51727, "cos_a2b/round1": 0.417484, "cos_c2b/round1": 0.294625, "cos_b2a/round1": 0.245578, "cos_b2c/round1": 0.114415, "rms_a/round1": 8.710664, "rms_b/round1": 19.519173, "rms_c/round1": 7.010756, "cos_a2b/round2": 0.544435, "cos_c2b/round2": 0.527425, "cos_b2a/round2": 0.095137, "cos_b2c/round2": 0.173627, "rms_a/round2": 22.976957, "rms_b/round2": 50.232861, "rms_c/round2": 12.872965}, "alpha_mean": 0.4178433329698663, "alpha_hist": [0.01196, 0.07709, 0.19498, 0.21457, 0.18306, 0.14893, 0.1042, 0.05573, 0.0094, 9e-05], "alpha_by_type": {"special": {"share": 0.00081, "mean": 0.55616, "std": 0.10444}, "whitespace": {"share": 0.02367, "mean": 0.51931, "std": 0.12454}, "digit": {"share": 0.03481, "mean": 0.43487, "std": 0.12709}, "word_start": {"share": 0.55244, "mean": 0.43918, "std": 0.16358}, "word_cont": {"share": 0.27419, "mean": 0.30652, "std": 0.12472}, "punct": {"share": 0.11409, "mean": 0.55482, "std": 0.16386}}, "alpha_p05": 0.16668, "alpha_p50": 0.3928, "alpha_p95": 0.72487}
{"step": 40410, "total_steps": 72479, "tokens": 21186478080, "loss": 4.160232782363892, "l_mix": 2.575994610786438, "l_a": 2.665911912918091, "l_c": 2.614881932735443, "z_loss": 35.08792495727539, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11496537178754807, "tokens_per_second": 1394711.085420679, "eta_seconds": 12055.10736076832, "data_wait_seconds": 2.1070241928100586e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 40420, "total_steps": 72479, "tokens": 21191720960, "loss": 4.222968280315399, "l_mix": 2.615015923976898, "l_a": 2.706676721572876, "l_c": 2.6531642973423004, "z_loss": 34.61446571350098, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1500462144613266, "tokens_per_second": 1384071.7219463934, "eta_seconds": 12143.986995387077, "data_wait_seconds": 1.6570091247558594e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 40430, "total_steps": 72479, "tokens": 21196963840, "loss": 4.205424070358276, "l_mix": 2.604246884584427, "l_a": 2.696625739336014, "l_c": 2.6406314074993134, "z_loss": 34.7733097076416, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12237320095300674, "tokens_per_second": 1346056.805660406, "eta_seconds": 12483.058695101738, "data_wait_seconds": 2.1964311599731445e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 40440, "total_steps": 72479, "tokens": 21202206720, "loss": 4.243738174438477, "l_mix": 2.627846449613571, "l_a": 2.7195842564105988, "l_c": 2.6667213439941406, "z_loss": 35.393763065338135, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12459626793861389, "tokens_per_second": 1313445.656269113, "eta_seconds": 12789.005126953125, "data_wait_seconds": 1.5467405319213867e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 40450, "total_steps": 72479, "tokens": 21207449600, "loss": 4.3899712562561035, "l_mix": 2.7196506559848785, "l_a": 2.810447186231613, "l_c": 2.7572878003120422, "z_loss": 34.25427532196045, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14801688492298126, "tokens_per_second": 1437256.6684847325, "eta_seconds": 11683.661464381217, "data_wait_seconds": 2.154707908630371e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 40460, "total_steps": 72479, "tokens": 21212692480, "loss": 3.576945334672928, "l_mix": 2.2150617986917496, "l_a": 2.269355982542038, "l_c": 2.270255669951439, "z_loss": 39.7376434803009, "lr": 0.0023, "wd": 0.01, "grad_norm": 1.3279507160186768, "tokens_per_second": 1421891.2787465195, "eta_seconds": 11806.231406664849, "data_wait_seconds": 1.665949821472168e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 40470, "total_steps": 72479, "tokens": 21217935360, "loss": 3.150587946176529, "l_mix": 1.9373691380023956, "l_a": 2.055577963590622, "l_c": 1.988484725356102, "z_loss": 36.66336536407471, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13268093764781952, "tokens_per_second": 1424460.201075427, "eta_seconds": 11781.259019613266, "data_wait_seconds": 1.9729137420654297e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 40480, "total_steps": 72479, "tokens": 21223178240, "loss": 3.1437688171863556, "l_mix": 1.9338945895433426, "l_a": 2.051271617412567, "l_c": 1.981642559170723, "z_loss": 36.98138236999512, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14929930865764618, "tokens_per_second": 1351487.8135082084, "eta_seconds": 12413.498327040672, "data_wait_seconds": 2.422928810119629e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 40490, "total_steps": 72479, "tokens": 21228421120, "loss": 4.209950566291809, "l_mix": 2.606462985277176, "l_a": 2.698955774307251, "l_c": 2.6460024416446686, "z_loss": 35.57876253128052, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12786541879177094, "tokens_per_second": 1409501.449131637, "eta_seconds": 11898.85178360939, "data_wait_seconds": 1.5944242477416992e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 40500, "total_steps": 72479, "tokens": 21233664000, "loss": 4.206188678741455, "l_mix": 2.6042394638061523, "l_a": 2.6975003480911255, "l_c": 2.6423300206661224, "z_loss": 34.81915855407715, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11763332784175873, "tokens_per_second": 1328455.2871701512, "eta_seconds": 12620.828200936317, "data_wait_seconds": 1.874566078186035e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.21084002, "gate_a2b/L1_absmax": 0.73328775, "gate_a2b/L1_rms": 0.22385508, "gate_a2b/L3_absmean": 0.21132593, "gate_a2b/L3_absmax": 0.59962249, "gate_a2b/L3_rms": 0.21767347, "gate_a2b/L5_absmean": 0.33027086, "gate_a2b/L5_absmax": 0.7171222, "gate_a2b/L5_rms": 0.3362574, "gate_c2b/L1_absmean": 0.02114748, "gate_c2b/L1_absmax": 0.63267714, "gate_c2b/L1_rms": 0.06900858, "gate_c2b/L3_absmean": 0.20251678, "gate_c2b/L3_absmax": 0.45624709, "gate_c2b/L3_rms": 0.20682485, "gate_c2b/L5_absmean": 0.57223427, "gate_c2b/L5_absmax": 0.80930543, "gate_c2b/L5_rms": 0.58481383, "gate_b2a/L7_absmean": 0.05358944, "gate_b2a/L7_absmax": 0.33504388, "gate_b2a/L7_rms": 0.05915317, "gate_b2a/L11_absmean": 0.08622201, "gate_b2a/L11_absmax": 0.59882432, "gate_b2a/L11_rms": 0.09464088, "gate_b2a/L14_absmean": 0.21958652, "gate_b2a/L14_absmax": 0.7810263, "gate_b2a/L14_rms": 0.22771387, "gate_b2c/L3_absmean": 0.0398622, "gate_b2c/L3_absmax": 0.60321426, "gate_b2c/L3_rms": 0.05199917, "gate_b2c/L5_absmean": 0.04404585, "gate_b2c/L5_absmax": 0.6259982, "gate_b2c/L5_rms": 0.05509047, "gate_b2c/L6_absmean": 0.06306447, "gate_b2c/L6_absmax": 0.42310032, "gate_b2c/L6_rms": 0.07071414}, "streams": {"cos_a2b/round0": 0.38, "cos_c2b/round0": 0.311865, "cos_b2a/round0": 0.144705, "cos_b2c/round0": 0.128893, "rms_a/round0": 4.65039, "rms_b/round0": 8.684239, "rms_c/round0": 3.52747, "cos_a2b/round1": 0.426504, "cos_c2b/round1": 0.30272, "cos_b2a/round1": 0.251635, "cos_b2c/round1": 0.118675, "rms_a/round1": 8.851535, "rms_b/round1": 19.729401, "rms_c/round1": 7.054465, "cos_a2b/round2": 0.55166, "cos_c2b/round2": 0.526378, "cos_b2a/round2": 0.103093, "cos_b2c/round2": 0.174845, "rms_a/round2": 23.50433, "rms_b/round2": 51.196339, "rms_c/round2": 13.062477}, "alpha_mean": 0.429347998464479, "alpha_hist": [0.01083, 0.08228, 0.18279, 0.1976, 0.18004, 0.14075, 0.12299, 0.07077, 0.01186, 0.00011], "alpha_by_type": {"special": {"share": 0.00113, "mean": 0.55685, "std": 0.11577}, "whitespace": {"share": 0.02664, "mean": 0.507, "std": 0.11247}, "digit": {"share": 0.03043, "mean": 0.39974, "std": 0.11919}, "word_start": {"share": 0.58586, "mean": 0.44811, "std": 0.17535}, "word_cont": {"share": 0.24918, "mean": 0.31817, "std": 0.12895}, "punct": {"share": 0.10677, "mean": 0.57355, "std": 0.17075}}, "alpha_p05": 0.18418, "alpha_p50": 0.41277, "alpha_p95": 0.7454}
{"step": 40510, "total_steps": 72479, "tokens": 21238906880, "loss": 4.1918076276779175, "l_mix": 2.5955998301506042, "l_a": 2.6868875920772552, "l_c": 2.633804976940155, "z_loss": 35.971651554107666, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13299071788787842, "tokens_per_second": 1435266.0762309462, "eta_seconds": 11677.948325800897, "data_wait_seconds": 2.0623207092285156e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 40520, "total_steps": 72479, "tokens": 21244149760, "loss": 4.1611925065517426, "l_mix": 2.5768993496894836, "l_a": 2.667333960533142, "l_c": 2.6136434376239777, "z_loss": 34.85837745666504, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15719056129455566, "tokens_per_second": 1372538.323655969, "eta_seconds": 12207.834129810333, "data_wait_seconds": 2.35140323638916e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 40530, "total_steps": 72479, "tokens": 21249392640, "loss": 4.726209819316864, "l_mix": 2.9318167567253113, "l_a": 3.008658140897751, "l_c": 2.9726518392562866, "z_loss": 34.053367614746094, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14933077991008759, "tokens_per_second": 1363711.249476831, "eta_seconds": 12283.008824944496, "data_wait_seconds": 2.6673078536987305e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 40540, "total_steps": 72479, "tokens": 21254635520, "loss": 4.843170404434204, "l_mix": 3.005063146352768, "l_a": 3.083139032125473, "l_c": 3.043884724378586, "z_loss": 33.09863615036011, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11900924146175385, "tokens_per_second": 1396968.651727328, "eta_seconds": 11986.836219477653, "data_wait_seconds": 2.586841583251953e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 40550, "total_steps": 72479, "tokens": 21259878400, "loss": 4.864056885242462, "l_mix": 3.0176811516284943, "l_a": 3.094172328710556, "l_c": 3.0604130923748016, "z_loss": 34.268930435180664, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16446037590503693, "tokens_per_second": 1398707.0178361563, "eta_seconds": 11968.190148854255, "data_wait_seconds": 1.773238182067871e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 40560, "total_steps": 72479, "tokens": 21265121280, "loss": 4.80607670545578, "l_mix": 2.982159733772278, "l_a": 3.057356506586075, "l_c": 3.022366523742676, "z_loss": 33.684906005859375, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14473971724510193, "tokens_per_second": 1388004.4705718393, "eta_seconds": 12056.696521377562, "data_wait_seconds": 2.7000904083251953e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 40570, "total_steps": 72479, "tokens": 21270364160, "loss": 4.788205564022064, "l_mix": 2.970984011888504, "l_a": 3.0472183525562286, "l_c": 3.01018688082695, "z_loss": 33.69236946105957, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13753855228424072, "tokens_per_second": 1389691.8661367397, "eta_seconds": 12038.284312987329, "data_wait_seconds": 1.5795230865478516e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 40580, "total_steps": 72479, "tokens": 21275607040, "loss": 3.137749493122101, "l_mix": 1.9304903447628021, "l_a": 2.0447760820388794, "l_c": 1.9794208705425262, "z_loss": 37.05982065200806, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1323300302028656, "tokens_per_second": 1441392.2952149953, "eta_seconds": 11602.852996730804, "data_wait_seconds": 3.126263618469238e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 40590, "total_steps": 72479, "tokens": 21280849920, "loss": 3.177730679512024, "l_mix": 1.9547862708568573, "l_a": 2.07038015127182, "l_c": 2.00610089302063, "z_loss": 36.89576053619385, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17054840922355652, "tokens_per_second": 1456681.863218821, "eta_seconds": 11477.46838493347, "data_wait_seconds": 4.330277442932129e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 40600, "total_steps": 72479, "tokens": 21286092800, "loss": 4.091747164726257, "l_mix": 2.5327968895435333, "l_a": 2.626422166824341, "l_c": 2.5700787007808685, "z_loss": 35.24233293533325, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12439598888158798, "tokens_per_second": 1390069.6067636444, "eta_seconds": 12023.69800093174, "data_wait_seconds": 2.0682811737060547e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.2101448, "gate_a2b/L1_absmax": 0.74399865, "gate_a2b/L1_rms": 0.22321638, "gate_a2b/L3_absmean": 0.21090512, "gate_a2b/L3_absmax": 0.593283, "gate_a2b/L3_rms": 0.21707451, "gate_a2b/L5_absmean": 0.32933304, "gate_a2b/L5_absmax": 0.71743256, "gate_a2b/L5_rms": 0.33525243, "gate_c2b/L1_absmean": 0.01820331, "gate_c2b/L1_absmax": 0.63947731, "gate_c2b/L1_rms": 0.06855528, "gate_c2b/L3_absmean": 0.2021586, "gate_c2b/L3_absmax": 0.46179929, "gate_c2b/L3_rms": 0.20640767, "gate_c2b/L5_absmean": 0.57154447, "gate_c2b/L5_absmax": 0.80315828, "gate_c2b/L5_rms": 0.58408874, "gate_b2a/L7_absmean": 0.05341423, "gate_b2a/L7_absmax": 0.3273257, "gate_b2a/L7_rms": 0.05888347, "gate_b2a/L11_absmean": 0.08545724, "gate_b2a/L11_absmax": 0.59109861, "gate_b2a/L11_rms": 0.0937079, "gate_b2a/L14_absmean": 0.21955533, "gate_b2a/L14_absmax": 0.78570122, "gate_b2a/L14_rms": 0.22761276, "gate_b2c/L3_absmean": 0.03996471, "gate_b2c/L3_absmax": 0.60264665, "gate_b2c/L3_rms": 0.05192858, "gate_b2c/L5_absmean": 0.04356136, "gate_b2c/L5_absmax": 0.61767572, "gate_b2c/L5_rms": 0.05443652, "gate_b2c/L6_absmean": 0.06265565, "gate_b2c/L6_absmax": 0.42468548, "gate_b2c/L6_rms": 0.07029653}, "streams": {"cos_a2b/round0": 0.375829, "cos_c2b/round0": 0.308921, "cos_b2a/round0": 0.151233, "cos_b2c/round0": 0.128537, "rms_a/round0": 4.6842, "rms_b/round0": 8.799063, "rms_c/round0": 3.539781, "cos_a2b/round1": 0.427395, "cos_c2b/round1": 0.305957, "cos_b2a/round1": 0.255339, "cos_b2c/round1": 0.121771, "rms_a/round1": 8.904502, "rms_b/round1": 19.773556, "rms_c/round1": 7.070517, "cos_a2b/round2": 0.554693, "cos_c2b/round2": 0.523821, "cos_b2a/round2": 0.100782, "cos_b2c/round2": 0.17775, "rms_a/round2": 23.499475, "rms_b/round2": 50.822735, "rms_c/round2": 13.096136}, "alpha_mean": 0.4488091311335367, "alpha_hist": [0.00865, 0.06433, 0.16072, 0.19368, 0.18111, 0.16483, 0.13049, 0.0804, 0.01546, 0.00034], "alpha_by_type": {"special": {"share": 0.00116, "mean": 0.60291, "std": 0.11525}, "whitespace": {"share": 0.02379, "mean": 0.54019, "std": 0.11345}, "digit": {"share": 0.02393, "mean": 0.43588, "std": 0.11853}, "word_start": {"share": 0.59715, "mean": 0.46971, "std": 0.16914}, "word_cont": {"share": 0.24718, "mean": 0.3302, "std": 0.13426}, "punct": {"share": 0.1068, "mean": 0.58732, "std": 0.17052}}, "alpha_p05": 0.1833, "alpha_p50": 0.44746, "alpha_p95": 0.74055}
{"step": 40610, "total_steps": 72479, "tokens": 21291335680, "loss": 4.29364949464798, "l_mix": 2.6595980525016785, "l_a": 2.7495961487293243, "l_c": 2.697242021560669, "z_loss": 35.0835599899292, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13052694499492645, "tokens_per_second": 1441729.4745866233, "eta_seconds": 11589.229856586457, "data_wait_seconds": 2.4318695068359375e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 40620, "total_steps": 72479, "tokens": 21296578560, "loss": 4.310974478721619, "l_mix": 2.6700287461280823, "l_a": 2.7604079246520996, "l_c": 2.7094109058380127, "z_loss": 34.52727174758911, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1637004017829895, "tokens_per_second": 1422414.700039263, "eta_seconds": 11742.912521600723, "data_wait_seconds": 1.519918441772461e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 40630, "total_steps": 72479, "tokens": 21301821440, "loss": 4.122051358222961, "l_mix": 2.5517988801002502, "l_a": 2.644831359386444, "l_c": 2.5893434882164, "z_loss": 35.31661081314087, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11926991492509842, "tokens_per_second": 1389659.3724657863, "eta_seconds": 12015.929113888742, "data_wait_seconds": 1.6480684280395508e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 40640, "total_steps": 72479, "tokens": 21307064320, "loss": 3.165572315454483, "l_mix": 1.9477567821741104, "l_a": 2.0606938153505325, "l_c": 1.9986909180879593, "z_loss": 36.62982702255249, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1390039026737213, "tokens_per_second": 1465931.2615181715, "eta_seconds": 11387.16805500984, "data_wait_seconds": 2.4080276489257812e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 40650, "total_steps": 72479, "tokens": 21312307200, "loss": 3.183753728866577, "l_mix": 1.95896577835083, "l_a": 2.075172781944275, "l_c": 2.0074535459280014, "z_loss": 36.491395473480225, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14519032835960388, "tokens_per_second": 1405859.4345640729, "eta_seconds": 11870.008011984824, "data_wait_seconds": 1.5556812286376953e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 40660, "total_steps": 72479, "tokens": 21317550080, "loss": 3.1589654088020325, "l_mix": 1.943097934126854, "l_a": 2.057515263557434, "l_c": 1.9953762888908386, "z_loss": 37.02160358428955, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1651643067598343, "tokens_per_second": 1372747.4713541097, "eta_seconds": 12152.50453569889, "data_wait_seconds": 2.2739171981811523e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 40670, "total_steps": 72479, "tokens": 21322792960, "loss": 4.721113324165344, "l_mix": 2.9285159707069397, "l_a": 3.007942795753479, "l_c": 2.9673816859722137, "z_loss": 34.30274963378906, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13663460314273834, "tokens_per_second": 1406296.108183508, "eta_seconds": 11858.865920877457, "data_wait_seconds": 2.1904706954956055e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 40680, "total_steps": 72479, "tokens": 21328035840, "loss": 4.735311329364777, "l_mix": 2.937016636133194, "l_a": 3.0140936374664307, "l_c": 2.980221778154373, "z_loss": 34.27977800369263, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1333065927028656, "tokens_per_second": 1394297.4001428268, "eta_seconds": 11957.157856202126, "data_wait_seconds": 3.3855438232421875e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 40690, "total_steps": 72479, "tokens": 21333278720, "loss": 3.1626831591129303, "l_mix": 1.9467584192752838, "l_a": 2.059729039669037, "l_c": 1.9933534562587738, "z_loss": 36.09954118728638, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20134419202804565, "tokens_per_second": 1430783.8814056017, "eta_seconds": 11648.573518753052, "data_wait_seconds": 1.633167266845703e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 40700, "total_steps": 72479, "tokens": 21338521600, "loss": 3.1769035160541534, "l_mix": 1.954245612025261, "l_a": 2.071199804544449, "l_c": 2.0043262392282486, "z_loss": 36.65767765045166, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12942944467067719, "tokens_per_second": 1432286.7734450458, "eta_seconds": 11632.6902271986, "data_wait_seconds": 1.919269561767578e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.21024786, "gate_a2b/L1_absmax": 0.74136072, "gate_a2b/L1_rms": 0.22317499, "gate_a2b/L3_absmean": 0.21058212, "gate_a2b/L3_absmax": 0.59447497, "gate_a2b/L3_rms": 0.2167376, "gate_a2b/L5_absmean": 0.32839465, "gate_a2b/L5_absmax": 0.71451277, "gate_a2b/L5_rms": 0.33423284, "gate_c2b/L1_absmean": 0.017706, "gate_c2b/L1_absmax": 0.63673478, "gate_c2b/L1_rms": 0.0682333, "gate_c2b/L3_absmean": 0.20245104, "gate_c2b/L3_absmax": 0.46394369, "gate_c2b/L3_rms": 0.20667771, "gate_c2b/L5_absmean": 0.57148665, "gate_c2b/L5_absmax": 0.80171067, "gate_c2b/L5_rms": 0.58401269, "gate_b2a/L7_absmean": 0.05345097, "gate_b2a/L7_absmax": 0.3286562, "gate_b2a/L7_rms": 0.05905932, "gate_b2a/L11_absmean": 0.08577465, "gate_b2a/L11_absmax": 0.59318048, "gate_b2a/L11_rms": 0.09413423, "gate_b2a/L14_absmean": 0.21893212, "gate_b2a/L14_absmax": 0.78117859, "gate_b2a/L14_rms": 0.2270219, "gate_b2c/L3_absmean": 0.03956657, "gate_b2c/L3_absmax": 0.60048151, "gate_b2c/L3_rms": 0.05166842, "gate_b2c/L5_absmean": 0.04336977, "gate_b2c/L5_absmax": 0.61839563, "gate_b2c/L5_rms": 0.05435136, "gate_b2c/L6_absmean": 0.06235167, "gate_b2c/L6_absmax": 0.42059565, "gate_b2c/L6_rms": 0.0699567}, "streams": {"cos_a2b/round0": 0.372552, "cos_c2b/round0": 0.275459, "cos_b2a/round0": 0.129482, "cos_b2c/round0": 0.106628, "rms_a/round0": 4.885244, "rms_b/round0": 8.906797, "rms_c/round0": 3.584155, "cos_a2b/round1": 0.418624, "cos_c2b/round1": 0.295927, "cos_b2a/round1": 0.234691, "cos_b2c/round1": 0.120193, "rms_a/round1": 9.409704, "rms_b/round1": 20.830957, "rms_c/round1": 7.454845, "cos_a2b/round2": 0.521669, "cos_c2b/round2": 0.514468, "cos_b2a/round2": 0.08288, "cos_b2c/round2": 0.162549, "rms_a/round2": 24.042686, "rms_b/round2": 52.646729, "rms_c/round2": 14.023797}, "alpha_mean": 0.4108388763742852, "alpha_hist": [0.03957, 0.16238, 0.19029, 0.15295, 0.11526, 0.09918, 0.09389, 0.08998, 0.05298, 0.00351], "alpha_by_type": {"special": {"share": 0.00105, "mean": 0.59867, "std": 0.11795}, "whitespace": {"share": 0.04083, "mean": 0.59392, "std": 0.17441}, "digit": {"share": 0.01421, "mean": 0.33099, "std": 0.1545}, "word_start": {"share": 0.54619, "mean": 0.42511, "std": 0.217}, "word_cont": {"share": 0.26778, "mean": 0.26413, "std": 0.14024}, "punct": {"share": 0.12994, "mean": 0.60287, "std": 0.19801}}, "alpha_p05": 0.11839, "alpha_p50": 0.3921, "alpha_p95": 0.80837}
{"step": 40710, "total_steps": 72479, "tokens": 21343764480, "loss": 4.310220301151276, "l_mix": 2.6694142520427704, "l_a": 2.7627972662448883, "l_c": 2.7065560817718506, "z_loss": 34.80624723434448, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17503419518470764, "tokens_per_second": 1412174.1971030121, "eta_seconds": 11794.65359597206, "data_wait_seconds": 1.8209218978881836e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 40720, "total_steps": 72479, "tokens": 21349007360, "loss": 4.272353321313858, "l_mix": 2.646220773458481, "l_a": 2.736653596162796, "l_c": 2.6837879717350006, "z_loss": 35.406689167022705, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13723288476467133, "tokens_per_second": 1391226.4176817378, "eta_seconds": 11968.477869868279, "data_wait_seconds": 1.6123056411743164e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 40730, "total_steps": 72479, "tokens": 21354250240, "loss": 4.146007776260376, "l_mix": 2.5666341185569763, "l_a": 2.660962998867035, "l_c": 2.6036161184310913, "z_loss": 35.46092939376831, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13369816541671753, "tokens_per_second": 1439665.7808038737, "eta_seconds": 11562.141667842865, "data_wait_seconds": 1.9609928131103516e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 40740, "total_steps": 72479, "tokens": 21359493120, "loss": 4.237943947315216, "l_mix": 2.6242157220840454, "l_a": 2.7168813347816467, "l_c": 2.6622124016284943, "z_loss": 34.66600275039673, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1534101665019989, "tokens_per_second": 1435575.653254763, "eta_seconds": 11591.431489014625, "data_wait_seconds": 1.6957521438598633e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 40750, "total_steps": 72479, "tokens": 21364736000, "loss": 3.223995089530945, "l_mix": 1.9834009259939194, "l_a": 2.0988757610321045, "l_c": 2.0364379584789276, "z_loss": 35.91338872909546, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15071934461593628, "tokens_per_second": 1441073.594078174, "eta_seconds": 11543.570030260087, "data_wait_seconds": 2.1725893020629883e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 40760, "total_steps": 72479, "tokens": 21369978880, "loss": 3.1452538073062897, "l_mix": 1.936187744140625, "l_a": 2.0475477278232574, "l_c": 1.9826722890138626, "z_loss": 36.58065223693848, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16317878663539886, "tokens_per_second": 1433868.272620402, "eta_seconds": 11597.92108490467, "data_wait_seconds": 1.9550323486328125e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 40770, "total_steps": 72479, "tokens": 21375221760, "loss": 3.1672639548778534, "l_mix": 1.9480703622102737, "l_a": 2.063827097415924, "l_c": 2.0001514852046967, "z_loss": 37.42159366607666, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16298189759254456, "tokens_per_second": 1433199.7207877522, "eta_seconds": 11599.673060822488, "data_wait_seconds": 3.421306610107422e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 40780, "total_steps": 72479, "tokens": 21380464640, "loss": 3.2281275391578674, "l_mix": 1.9860346019268036, "l_a": 2.101267784833908, "l_c": 2.0390418022871017, "z_loss": 35.53458309173584, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19584763050079346, "tokens_per_second": 1426902.5819019794, "eta_seconds": 11647.189880228043, "data_wait_seconds": 1.9222497940063477e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 40790, "total_steps": 72479, "tokens": 21385707520, "loss": 3.233801305294037, "l_mix": 1.9909226298332214, "l_a": 2.1054300516843796, "l_c": 2.037498712539673, "z_loss": 35.419522285461426, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17567718029022217, "tokens_per_second": 1406974.7174320468, "eta_seconds": 11808.4299782753, "data_wait_seconds": 2.893805503845215e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 40800, "total_steps": 72479, "tokens": 21390950400, "loss": 3.177911698818207, "l_mix": 1.9551773369312286, "l_a": 2.0693731009960175, "l_c": 2.006408080458641, "z_loss": 37.70998954772949, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19335776567459106, "tokens_per_second": 1417548.4815805415, "eta_seconds": 11716.650095438958, "data_wait_seconds": 2.816319465637207e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.20858698, "gate_a2b/L1_absmax": 0.74203467, "gate_a2b/L1_rms": 0.22165817, "gate_a2b/L3_absmean": 0.2093354, "gate_a2b/L3_absmax": 0.59216005, "gate_a2b/L3_rms": 0.21553341, "gate_a2b/L5_absmean": 0.32720542, "gate_a2b/L5_absmax": 0.71522641, "gate_a2b/L5_rms": 0.33308953, "gate_c2b/L1_absmean": 0.01731717, "gate_c2b/L1_absmax": 0.6392808, "gate_c2b/L1_rms": 0.06833948, "gate_c2b/L3_absmean": 0.20266628, "gate_c2b/L3_absmax": 0.46232507, "gate_c2b/L3_rms": 0.20689444, "gate_c2b/L5_absmean": 0.57206076, "gate_c2b/L5_absmax": 0.80240017, "gate_c2b/L5_rms": 0.58473265, "gate_b2a/L7_absmean": 0.05318663, "gate_b2a/L7_absmax": 0.32856333, "gate_b2a/L7_rms": 0.05875244, "gate_b2a/L11_absmean": 0.08506624, "gate_b2a/L11_absmax": 0.59458613, "gate_b2a/L11_rms": 0.09355608, "gate_b2a/L14_absmean": 0.21856138, "gate_b2a/L14_absmax": 0.79012179, "gate_b2a/L14_rms": 0.22683266, "gate_b2c/L3_absmean": 0.03940933, "gate_b2c/L3_absmax": 0.60421163, "gate_b2c/L3_rms": 0.05151153, "gate_b2c/L5_absmean": 0.04303142, "gate_b2c/L5_absmax": 0.6228919, "gate_b2c/L5_rms": 0.05406812, "gate_b2c/L6_absmean": 0.06191197, "gate_b2c/L6_absmax": 0.42586139, "gate_b2c/L6_rms": 0.06963521}, "streams": {"cos_a2b/round0": 0.373257, "cos_c2b/round0": 0.281428, "cos_b2a/round0": 0.137816, "cos_b2c/round0": 0.108586, "rms_a/round0": 4.909532, "rms_b/round0": 8.860276, "rms_c/round0": 3.599377, "cos_a2b/round1": 0.418364, "cos_c2b/round1": 0.303534, "cos_b2a/round1": 0.235883, "cos_b2c/round1": 0.122618, "rms_a/round1": 9.461128, "rms_b/round1": 20.76144, "rms_c/round1": 7.459698, "cos_a2b/round2": 0.526387, "cos_c2b/round2": 0.512769, "cos_b2a/round2": 0.084939, "cos_b2c/round2": 0.166389, "rms_a/round2": 24.131369, "rms_b/round2": 52.803875, "rms_c/round2": 14.025504}, "alpha_mean": 0.4165027475540626, "alpha_hist": [0.0331, 0.15463, 0.18526, 0.15181, 0.12125, 0.10966, 0.10764, 0.09155, 0.04309, 0.00201], "alpha_by_type": {"special": {"share": 0.00108, "mean": 0.63784, "std": 0.11181}, "whitespace": {"share": 0.03674, "mean": 0.60944, "std": 0.16747}, "digit": {"share": 0.0101, "mean": 0.36113, "std": 0.14449}, "word_start": {"share": 0.57912, "mean": 0.4148, "std": 0.21024}, "word_cont": {"share": 0.24852, "mean": 0.28597, "std": 0.14299}, "punct": {"share": 0.12444, "mean": 0.63073, "std": 0.17506}}, "alpha_p05": 0.11086, "alpha_p50": 0.37731, "alpha_p95": 0.77356}
{"step": 40810, "total_steps": 72479, "tokens": 21396193280, "loss": 4.18985515832901, "l_mix": 2.594101160764694, "l_a": 2.6855717599391937, "l_c": 2.633608102798462, "z_loss": 35.72508382797241, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12498654425144196, "tokens_per_second": 1425853.85830223, "eta_seconds": 11644.725422120095, "data_wait_seconds": 2.0742416381835938e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 40820, "total_steps": 72479, "tokens": 21401436160, "loss": 4.173844814300537, "l_mix": 2.584637999534607, "l_a": 2.67496195435524, "l_c": 2.622393846511841, "z_loss": 34.7752571105957, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11266367882490158, "tokens_per_second": 1394097.189641365, "eta_seconds": 11906.2242685318, "data_wait_seconds": 1.9431114196777344e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 40830, "total_steps": 72479, "tokens": 21406679040, "loss": 4.159777283668518, "l_mix": 2.574875295162201, "l_a": 2.669834613800049, "l_c": 2.613171875476837, "z_loss": 35.85590171813965, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16330038011074066, "tokens_per_second": 1453667.086358414, "eta_seconds": 11414.71184682846, "data_wait_seconds": 1.52587890625e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 40840, "total_steps": 72479, "tokens": 21411921920, "loss": 4.229501247406006, "l_mix": 2.6191802322864532, "l_a": 2.7123211324214935, "l_c": 2.6554153859615326, "z_loss": 35.36146688461304, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1384052038192749, "tokens_per_second": 1394544.184547168, "eta_seconds": 11894.888821601868, "data_wait_seconds": 1.5228986740112305e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 40850, "total_steps": 72479, "tokens": 21417164800, "loss": 4.080168068408966, "l_mix": 2.525535970926285, "l_a": 2.616198629140854, "l_c": 2.5659082531929016, "z_loss": 35.4148473739624, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20851069688796997, "tokens_per_second": 1414888.6895560508, "eta_seconds": 11720.148216891288, "data_wait_seconds": 2.9295682907104492e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 40860, "total_steps": 72479, "tokens": 21422407680, "loss": 3.2398775219917297, "l_mix": 1.9953585267066956, "l_a": 2.1067409813404083, "l_c": 2.041655510663986, "z_loss": 35.35829257965088, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.24010565876960754, "tokens_per_second": 1276965.4509019526, "eta_seconds": 12981.919174313545, "data_wait_seconds": 2.1725893020629883e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 40870, "total_steps": 72479, "tokens": 21427650560, "loss": 3.1998123228549957, "l_mix": 1.9684437960386276, "l_a": 2.0819997787475586, "l_c": 2.022561654448509, "z_loss": 37.45791530609131, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17543593049049377, "tokens_per_second": 1403520.2107338738, "eta_seconds": 11807.610083031654, "data_wait_seconds": 1.558661460876465e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 40880, "total_steps": 72479, "tokens": 21432893440, "loss": 3.161557674407959, "l_mix": 1.9456554502248764, "l_a": 2.0560204088687897, "l_c": 1.996986597776413, "z_loss": 36.1121039390564, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15244406461715698, "tokens_per_second": 1464746.7400750802, "eta_seconds": 11310.471673178674, "data_wait_seconds": 1.952052116394043e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 40890, "total_steps": 72479, "tokens": 21438136320, "loss": 3.190147876739502, "l_mix": 1.962249293923378, "l_a": 2.0794832706451416, "l_c": 2.013512074947357, "z_loss": 36.485488414764404, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1494111567735672, "tokens_per_second": 1413993.0042821865, "eta_seconds": 11712.740856456758, "data_wait_seconds": 2.4616718292236328e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 40900, "total_steps": 72479, "tokens": 21443379200, "loss": 3.1802395582199097, "l_mix": 1.9565019458532333, "l_a": 2.07343327999115, "l_c": 2.0056918561458588, "z_loss": 36.83220672607422, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12379226088523865, "tokens_per_second": 1441001.3533421801, "eta_seconds": 11489.57335369587, "data_wait_seconds": 2.232193946838379e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.20882522, "gate_a2b/L1_absmax": 0.74284047, "gate_a2b/L1_rms": 0.22169796, "gate_a2b/L3_absmean": 0.20918207, "gate_a2b/L3_absmax": 0.59166932, "gate_a2b/L3_rms": 0.21541564, "gate_a2b/L5_absmean": 0.32760182, "gate_a2b/L5_absmax": 0.71901518, "gate_a2b/L5_rms": 0.33346406, "gate_c2b/L1_absmean": 0.02043058, "gate_c2b/L1_absmax": 0.63655812, "gate_c2b/L1_rms": 0.0681985, "gate_c2b/L3_absmean": 0.20249581, "gate_c2b/L3_absmax": 0.46451086, "gate_c2b/L3_rms": 0.20674159, "gate_c2b/L5_absmean": 0.57105196, "gate_c2b/L5_absmax": 0.80477327, "gate_c2b/L5_rms": 0.58363336, "gate_b2a/L7_absmean": 0.05325206, "gate_b2a/L7_absmax": 0.3278302, "gate_b2a/L7_rms": 0.05877405, "gate_b2a/L11_absmean": 0.08569402, "gate_b2a/L11_absmax": 0.59122348, "gate_b2a/L11_rms": 0.09401741, "gate_b2a/L14_absmean": 0.21967626, "gate_b2a/L14_absmax": 0.78610927, "gate_b2a/L14_rms": 0.22780481, "gate_b2c/L3_absmean": 0.03975617, "gate_b2c/L3_absmax": 0.59550506, "gate_b2c/L3_rms": 0.05153873, "gate_b2c/L5_absmean": 0.04314983, "gate_b2c/L5_absmax": 0.62283057, "gate_b2c/L5_rms": 0.05420865, "gate_b2c/L6_absmean": 0.06221362, "gate_b2c/L6_absmax": 0.42435762, "gate_b2c/L6_rms": 0.06978121}, "streams": {"cos_a2b/round0": 0.372045, "cos_c2b/round0": 0.276673, "cos_b2a/round0": 0.132963, "cos_b2c/round0": 0.10517, "rms_a/round0": 4.933239, "rms_b/round0": 8.956257, "rms_c/round0": 3.6392, "cos_a2b/round1": 0.415466, "cos_c2b/round1": 0.30123, "cos_b2a/round1": 0.233806, "cos_b2c/round1": 0.118643, "rms_a/round1": 9.417019, "rms_b/round1": 20.838545, "rms_c/round1": 7.492935, "cos_a2b/round2": 0.523287, "cos_c2b/round2": 0.513625, "cos_b2a/round2": 0.083529, "cos_b2c/round2": 0.16153, "rms_a/round2": 24.14534, "rms_b/round2": 52.759266, "rms_c/round2": 13.920019}, "alpha_mean": 0.41389670113419186, "alpha_hist": [0.03273, 0.14572, 0.19109, 0.15985, 0.12679, 0.11105, 0.10345, 0.0891, 0.03877, 0.00145], "alpha_by_type": {"special": {"share": 0.00107, "mean": 0.58121, "std": 0.10259}, "whitespace": {"share": 0.04106, "mean": 0.59393, "std": 0.16231}, "digit": {"share": 0.01236, "mean": 0.39635, "std": 0.17632}, "word_start": {"share": 0.55147, "mean": 0.41543, "std": 0.20458}, "word_cont": {"share": 0.25722, "mean": 0.27958, "std": 0.14117}, "punct": {"share": 0.13683, "mean": 0.60647, "std": 0.181}}, "alpha_p05": 0.13162, "alpha_p50": 0.37114, "alpha_p95": 0.78266}
{"step": 40910, "total_steps": 72479, "tokens": 21448622080, "loss": 3.13207346200943, "l_mix": 1.9264667928218842, "l_a": 2.0431239008903503, "l_c": 1.9755648225545883, "z_loss": 37.369776248931885, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13170790672302246, "tokens_per_second": 1423851.0039221856, "eta_seconds": 11624.283598780632, "data_wait_seconds": 2.041459083557129e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 40920, "total_steps": 72479, "tokens": 21453864960, "loss": 4.259257912635803, "l_mix": 2.637438118457794, "l_a": 2.731892913579941, "l_c": 2.6741726398468018, "z_loss": 35.93615102767944, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19775737822055817, "tokens_per_second": 1417583.4805709315, "eta_seconds": 11671.979265260696, "data_wait_seconds": 1.800060272216797e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 40930, "total_steps": 72479, "tokens": 21459107840, "loss": 4.222668826580048, "l_mix": 2.6154424846172333, "l_a": 2.703615814447403, "l_c": 2.6538048088550568, "z_loss": 34.490394115448, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20092430710792542, "tokens_per_second": 1379328.0463740365, "eta_seconds": 11991.898631715774, "data_wait_seconds": 1.8805265426635742e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 40940, "total_steps": 72479, "tokens": 21464350720, "loss": 4.185536026954651, "l_mix": 2.591295510530472, "l_a": 2.6831976175308228, "l_c": 2.6309372186660767, "z_loss": 34.31897258758545, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20469002425670624, "tokens_per_second": 1404431.1957669524, "eta_seconds": 11773.819380998613, "data_wait_seconds": 1.7642974853515625e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 40950, "total_steps": 72479, "tokens": 21469593600, "loss": 4.276969611644745, "l_mix": 2.6494394838809967, "l_a": 2.740517556667328, "l_c": 2.6845825910568237, "z_loss": 34.501920223236084, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16144075989723206, "tokens_per_second": 1403601.2846562662, "eta_seconds": 11777.045613098144, "data_wait_seconds": 1.7702579498291016e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 40960, "total_steps": 72479, "tokens": 21474836480, "loss": 4.2933319211006165, "l_mix": 2.659053236246109, "l_a": 2.7500956654548645, "l_c": 2.697499603033066, "z_loss": 35.52990770339966, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.10987009108066559, "tokens_per_second": 1365968.8490042074, "eta_seconds": 12097.664953374862, "data_wait_seconds": 2.2530555725097656e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 40970, "total_steps": 72479, "tokens": 21480079360, "loss": 3.1636262238025665, "l_mix": 1.9469636231660843, "l_a": 2.0575734972953796, "l_c": 1.9979684799909592, "z_loss": 36.271148681640625, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1760522723197937, "tokens_per_second": 1433451.5927421313, "eta_seconds": 11524.484451127053, "data_wait_seconds": 1.773238182067871e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 40980, "total_steps": 72479, "tokens": 21485322240, "loss": 3.1900071501731873, "l_mix": 1.9628095924854279, "l_a": 2.0789740085601807, "l_c": 2.0116841942071915, "z_loss": 36.079397201538086, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1678420454263687, "tokens_per_second": 1426898.3228262644, "eta_seconds": 11573.738259983063, "data_wait_seconds": 2.0384788513183594e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 40990, "total_steps": 72479, "tokens": 21490565120, "loss": 3.2041001319885254, "l_mix": 1.9718733131885529, "l_a": 2.0868201851844788, "l_c": 2.0206021070480347, "z_loss": 36.512739181518555, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18764150142669678, "tokens_per_second": 1333827.1022624539, "eta_seconds": 12377.39494421482, "data_wait_seconds": 6.0230493545532227e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 41000, "total_steps": 72479, "tokens": 21495808000, "loss": 3.1861520409584045, "l_mix": 1.9615025073289871, "l_a": 2.0703281611204147, "l_c": 2.0118367224931717, "z_loss": 35.02570867538452, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.24100318551063538, "tokens_per_second": 1409399.6383489291, "eta_seconds": 11709.994456458093, "data_wait_seconds": 2.1010637283325195e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.20883226, "gate_a2b/L1_absmax": 0.74279368, "gate_a2b/L1_rms": 0.22167775, "gate_a2b/L3_absmean": 0.20891581, "gate_a2b/L3_absmax": 0.59204459, "gate_a2b/L3_rms": 0.21506354, "gate_a2b/L5_absmean": 0.32568437, "gate_a2b/L5_absmax": 0.71765792, "gate_a2b/L5_rms": 0.33160454, "gate_c2b/L1_absmean": 0.01765342, "gate_c2b/L1_absmax": 0.63723183, "gate_c2b/L1_rms": 0.0679554, "gate_c2b/L3_absmean": 0.20259295, "gate_c2b/L3_absmax": 0.45936483, "gate_c2b/L3_rms": 0.20675965, "gate_c2b/L5_absmean": 0.57157391, "gate_c2b/L5_absmax": 0.80306554, "gate_c2b/L5_rms": 0.58417755, "gate_b2a/L7_absmean": 0.05290165, "gate_b2a/L7_absmax": 0.32712755, "gate_b2a/L7_rms": 0.05843841, "gate_b2a/L11_absmean": 0.08516522, "gate_b2a/L11_absmax": 0.59110028, "gate_b2a/L11_rms": 0.09349073, "gate_b2a/L14_absmean": 0.21943721, "gate_b2a/L14_absmax": 0.7861113, "gate_b2a/L14_rms": 0.22750324, "gate_b2c/L3_absmean": 0.03925204, "gate_b2c/L3_absmax": 0.59562296, "gate_b2c/L3_rms": 0.05112273, "gate_b2c/L5_absmean": 0.04313786, "gate_b2c/L5_absmax": 0.61799073, "gate_b2c/L5_rms": 0.0540503, "gate_b2c/L6_absmean": 0.06183126, "gate_b2c/L6_absmax": 0.42259187, "gate_b2c/L6_rms": 0.06940007}, "streams": {"cos_a2b/round0": 0.371679, "cos_c2b/round0": 0.273721, "cos_b2a/round0": 0.136678, "cos_b2c/round0": 0.105958, "rms_a/round0": 4.952788, "rms_b/round0": 9.061537, "rms_c/round0": 3.655242, "cos_a2b/round1": 0.420273, "cos_c2b/round1": 0.305165, "cos_b2a/round1": 0.242754, "cos_b2c/round1": 0.124601, "rms_a/round1": 9.536342, "rms_b/round1": 21.197212, "rms_c/round1": 7.628943, "cos_a2b/round2": 0.526554, "cos_c2b/round2": 0.509548, "cos_b2a/round2": 0.085613, "cos_b2c/round2": 0.169671, "rms_a/round2": 24.524004, "rms_b/round2": 53.256252, "rms_c/round2": 14.300466}, "alpha_mean": 0.41375223565282226, "alpha_hist": [0.02986, 0.15065, 0.19908, 0.15529, 0.11945, 0.10927, 0.10307, 0.08961, 0.04132, 0.0024], "alpha_by_type": {"special": {"share": 0.00111, "mean": 0.60987, "std": 0.11092}, "whitespace": {"share": 0.03484, "mean": 0.62105, "std": 0.16147}, "digit": {"share": 0.00848, "mean": 0.36627, "std": 0.17225}, "word_start": {"share": 0.57599, "mean": 0.42109, "std": 0.20465}, "word_cont": {"share": 0.26535, "mean": 0.27854, "std": 0.14134}, "punct": {"share": 0.11423, "mean": 0.62923, "std": 0.17955}}, "alpha_p05": 0.1341, "alpha_p50": 0.38353, "alpha_p95": 0.77764}
{"step": 41010, "total_steps": 72479, "tokens": 21501050880, "loss": 3.2092041075229645, "l_mix": 1.9747983515262604, "l_a": 2.0935516953468323, "l_c": 2.021133989095688, "z_loss": 35.2168755531311, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1819019466638565, "tokens_per_second": 1289743.9609958336, "eta_seconds": 12792.321244335175, "data_wait_seconds": 2.0831823348999023e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 41020, "total_steps": 72479, "tokens": 21506293760, "loss": 3.1790495812892914, "l_mix": 1.9555469900369644, "l_a": 2.071919709444046, "l_c": 2.006422221660614, "z_loss": 37.08085012435913, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14847005903720856, "tokens_per_second": 1428473.8769722183, "eta_seconds": 11546.291785860061, "data_wait_seconds": 1.677870750427246e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 41030, "total_steps": 72479, "tokens": 21511536640, "loss": 4.253211557865143, "l_mix": 2.634209245443344, "l_a": 2.723389446735382, "l_c": 2.67328479886055, "z_loss": 34.89897346496582, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15473471581935883, "tokens_per_second": 1388377.5250585603, "eta_seconds": 11875.972503447532, "data_wait_seconds": 1.8924474716186523e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 41040, "total_steps": 72479, "tokens": 21516779520, "loss": 4.143627822399139, "l_mix": 2.5656435191631317, "l_a": 2.6580814123153687, "l_c": 2.6018658876419067, "z_loss": 35.44335746765137, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12262340635061264, "tokens_per_second": 1452245.1218235197, "eta_seconds": 11350.074573707581, "data_wait_seconds": 1.8894672393798828e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 41050, "total_steps": 72479, "tokens": 21522022400, "loss": 4.144255071878433, "l_mix": 2.5661578476428986, "l_a": 2.6549828946590424, "l_c": 2.605340749025345, "z_loss": 34.72726058959961, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1533931940793991, "tokens_per_second": 1456161.368488498, "eta_seconds": 11315.948842334747, "data_wait_seconds": 1.6957521438598633e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 41060, "total_steps": 72479, "tokens": 21527265280, "loss": 4.205624163150787, "l_mix": 2.604417860507965, "l_a": 2.695622205734253, "l_c": 2.6417316198349, "z_loss": 35.32216215133667, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18915489315986633, "tokens_per_second": 1389318.3684154097, "eta_seconds": 11856.609000849725, "data_wait_seconds": 1.6629695892333984e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 41070, "total_steps": 72479, "tokens": 21532508160, "loss": 3.22099769115448, "l_mix": 1.9816855937242508, "l_a": 2.0997361838817596, "l_c": 2.0313041359186172, "z_loss": 35.8219952583313, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18216727674007416, "tokens_per_second": 1413689.8474111846, "eta_seconds": 11648.496890711784, "data_wait_seconds": 4.151463508605957e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 41080, "total_steps": 72479, "tokens": 21537751040, "loss": 4.799597680568695, "l_mix": 2.9774765074253082, "l_a": 3.0574651658535004, "l_c": 3.016271948814392, "z_loss": 34.69848918914795, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16638894379138947, "tokens_per_second": 1378910.465043453, "eta_seconds": 11938.49733490944, "data_wait_seconds": 1.946091651916504e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 41090, "total_steps": 72479, "tokens": 21542993920, "loss": 3.192717969417572, "l_mix": 1.9648310840129852, "l_a": 2.0787383317947388, "l_c": 2.0142180770635605, "z_loss": 35.919150829315186, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1666557639837265, "tokens_per_second": 1430412.6287330107, "eta_seconds": 11504.985136055946, "data_wait_seconds": 2.1398067474365234e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 41100, "total_steps": 72479, "tokens": 21548236800, "loss": 3.1649347841739655, "l_mix": 1.947242185473442, "l_a": 2.0614993572235107, "l_c": 1.9974757581949234, "z_loss": 36.22294282913208, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16744597256183624, "tokens_per_second": 1387821.7409171846, "eta_seconds": 11854.284067583083, "data_wait_seconds": 2.5987625122070312e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.20777006, "gate_a2b/L1_absmax": 0.73992836, "gate_a2b/L1_rms": 0.22055964, "gate_a2b/L3_absmean": 0.20831995, "gate_a2b/L3_absmax": 0.58984005, "gate_a2b/L3_rms": 0.21450485, "gate_a2b/L5_absmean": 0.32470599, "gate_a2b/L5_absmax": 0.71695429, "gate_a2b/L5_rms": 0.33060539, "gate_c2b/L1_absmean": 0.01777131, "gate_c2b/L1_absmax": 0.63541359, "gate_c2b/L1_rms": 0.06781661, "gate_c2b/L3_absmean": 0.20184503, "gate_c2b/L3_absmax": 0.46377569, "gate_c2b/L3_rms": 0.20606114, "gate_c2b/L5_absmean": 0.57125539, "gate_c2b/L5_absmax": 0.80269355, "gate_c2b/L5_rms": 0.58378053, "gate_b2a/L7_absmean": 0.05292913, "gate_b2a/L7_absmax": 0.3259846, "gate_b2a/L7_rms": 0.05846668, "gate_b2a/L11_absmean": 0.08517706, "gate_b2a/L11_absmax": 0.59148526, "gate_b2a/L11_rms": 0.09355302, "gate_b2a/L14_absmean": 0.21903376, "gate_b2a/L14_absmax": 0.7830804, "gate_b2a/L14_rms": 0.22710316, "gate_b2c/L3_absmean": 0.03948268, "gate_b2c/L3_absmax": 0.59432584, "gate_b2c/L3_rms": 0.05121827, "gate_b2c/L5_absmean": 0.04295942, "gate_b2c/L5_absmax": 0.61439681, "gate_b2c/L5_rms": 0.05380118, "gate_b2c/L6_absmean": 0.06183308, "gate_b2c/L6_absmax": 0.41944727, "gate_b2c/L6_rms": 0.06934622}, "streams": {"cos_a2b/round0": 0.369957, "cos_c2b/round0": 0.272012, "cos_b2a/round0": 0.13646, "cos_b2c/round0": 0.104543, "rms_a/round0": 4.953917, "rms_b/round0": 9.03859, "rms_c/round0": 3.652425, "cos_a2b/round1": 0.420114, "cos_c2b/round1": 0.305354, "cos_b2a/round1": 0.238837, "cos_b2c/round1": 0.121951, "rms_a/round1": 9.572989, "rms_b/round1": 21.070374, "rms_c/round1": 7.593268, "cos_a2b/round2": 0.530456, "cos_c2b/round2": 0.508904, "cos_b2a/round2": 0.085118, "cos_b2c/round2": 0.164314, "rms_a/round2": 24.754936, "rms_b/round2": 53.180042, "rms_c/round2": 14.21521}, "alpha_mean": 0.42164760815616376, "alpha_hist": [0.03078, 0.14868, 0.17947, 0.15402, 0.12567, 0.11365, 0.10954, 0.09204, 0.04373, 0.00241], "alpha_by_type": {"special": {"share": 0.00116, "mean": 0.60029, "std": 0.10136}, "whitespace": {"share": 0.03293, "mean": 0.62806, "std": 0.17042}, "digit": {"share": 0.0098, "mean": 0.41034, "std": 0.16553}, "word_start": {"share": 0.58403, "mean": 0.42918, "std": 0.20538}, "word_cont": {"share": 0.25507, "mean": 0.28075, "std": 0.14628}, "punct": {"share": 0.11702, "mean": 0.63228, "std": 0.17724}}, "alpha_p05": 0.11757, "alpha_p50": 0.37433, "alpha_p95": 0.79214}
{"step": 41110, "total_steps": 72479, "tokens": 21553479680, "loss": 3.1310738623142242, "l_mix": 1.926110029220581, "l_a": 2.042231395840645, "l_c": 1.9743143916130066, "z_loss": 37.578036308288574, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1661519855260849, "tokens_per_second": 1472501.085649708, "eta_seconds": 11169.017416882514, "data_wait_seconds": 2.1398067474365234e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 41120, "total_steps": 72479, "tokens": 21558722560, "loss": 3.171547383069992, "l_mix": 1.951659768819809, "l_a": 2.0633672177791595, "l_c": 2.0029247254133224, "z_loss": 37.3909478187561, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15240585803985596, "tokens_per_second": 1371442.305206112, "eta_seconds": 11988.21658744812, "data_wait_seconds": 2.3186206817626953e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 41130, "total_steps": 72479, "tokens": 21563965440, "loss": 3.203176826238632, "l_mix": 1.9717595130205154, "l_a": 2.085275173187256, "l_c": 2.019448921084404, "z_loss": 36.71760416030884, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.21491917967796326, "tokens_per_second": 1430239.4930228596, "eta_seconds": 11491.714913606644, "data_wait_seconds": 1.6242265701293945e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 41140, "total_steps": 72479, "tokens": 21569208320, "loss": 4.160700678825378, "l_mix": 2.575442850589752, "l_a": 2.6708163022994995, "l_c": 2.6133761405944824, "z_loss": 35.98002576828003, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15529823303222656, "tokens_per_second": 1292448.0838687082, "eta_seconds": 12712.821379113197, "data_wait_seconds": 1.7702579498291016e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 41150, "total_steps": 72479, "tokens": 21574451200, "loss": 4.11638543009758, "l_mix": 2.549063414335251, "l_a": 2.637760639190674, "l_c": 2.5866455733776093, "z_loss": 36.04048204421997, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17372946441173553, "tokens_per_second": 1413179.0927229116, "eta_seconds": 11623.027001023293, "data_wait_seconds": 1.811981201171875e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 41160, "total_steps": 72479, "tokens": 21579694080, "loss": 4.421499848365784, "l_mix": 2.7400863468647003, "l_a": 2.827020615339279, "l_c": 2.777690500020981, "z_loss": 34.46488857269287, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20101633667945862, "tokens_per_second": 1350521.5147667434, "eta_seconds": 12158.396362042426, "data_wait_seconds": 1.6689300537109375e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 41170, "total_steps": 72479, "tokens": 21584936960, "loss": 4.231847286224365, "l_mix": 2.6217890977859497, "l_a": 2.7085041105747223, "l_c": 2.6583563685417175, "z_loss": 34.90238571166992, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15631577372550964, "tokens_per_second": 1335672.7234048445, "eta_seconds": 12289.637052822114, "data_wait_seconds": 1.615285873413086e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 41180, "total_steps": 72479, "tokens": 21590179840, "loss": 3.1971838176250458, "l_mix": 1.9678909927606583, "l_a": 2.0787429809570312, "l_c": 2.0188994109630585, "z_loss": 36.959282875061035, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16729722917079926, "tokens_per_second": 1392545.3671487141, "eta_seconds": 11783.953685903549, "data_wait_seconds": 2.7626752853393555e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 41190, "total_steps": 72479, "tokens": 21595422720, "loss": 3.1897265315055847, "l_mix": 1.9624358713626862, "l_a": 2.079158276319504, "l_c": 2.0118104964494705, "z_loss": 37.56074857711792, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18106107413768768, "tokens_per_second": 1438405.3137047703, "eta_seconds": 11404.606946110725, "data_wait_seconds": 2.9593706130981445e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 41200, "total_steps": 72479, "tokens": 21600665600, "loss": 4.7014302015304565, "l_mix": 2.915197968482971, "l_a": 2.9965222775936127, "l_c": 2.9575847685337067, "z_loss": 33.844143867492676, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1641082614660263, "tokens_per_second": 1290827.3426343142, "eta_seconds": 12704.413526391983, "data_wait_seconds": 3.269314765930176e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.20773052, "gate_a2b/L1_absmax": 0.73863435, "gate_a2b/L1_rms": 0.220397, "gate_a2b/L3_absmean": 0.20800237, "gate_a2b/L3_absmax": 0.59075505, "gate_a2b/L3_rms": 0.21420518, "gate_a2b/L5_absmean": 0.32373577, "gate_a2b/L5_absmax": 0.71693373, "gate_a2b/L5_rms": 0.32960621, "gate_c2b/L1_absmean": 0.01755222, "gate_c2b/L1_absmax": 0.63569951, "gate_c2b/L1_rms": 0.067713, "gate_c2b/L3_absmean": 0.20198984, "gate_c2b/L3_absmax": 0.46520218, "gate_c2b/L3_rms": 0.20616618, "gate_c2b/L5_absmean": 0.57029039, "gate_c2b/L5_absmax": 0.80446643, "gate_c2b/L5_rms": 0.58277881, "gate_b2a/L7_absmean": 0.05297627, "gate_b2a/L7_absmax": 0.32696843, "gate_b2a/L7_rms": 0.05854499, "gate_b2a/L11_absmean": 0.08491229, "gate_b2a/L11_absmax": 0.59229743, "gate_b2a/L11_rms": 0.09326841, "gate_b2a/L14_absmean": 0.21873519, "gate_b2a/L14_absmax": 0.78279746, "gate_b2a/L14_rms": 0.22679502, "gate_b2c/L3_absmean": 0.03920541, "gate_b2c/L3_absmax": 0.59304583, "gate_b2c/L3_rms": 0.05101851, "gate_b2c/L5_absmean": 0.04285235, "gate_b2c/L5_absmax": 0.61382622, "gate_b2c/L5_rms": 0.05371616, "gate_b2c/L6_absmean": 0.0615653, "gate_b2c/L6_absmax": 0.41725251, "gate_b2c/L6_rms": 0.06904555}, "streams": {"cos_a2b/round0": 0.370244, "cos_c2b/round0": 0.305331, "cos_b2a/round0": 0.149302, "cos_b2c/round0": 0.130349, "rms_a/round0": 4.755447, "rms_b/round0": 8.960081, "rms_c/round0": 3.541601, "cos_a2b/round1": 0.42367, "cos_c2b/round1": 0.308761, "cos_b2a/round1": 0.250944, "cos_b2c/round1": 0.121332, "rms_a/round1": 9.057428, "rms_b/round1": 19.876114, "rms_c/round1": 7.222348, "cos_a2b/round2": 0.561959, "cos_c2b/round2": 0.521761, "cos_b2a/round2": 0.104857, "cos_b2c/round2": 0.181759, "rms_a/round2": 24.082747, "rms_b/round2": 51.395653, "rms_c/round2": 13.39063}, "alpha_mean": 0.4441859761941913, "alpha_hist": [0.00725, 0.05675, 0.15912, 0.19586, 0.202, 0.17859, 0.13, 0.06384, 0.00659, 0.0], "alpha_by_type": {"special": {"share": 0.00078, "mean": 0.54336, "std": 0.1336}, "whitespace": {"share": 0.04456, "mean": 0.51167, "std": 0.1085}, "digit": {"share": 0.02457, "mean": 0.38096, "std": 0.11833}, "word_start": {"share": 0.58907, "mean": 0.46814, "std": 0.15554}, "word_cont": {"share": 0.24405, "mean": 0.32233, "std": 0.12698}, "punct": {"share": 0.09698, "mean": 0.58956, "std": 0.1478}}, "alpha_p05": 0.20434, "alpha_p50": 0.4371, "alpha_p95": 0.70253}
{"step": 41210, "total_steps": 72479, "tokens": 21605908480, "loss": 4.668894648551941, "l_mix": 2.8968438506126404, "l_a": 2.9721536338329315, "l_c": 2.9346823394298553, "z_loss": 34.767430782318115, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14253073930740356, "tokens_per_second": 1423572.4503332865, "eta_seconds": 11516.071042370797, "data_wait_seconds": 1.6510486602783203e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 41220, "total_steps": 72479, "tokens": 21611151360, "loss": 3.1612701416015625, "l_mix": 1.9452720135450363, "l_a": 2.056954190135002, "l_c": 1.9963727295398712, "z_loss": 37.46333360671997, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14598983526229858, "tokens_per_second": 1372792.719388716, "eta_seconds": 11938.23245165348, "data_wait_seconds": 3.316998481750488e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 41230, "total_steps": 72479, "tokens": 21616394240, "loss": 3.158580869436264, "l_mix": 1.9431008994579315, "l_a": 2.060698449611664, "l_c": 1.9909012764692307, "z_loss": 37.53197479248047, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15609802305698395, "tokens_per_second": 1343659.2909281226, "eta_seconds": 12193.177111649515, "data_wait_seconds": 3.0487775802612305e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 41240, "total_steps": 72479, "tokens": 21621637120, "loss": 3.1825180053710938, "l_mix": 1.9580858200788498, "l_a": 2.072427421808243, "l_c": 2.0090130418539047, "z_loss": 36.53043556213379, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1875118464231491, "tokens_per_second": 1433533.4514993955, "eta_seconds": 11425.078929877282, "data_wait_seconds": 1.621246337890625e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 41250, "total_steps": 72479, "tokens": 21626880000, "loss": 4.272858619689941, "l_mix": 2.6466298699378967, "l_a": 2.736188054084778, "l_c": 2.6845744252204895, "z_loss": 36.146607398986816, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13626831769943237, "tokens_per_second": 1438001.1325705317, "eta_seconds": 11385.936757040023, "data_wait_seconds": 1.621246337890625e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 41260, "total_steps": 72479, "tokens": 21632122880, "loss": 4.189799010753632, "l_mix": 2.594588577747345, "l_a": 2.6843963861465454, "l_c": 2.6329712569713593, "z_loss": 35.45050573348999, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1244862973690033, "tokens_per_second": 1349651.1047946662, "eta_seconds": 12127.391304206849, "data_wait_seconds": 1.7702579498291016e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 41270, "total_steps": 72479, "tokens": 21637365760, "loss": 4.168828010559082, "l_mix": 2.58168488740921, "l_a": 2.672031670808792, "l_c": 2.618445336818695, "z_loss": 35.498831272125244, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1361880898475647, "tokens_per_second": 1381102.2093708708, "eta_seconds": 11847.424528741838, "data_wait_seconds": 2.104043960571289e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 41280, "total_steps": 72479, "tokens": 21642608640, "loss": 4.346660792827606, "l_mix": 2.69382780790329, "l_a": 2.7800690829753876, "l_c": 2.7293738424777985, "z_loss": 35.18039321899414, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19323378801345825, "tokens_per_second": 1342736.6989884824, "eta_seconds": 12182.031908655166, "data_wait_seconds": 1.576542854309082e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 41290, "total_steps": 72479, "tokens": 21647851520, "loss": 3.1588867604732513, "l_mix": 1.943578913807869, "l_a": 2.057428255677223, "l_c": 1.9935976713895798, "z_loss": 37.657708168029785, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15300798416137695, "tokens_per_second": 1477398.2663845194, "eta_seconds": 11068.118058657647, "data_wait_seconds": 2.0831823348999023e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 41300, "total_steps": 72479, "tokens": 21653094400, "loss": 3.225745290517807, "l_mix": 1.9840412884950638, "l_a": 2.0993528962135315, "l_c": 2.039660304784775, "z_loss": 37.63327693939209, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17709416151046753, "tokens_per_second": 1383084.9900524598, "eta_seconds": 11819.068003463746, "data_wait_seconds": 4.2945146560668945e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.20721827, "gate_a2b/L1_absmax": 0.73825204, "gate_a2b/L1_rms": 0.22010502, "gate_a2b/L3_absmean": 0.20767736, "gate_a2b/L3_absmax": 0.58827382, "gate_a2b/L3_rms": 0.21390255, "gate_a2b/L5_absmean": 0.32300934, "gate_a2b/L5_absmax": 0.7180447, "gate_a2b/L5_rms": 0.32889041, "gate_c2b/L1_absmean": 0.01799057, "gate_c2b/L1_absmax": 0.63489264, "gate_c2b/L1_rms": 0.06771702, "gate_c2b/L3_absmean": 0.20116043, "gate_c2b/L3_absmax": 0.4522441, "gate_c2b/L3_rms": 0.20530866, "gate_c2b/L5_absmean": 0.57012266, "gate_c2b/L5_absmax": 0.80412894, "gate_c2b/L5_rms": 0.58262837, "gate_b2a/L7_absmean": 0.05274735, "gate_b2a/L7_absmax": 0.32637101, "gate_b2a/L7_rms": 0.05835433, "gate_b2a/L11_absmean": 0.08499835, "gate_b2a/L11_absmax": 0.58926356, "gate_b2a/L11_rms": 0.09333979, "gate_b2a/L14_absmean": 0.21863051, "gate_b2a/L14_absmax": 0.78608203, "gate_b2a/L14_rms": 0.22679673, "gate_b2c/L3_absmean": 0.03890276, "gate_b2c/L3_absmax": 0.59229934, "gate_b2c/L3_rms": 0.05077962, "gate_b2c/L5_absmean": 0.04280097, "gate_b2c/L5_absmax": 0.61475259, "gate_b2c/L5_rms": 0.05368757, "gate_b2c/L6_absmean": 0.06150971, "gate_b2c/L6_absmax": 0.41615981, "gate_b2c/L6_rms": 0.0689748}, "streams": {"cos_a2b/round0": 0.370166, "cos_c2b/round0": 0.271273, "cos_b2a/round0": 0.132595, "cos_b2c/round0": 0.104809, "rms_a/round0": 4.999614, "rms_b/round0": 9.119734, "rms_c/round0": 3.665586, "cos_a2b/round1": 0.417077, "cos_c2b/round1": 0.301001, "cos_b2a/round1": 0.235139, "cos_b2c/round1": 0.118733, "rms_a/round1": 9.652615, "rms_b/round1": 21.243816, "rms_c/round1": 7.646639, "cos_a2b/round2": 0.520845, "cos_c2b/round2": 0.516695, "cos_b2a/round2": 0.08365, "cos_b2c/round2": 0.162158, "rms_a/round2": 24.821285, "rms_b/round2": 53.943874, "rms_c/round2": 14.284043}, "alpha_mean": 0.42389839514536476, "alpha_hist": [0.02887, 0.13857, 0.19054, 0.15831, 0.12486, 0.11008, 0.10406, 0.09117, 0.04955, 0.004], "alpha_by_type": {"special": {"share": 0.00104, "mean": 0.65124, "std": 0.12635}, "whitespace": {"share": 0.03613, "mean": 0.61723, "std": 0.18034}, "digit": {"share": 0.01151, "mean": 0.3804, "std": 0.16671}, "word_start": {"share": 0.56342, "mean": 0.4305, "std": 0.20861}, "word_cont": {"share": 0.2627, "mean": 0.28976, "std": 0.14544}, "punct": {"share": 0.12521, "mean": 0.62195, "std": 0.18919}}, "alpha_p05": 0.12107, "alpha_p50": 0.38283, "alpha_p95": 0.80746}
{"step": 41310, "total_steps": 72479, "tokens": 21658337280, "loss": 3.0144990980625153, "l_mix": 1.8650925308465958, "l_a": 1.938321128487587, "l_c": 1.8930339813232422, "z_loss": 41.1704797744751, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1470421552658081, "tokens_per_second": 1051740.6724535865, "eta_seconds": 15537.606465172768, "data_wait_seconds": 2.3365020751953125e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 41320, "total_steps": 72479, "tokens": 21663580160, "loss": 4.867327988147736, "l_mix": 3.0203031301498413, "l_a": 3.0979616940021515, "l_c": 3.0587879419326782, "z_loss": 33.361515045166016, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13861624896526337, "tokens_per_second": 1329691.3773310396, "eta_seconds": 12285.775534462928, "data_wait_seconds": 1.6480684280395508e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 41330, "total_steps": 72479, "tokens": 21668823040, "loss": 4.684751570224762, "l_mix": 2.907097727060318, "l_a": 2.981831133365631, "l_c": 2.9436812102794647, "z_loss": 34.65022850036621, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1483335793018341, "tokens_per_second": 1422261.0643186348, "eta_seconds": 11482.45376443863, "data_wait_seconds": 1.9371509552001953e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 41340, "total_steps": 72479, "tokens": 21674065920, "loss": 4.781634032726288, "l_mix": 2.965889483690262, "l_a": 3.0455547273159027, "l_c": 3.0069272816181183, "z_loss": 33.892977714538574, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1472933292388916, "tokens_per_second": 1422153.5390598844, "eta_seconds": 11479.635344290733, "data_wait_seconds": 1.615285873413086e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 41350, "total_steps": 72479, "tokens": 21679308800, "loss": 4.779768168926239, "l_mix": 2.9660127758979797, "l_a": 3.0436118841171265, "l_c": 3.0022392570972443, "z_loss": 33.23916482925415, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16116471588611603, "tokens_per_second": 1416280.546270182, "eta_seconds": 11523.53691151142, "data_wait_seconds": 1.7195940017700195e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 41360, "total_steps": 72479, "tokens": 21684551680, "loss": 4.176120698451996, "l_mix": 2.585308253765106, "l_a": 2.6799188256263733, "l_c": 2.622789114713669, "z_loss": 35.68634271621704, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12346373498439789, "tokens_per_second": 1385295.7953548543, "eta_seconds": 11777.497864866256, "data_wait_seconds": 3.2514333724975586e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 41370, "total_steps": 72479, "tokens": 21689794560, "loss": 4.1794672310352325, "l_mix": 2.5886756777763367, "l_a": 2.679083466529846, "l_c": 2.623554974794388, "z_loss": 35.797340869903564, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16007275879383087, "tokens_per_second": 1371109.4125335284, "eta_seconds": 11895.531635117532, "data_wait_seconds": 1.5467405319213867e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 41380, "total_steps": 72479, "tokens": 21695037440, "loss": 4.169397056102753, "l_mix": 2.5810098946094513, "l_a": 2.673885315656662, "l_c": 2.620738446712494, "z_loss": 36.139042377471924, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13845594227313995, "tokens_per_second": 1179069.8366305833, "eta_seconds": 13828.555362415313, "data_wait_seconds": 2.142786979675293e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 41390, "total_steps": 72479, "tokens": 21700280320, "loss": 4.173997104167938, "l_mix": 2.5851005613803864, "l_a": 2.6743045151233673, "l_c": 2.622017115354538, "z_loss": 35.487281799316406, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16872361302375793, "tokens_per_second": 1362662.7148948738, "eta_seconds": 11961.573068547248, "data_wait_seconds": 3.984570503234863e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 41400, "total_steps": 72479, "tokens": 21705523200, "loss": 3.2151322960853577, "l_mix": 1.9785847961902618, "l_a": 2.0914317071437836, "l_c": 2.0303931683301926, "z_loss": 36.3190279006958, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13993069529533386, "tokens_per_second": 1307040.514470432, "eta_seconds": 12466.596537446974, "data_wait_seconds": 3.960728645324707e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.20635037, "gate_a2b/L1_absmax": 0.73565131, "gate_a2b/L1_rms": 0.21918651, "gate_a2b/L3_absmean": 0.2070154, "gate_a2b/L3_absmax": 0.58694404, "gate_a2b/L3_rms": 0.21315345, "gate_a2b/L5_absmean": 0.32166839, "gate_a2b/L5_absmax": 0.71785194, "gate_a2b/L5_rms": 0.32758361, "gate_c2b/L1_absmean": 0.01842809, "gate_c2b/L1_absmax": 0.63599306, "gate_c2b/L1_rms": 0.06798047, "gate_c2b/L3_absmean": 0.20088983, "gate_c2b/L3_absmax": 0.45423689, "gate_c2b/L3_rms": 0.20515525, "gate_c2b/L5_absmean": 0.56926608, "gate_c2b/L5_absmax": 0.80002952, "gate_c2b/L5_rms": 0.58179069, "gate_b2a/L7_absmean": 0.05282262, "gate_b2a/L7_absmax": 0.32608867, "gate_b2a/L7_rms": 0.05832623, "gate_b2a/L11_absmean": 0.08465225, "gate_b2a/L11_absmax": 0.59155607, "gate_b2a/L11_rms": 0.0931029, "gate_b2a/L14_absmean": 0.21859375, "gate_b2a/L14_absmax": 0.78301615, "gate_b2a/L14_rms": 0.22667742, "gate_b2c/L3_absmean": 0.03927533, "gate_b2c/L3_absmax": 0.59235591, "gate_b2c/L3_rms": 0.05093597, "gate_b2c/L5_absmean": 0.04254816, "gate_b2c/L5_absmax": 0.61408001, "gate_b2c/L5_rms": 0.05340398, "gate_b2c/L6_absmean": 0.06143967, "gate_b2c/L6_absmax": 0.4176794, "gate_b2c/L6_rms": 0.06890945}, "streams": {"cos_a2b/round0": 0.369268, "cos_c2b/round0": 0.274127, "cos_b2a/round0": 0.137729, "cos_b2c/round0": 0.10634, "rms_a/round0": 5.006682, "rms_b/round0": 9.18351, "rms_c/round0": 3.687329, "cos_a2b/round1": 0.421957, "cos_c2b/round1": 0.304952, "cos_b2a/round1": 0.241316, "cos_b2c/round1": 0.123969, "rms_a/round1": 9.700509, "rms_b/round1": 21.469673, "rms_c/round1": 7.659098, "cos_a2b/round2": 0.528941, "cos_c2b/round2": 0.510362, "cos_b2a/round2": 0.087863, "cos_b2c/round2": 0.164829, "rms_a/round2": 25.10693, "rms_b/round2": 54.302338, "rms_c/round2": 14.404993}, "alpha_mean": 0.4169596654617891, "alpha_hist": [0.03555, 0.15331, 0.18916, 0.14682, 0.1163, 0.11118, 0.108, 0.09418, 0.04276, 0.00275], "alpha_by_type": {"special": {"share": 0.00113, "mean": 0.59354, "std": 0.12552}, "whitespace": {"share": 0.02963, "mean": 0.64343, "std": 0.16531}, "digit": {"share": 0.00883, "mean": 0.44662, "std": 0.1831}, "word_start": {"share": 0.58568, "mean": 0.42968, "std": 0.20549}, "word_cont": {"share": 0.26041, "mean": 0.26532, "std": 0.1444}, "punct": {"share": 0.11432, "mean": 0.63446, "std": 0.17748}}, "alpha_p05": 0.11556, "alpha_p50": 0.38955, "alpha_p95": 0.78166}
{"step": 41410, "total_steps": 72479, "tokens": 21710766080, "loss": 3.1836479902267456, "l_mix": 1.9586558192968369, "l_a": 2.075519025325775, "l_c": 2.007787972688675, "z_loss": 37.80782747268677, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19895517826080322, "tokens_per_second": 1481432.7745864545, "eta_seconds": 10995.50661456585, "data_wait_seconds": 1.8984079360961914e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 41420, "total_steps": 72479, "tokens": 21716008960, "loss": 3.1483159363269806, "l_mix": 1.9499771445989609, "l_a": 2.0156979262828827, "l_c": 1.9787644892930984, "z_loss": 40.9542031288147, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16996106505393982, "tokens_per_second": 1435946.871932851, "eta_seconds": 11340.155621552467, "data_wait_seconds": 4.3272972106933594e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 41430, "total_steps": 72479, "tokens": 21721251840, "loss": 4.289573550224304, "l_mix": 2.656733065843582, "l_a": 2.748533844947815, "l_c": 2.6942673921585083, "z_loss": 36.24633312225342, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18085353076457977, "tokens_per_second": 1196963.7479777297, "eta_seconds": 13599.92576174736, "data_wait_seconds": 0.05602720379829407, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 41440, "total_steps": 72479, "tokens": 21726494720, "loss": 4.3333545327186584, "l_mix": 2.6847178041934967, "l_a": 2.7712405920028687, "l_c": 2.7242152988910675, "z_loss": 35.61803579330444, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15276333689689636, "tokens_per_second": 1479886.9776383971, "eta_seconds": 10996.363558769226, "data_wait_seconds": 1.8358230590820312e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 41450, "total_steps": 72479, "tokens": 21731737600, "loss": 4.277166247367859, "l_mix": 2.6492317616939545, "l_a": 2.737255275249481, "l_c": 2.6891930401325226, "z_loss": 35.56192684173584, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19502650201320648, "tokens_per_second": 1310397.5012443878, "eta_seconds": 12414.65458881855, "data_wait_seconds": 1.7136335372924805e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 41460, "total_steps": 72479, "tokens": 21736980480, "loss": 4.393957793712616, "l_mix": 2.723818212747574, "l_a": 2.809752881526947, "l_c": 2.757379025220871, "z_loss": 34.664499282836914, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1669970005750656, "tokens_per_second": 1464290.6669150852, "eta_seconds": 11106.325977110862, "data_wait_seconds": 1.8388032913208008e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 41470, "total_steps": 72479, "tokens": 21742223360, "loss": 4.4572946429252625, "l_mix": 2.7614349722862244, "l_a": 2.8505645096302032, "l_c": 2.8023006916046143, "z_loss": 35.551337242126465, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15752147138118744, "tokens_per_second": 1345137.4197503112, "eta_seconds": 12086.234724640846, "data_wait_seconds": 1.5944242477416992e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20764, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 41480, "total_steps": 72479, "tokens": 21747466240, "loss": 4.273687958717346, "l_mix": 2.6479683220386505, "l_a": 2.736485332250595, "l_c": 2.6825800836086273, "z_loss": 34.997753620147705, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1900399774312973, "tokens_per_second": 1125270.3603861535, "eta_seconds": 14443.110104155541, "data_wait_seconds": 2.8192996978759766e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 41490, "total_steps": 72479, "tokens": 21752709120, "loss": 4.346790611743927, "l_mix": 2.6935813426971436, "l_a": 2.779890328645706, "l_c": 2.7308070957660675, "z_loss": 35.73134660720825, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15382762253284454, "tokens_per_second": 1345395.5871197754, "eta_seconds": 12076.121690559386, "data_wait_seconds": 1.728534698486328e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 41500, "total_steps": 72479, "tokens": 21757952000, "loss": 3.160227805376053, "l_mix": 1.944333329796791, "l_a": 2.0559301674365997, "l_c": 1.9970512241125107, "z_loss": 37.21310806274414, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15291187167167664, "tokens_per_second": 1455473.4127412874, "eta_seconds": 11159.199343538285, "data_wait_seconds": 1.5944242477416992e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.20698412, "gate_a2b/L1_absmax": 0.73490715, "gate_a2b/L1_rms": 0.21979225, "gate_a2b/L3_absmean": 0.20652223, "gate_a2b/L3_absmax": 0.58756179, "gate_a2b/L3_rms": 0.21266602, "gate_a2b/L5_absmean": 0.32098332, "gate_a2b/L5_absmax": 0.716995, "gate_a2b/L5_rms": 0.32687479, "gate_c2b/L1_absmean": 0.0176757, "gate_c2b/L1_absmax": 0.63362485, "gate_c2b/L1_rms": 0.06755947, "gate_c2b/L3_absmean": 0.20086394, "gate_c2b/L3_absmax": 0.45490766, "gate_c2b/L3_rms": 0.20503692, "gate_c2b/L5_absmean": 0.56871313, "gate_c2b/L5_absmax": 0.79499662, "gate_c2b/L5_rms": 0.58108878, "gate_b2a/L7_absmean": 0.05276192, "gate_b2a/L7_absmax": 0.32619256, "gate_b2a/L7_rms": 0.05831642, "gate_b2a/L11_absmean": 0.08468552, "gate_b2a/L11_absmax": 0.5902521, "gate_b2a/L11_rms": 0.09306701, "gate_b2a/L14_absmean": 0.21821691, "gate_b2a/L14_absmax": 0.78175867, "gate_b2a/L14_rms": 0.22628759, "gate_b2c/L3_absmean": 0.03903542, "gate_b2c/L3_absmax": 0.59162378, "gate_b2c/L3_rms": 0.05072762, "gate_b2c/L5_absmean": 0.04242118, "gate_b2c/L5_absmax": 0.61274248, "gate_b2c/L5_rms": 0.05331529, "gate_b2c/L6_absmean": 0.06111389, "gate_b2c/L6_absmax": 0.41465646, "gate_b2c/L6_rms": 0.06851388}, "streams": {"cos_a2b/round0": 0.371272, "cos_c2b/round0": 0.273969, "cos_b2a/round0": 0.13538, "cos_b2c/round0": 0.106435, "rms_a/round0": 5.039272, "rms_b/round0": 9.174352, "rms_c/round0": 3.711385, "cos_a2b/round1": 0.417492, "cos_c2b/round1": 0.305715, "cos_b2a/round1": 0.236664, "cos_b2c/round1": 0.120798, "rms_a/round1": 9.674046, "rms_b/round1": 21.389561, "rms_c/round1": 7.700255, "cos_a2b/round2": 0.521349, "cos_c2b/round2": 0.511013, "cos_b2a/round2": 0.088565, "cos_b2c/round2": 0.164943, "rms_a/round2": 24.906815, "rms_b/round2": 54.336559, "rms_c/round2": 14.41435}, "alpha_mean": 0.415674970413491, "alpha_hist": [0.03014, 0.15723, 0.19547, 0.14975, 0.12164, 0.1024, 0.10078, 0.09033, 0.04913, 0.00313], "alpha_by_type": {"special": {"share": 0.00113, "mean": 0.58628, "std": 0.11897}, "whitespace": {"share": 0.03418, "mean": 0.63031, "std": 0.17116}, "digit": {"share": 0.01099, "mean": 0.39101, "std": 0.17001}, "word_start": {"share": 0.57564, "mean": 0.42159, "std": 0.21149}, "word_cont": {"share": 0.26523, "mean": 0.2815, "std": 0.14211}, "punct": {"share": 0.11284, "mean": 0.63659, "std": 0.18086}}, "alpha_p05": 0.12722, "alpha_p50": 0.38076, "alpha_p95": 0.79944}
{"step": 41510, "total_steps": 72479, "tokens": 21763194880, "loss": 3.1757669746875763, "l_mix": 1.9543185532093048, "l_a": 2.067490041255951, "l_c": 2.004004567861557, "z_loss": 36.15918445587158, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17887702584266663, "tokens_per_second": 1431444.6831583753, "eta_seconds": 11342.858905434608, "data_wait_seconds": 2.0951032638549805e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 41520, "total_steps": 72479, "tokens": 21768437760, "loss": 3.176609992980957, "l_mix": 1.9539749473333359, "l_a": 2.0713111758232117, "l_c": 2.004138857126236, "z_loss": 38.946204662323, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20797769725322723, "tokens_per_second": 1321142.6155980055, "eta_seconds": 12285.904640698433, "data_wait_seconds": 1.8596649169921875e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 41530, "total_steps": 72479, "tokens": 21773680640, "loss": 2.7882683277130127, "l_mix": 1.7240359038114548, "l_a": 1.7927802801132202, "l_c": 1.7546610683202744, "z_loss": 43.26025724411011, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15484915673732758, "tokens_per_second": 1376339.1190821799, "eta_seconds": 11789.383217430115, "data_wait_seconds": 2.5331974029541016e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 41540, "total_steps": 72479, "tokens": 21778923520, "loss": 4.314870059490204, "l_mix": 2.672223210334778, "l_a": 2.7690775096416473, "l_c": 2.706411749124527, "z_loss": 35.836944580078125, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2287697046995163, "tokens_per_second": 1426147.5489698804, "eta_seconds": 11373.960880637167, "data_wait_seconds": 1.9818544387817383e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 41550, "total_steps": 72479, "tokens": 21784166400, "loss": 4.353168159723282, "l_mix": 2.6977602541446686, "l_a": 2.7831921875476837, "l_c": 2.7348339557647705, "z_loss": 33.924877643585205, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17290084064006805, "tokens_per_second": 1415319.6957461694, "eta_seconds": 11457.272587060928, "data_wait_seconds": 1.52587890625e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 41560, "total_steps": 72479, "tokens": 21789409280, "loss": 4.41070693731308, "l_mix": 2.733654648065567, "l_a": 2.8192016184329987, "l_c": 2.770972430706024, "z_loss": 34.32772970199585, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1478431671857834, "tokens_per_second": 1398807.3785971757, "eta_seconds": 11588.772635912896, "data_wait_seconds": 1.7523765563964844e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 41570, "total_steps": 72479, "tokens": 21794652160, "loss": 4.407482981681824, "l_mix": 2.7307850122451782, "l_a": 2.8216632306575775, "l_c": 2.7673293352127075, "z_loss": 35.5329213142395, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17065289616584778, "tokens_per_second": 1284058.8815363979, "eta_seconds": 12620.307390117645, "data_wait_seconds": 1.895427703857422e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 41580, "total_steps": 72479, "tokens": 21799895040, "loss": 4.428446471691132, "l_mix": 2.745098888874054, "l_a": 2.828222155570984, "l_c": 2.7829361855983734, "z_loss": 35.372175216674805, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1363586038351059, "tokens_per_second": 1350828.22005232, "eta_seconds": 11992.623985433578, "data_wait_seconds": 2.619624137878418e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 41590, "total_steps": 72479, "tokens": 21805137920, "loss": 4.755808472633362, "l_mix": 2.9511638283729553, "l_a": 3.025586426258087, "l_c": 2.989895612001419, "z_loss": 33.40740633010864, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15407834947109222, "tokens_per_second": 1445654.1397811149, "eta_seconds": 11202.355796146392, "data_wait_seconds": 2.3305416107177734e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 41600, "total_steps": 72479, "tokens": 21810380800, "loss": 4.747439324855804, "l_mix": 2.944822669029236, "l_a": 3.026587426662445, "l_c": 2.9821342527866364, "z_loss": 33.438294410705566, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1410902738571167, "tokens_per_second": 1369454.6194080494, "eta_seconds": 11821.851503920556, "data_wait_seconds": 3.248453140258789e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.20626946, "gate_a2b/L1_absmax": 0.73381656, "gate_a2b/L1_rms": 0.21894602, "gate_a2b/L3_absmean": 0.20580792, "gate_a2b/L3_absmax": 0.58606613, "gate_a2b/L3_rms": 0.2119973, "gate_a2b/L5_absmean": 0.31968448, "gate_a2b/L5_absmax": 0.71809554, "gate_a2b/L5_rms": 0.32555974, "gate_c2b/L1_absmean": 0.01821383, "gate_c2b/L1_absmax": 0.63228995, "gate_c2b/L1_rms": 0.06756519, "gate_c2b/L3_absmean": 0.20070402, "gate_c2b/L3_absmax": 0.45485124, "gate_c2b/L3_rms": 0.20494124, "gate_c2b/L5_absmean": 0.56835061, "gate_c2b/L5_absmax": 0.80250633, "gate_c2b/L5_rms": 0.58080894, "gate_b2a/L7_absmean": 0.05236678, "gate_b2a/L7_absmax": 0.32515422, "gate_b2a/L7_rms": 0.05796697, "gate_b2a/L11_absmean": 0.08457226, "gate_b2a/L11_absmax": 0.59042072, "gate_b2a/L11_rms": 0.09295161, "gate_b2a/L14_absmean": 0.21753967, "gate_b2a/L14_absmax": 0.78173155, "gate_b2a/L14_rms": 0.22566307, "gate_b2c/L3_absmean": 0.03887005, "gate_b2c/L3_absmax": 0.59023267, "gate_b2c/L3_rms": 0.05057437, "gate_b2c/L5_absmean": 0.04224654, "gate_b2c/L5_absmax": 0.61411756, "gate_b2c/L5_rms": 0.05319469, "gate_b2c/L6_absmean": 0.06105088, "gate_b2c/L6_absmax": 0.41283277, "gate_b2c/L6_rms": 0.06843971}, "streams": {"cos_a2b/round0": 0.369145, "cos_c2b/round0": 0.309676, "cos_b2a/round0": 0.152008, "cos_b2c/round0": 0.132154, "rms_a/round0": 4.813327, "rms_b/round0": 9.035636, "rms_c/round0": 3.567399, "cos_a2b/round1": 0.420863, "cos_c2b/round1": 0.306192, "cos_b2a/round1": 0.250627, "cos_b2c/round1": 0.122299, "rms_a/round1": 9.141297, "rms_b/round1": 20.115313, "rms_c/round1": 7.256574, "cos_a2b/round2": 0.569686, "cos_c2b/round2": 0.509514, "cos_b2a/round2": 0.111191, "cos_b2c/round2": 0.178252, "rms_a/round2": 24.641979, "rms_b/round2": 51.980885, "rms_c/round2": 13.455743}, "alpha_mean": 0.4662122614000168, "alpha_hist": [0.00533, 0.04576, 0.14244, 0.17848, 0.19035, 0.19409, 0.15245, 0.08022, 0.01079, 9e-05], "alpha_by_type": {"special": {"share": 0.00081, "mean": 0.56073, "std": 0.13158}, "whitespace": {"share": 0.05368, "mean": 0.53998, "std": 0.11261}, "digit": {"share": 0.02394, "mean": 0.46857, "std": 0.11913}, "word_start": {"share": 0.58253, "mean": 0.48573, "std": 0.1593}, "word_cont": {"share": 0.24088, "mean": 0.34443, "std": 0.13229}, "punct": {"share": 0.09816, "mean": 0.60753, "std": 0.15129}}, "alpha_p05": 0.18156, "alpha_p50": 0.47981, "alpha_p95": 0.72836}
{"step": 41610, "total_steps": 72479, "tokens": 21815623680, "loss": 4.797888815402985, "l_mix": 2.9765281975269318, "l_a": 3.057218551635742, "l_c": 3.013982981443405, "z_loss": 32.85301876068115, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.178813174366951, "tokens_per_second": 1384167.5538518257, "eta_seconds": 11692.40402071476, "data_wait_seconds": 2.3066997528076172e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 41620, "total_steps": 72479, "tokens": 21820866560, "loss": 4.6967920660972595, "l_mix": 2.913750261068344, "l_a": 2.9904395937919617, "l_c": 2.953033000230789, "z_loss": 33.64688682556152, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14063353836536407, "tokens_per_second": 1456421.8587283087, "eta_seconds": 11108.734255146981, "data_wait_seconds": 1.8537044525146484e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 41630, "total_steps": 72479, "tokens": 21826109440, "loss": 4.735384285449982, "l_mix": 2.9367375671863556, "l_a": 3.020499736070633, "l_c": 2.974989175796509, "z_loss": 35.143250465393066, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16894103586673737, "tokens_per_second": 1354066.7697191508, "eta_seconds": 11944.58122279644, "data_wait_seconds": 3.1828880310058594e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 41640, "total_steps": 72479, "tokens": 21831352320, "loss": 3.1131838858127594, "l_mix": 1.9150724411010742, "l_a": 2.031280606985092, "l_c": 1.9624239355325699, "z_loss": 37.34648275375366, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12553739547729492, "tokens_per_second": 1477061.8585164344, "eta_seconds": 10946.40521571636, "data_wait_seconds": 2.5659799575805664e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 41650, "total_steps": 72479, "tokens": 21836595200, "loss": 4.488250434398651, "l_mix": 2.7809433937072754, "l_a": 2.8702999651432037, "l_c": 2.8207234740257263, "z_loss": 34.97850847244263, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11115594953298569, "tokens_per_second": 1324491.231528316, "eta_seconds": 12203.383735013009, "data_wait_seconds": 1.7851591110229492e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 41660, "total_steps": 72479, "tokens": 21841838080, "loss": 4.330537557601929, "l_mix": 2.683252900838852, "l_a": 2.771001487970352, "l_c": 2.719947189092636, "z_loss": 35.130642890930176, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12149695307016373, "tokens_per_second": 1426698.8228801663, "eta_seconds": 11325.468005490302, "data_wait_seconds": 1.6957521438598633e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 41670, "total_steps": 72479, "tokens": 21847080960, "loss": 4.3940717577934265, "l_mix": 2.7235874831676483, "l_a": 2.8103547990322113, "l_c": 2.757925868034363, "z_loss": 34.613863945007324, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13014541566371918, "tokens_per_second": 1441361.6847196566, "eta_seconds": 11206.617439079284, "data_wait_seconds": 2.2202730178833008e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 41680, "total_steps": 72479, "tokens": 21852323840, "loss": 4.417125880718231, "l_mix": 2.736988604068756, "l_a": 2.824918806552887, "l_c": 2.7755385637283325, "z_loss": 34.33074188232422, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16981206834316254, "tokens_per_second": 1439307.615631828, "eta_seconds": 11218.968020892144, "data_wait_seconds": 1.5497207641601562e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 41690, "total_steps": 72479, "tokens": 21857566720, "loss": 4.4271209836006165, "l_mix": 2.744199812412262, "l_a": 2.830711394548416, "l_c": 2.77902552485466, "z_loss": 34.020495891571045, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16004815697669983, "tokens_per_second": 1380151.3214492202, "eta_seconds": 11696.038674259185, "data_wait_seconds": 2.059340476989746e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 41700, "total_steps": 72479, "tokens": 21862809600, "loss": 2.9712521731853485, "l_mix": 1.8389631509780884, "l_a": 1.9081213921308517, "l_c": 1.8661752492189407, "z_loss": 41.862900257110596, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17123882472515106, "tokens_per_second": 1445303.8182183253, "eta_seconds": 11165.16828405857, "data_wait_seconds": 1.755356788635254e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.20538746, "gate_a2b/L1_absmax": 0.74561632, "gate_a2b/L1_rms": 0.21843384, "gate_a2b/L3_absmean": 0.20550238, "gate_a2b/L3_absmax": 0.58824492, "gate_a2b/L3_rms": 0.21162818, "gate_a2b/L5_absmean": 0.31909439, "gate_a2b/L5_absmax": 0.72463095, "gate_a2b/L5_rms": 0.32493007, "gate_c2b/L1_absmean": 0.01796517, "gate_c2b/L1_absmax": 0.64069802, "gate_c2b/L1_rms": 0.06809553, "gate_c2b/L3_absmean": 0.19958155, "gate_c2b/L3_absmax": 0.44807181, "gate_c2b/L3_rms": 0.20379728, "gate_c2b/L5_absmean": 0.5671916, "gate_c2b/L5_absmax": 0.79542178, "gate_c2b/L5_rms": 0.57959253, "gate_b2a/L7_absmean": 0.05250804, "gate_b2a/L7_absmax": 0.31488219, "gate_b2a/L7_rms": 0.05783364, "gate_b2a/L11_absmean": 0.08498111, "gate_b2a/L11_absmax": 0.58448523, "gate_b2a/L11_rms": 0.09325533, "gate_b2a/L14_absmean": 0.21854234, "gate_b2a/L14_absmax": 0.78036344, "gate_b2a/L14_rms": 0.22660853, "gate_b2c/L3_absmean": 0.03884964, "gate_b2c/L3_absmax": 0.59158289, "gate_b2c/L3_rms": 0.05059228, "gate_b2c/L5_absmean": 0.04265197, "gate_b2c/L5_absmax": 0.60872638, "gate_b2c/L5_rms": 0.053348, "gate_b2c/L6_absmean": 0.061049, "gate_b2c/L6_absmax": 0.41992602, "gate_b2c/L6_rms": 0.06850278}, "streams": {"cos_a2b/round0": 0.389913, "cos_c2b/round0": 0.324584, "cos_b2a/round0": 0.139493, "cos_b2c/round0": 0.136229, "rms_a/round0": 4.938138, "rms_b/round0": 8.646914, "rms_c/round0": 3.432945, "cos_a2b/round1": 0.395239, "cos_c2b/round1": 0.26486, "cos_b2a/round1": 0.235109, "cos_b2c/round1": 0.104662, "rms_a/round1": 9.291107, "rms_b/round1": 19.805801, "rms_c/round1": 7.22438, "cos_a2b/round2": 0.509867, "cos_c2b/round2": 0.561155, "cos_b2a/round2": 0.101577, "cos_b2c/round2": 0.157561, "rms_a/round2": 23.788538, "rms_b/round2": 53.443665, "rms_c/round2": 13.399018}, "alpha_mean": 0.4217915269673682, "alpha_hist": [0.0107, 0.04265, 0.1644, 0.25388, 0.23131, 0.16464, 0.0948, 0.03458, 0.00299, 6e-05], "alpha_by_type": {"special": {"share": 0.00035, "mean": 0.53422, "std": 0.14101}, "whitespace": {"share": 0.11639, "mean": 0.47638, "std": 0.11876}, "digit": {"share": 0.13774, "mean": 0.38928, "std": 0.119}, "word_start": {"share": 0.2522, "mean": 0.41112, "std": 0.1523}, "word_cont": {"share": 0.24095, "mean": 0.33065, "std": 0.11314}, "punct": {"share": 0.25237, "mean": 0.51188, "std": 0.13916}}, "alpha_p05": 0.20657, "alpha_p50": 0.42179, "alpha_p95": 0.67533}
{"step": 41710, "total_steps": 72479, "tokens": 21868052480, "loss": 4.808432519435883, "l_mix": 2.9842372834682465, "l_a": 3.0614533126354218, "l_c": 3.0191974341869354, "z_loss": 34.48710012435913, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14445729553699493, "tokens_per_second": 1441624.845162672, "eta_seconds": 11190.024593520166, "data_wait_seconds": 1.9341707229614258e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 41720, "total_steps": 72479, "tokens": 21873295360, "loss": 4.80947870016098, "l_mix": 2.984213560819626, "l_a": 3.0624181926250458, "l_c": 3.021798610687256, "z_loss": 34.184093952178955, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13113649189472198, "tokens_per_second": 1445810.209634346, "eta_seconds": 11154.005196905136, "data_wait_seconds": 1.761317253112793e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 41730, "total_steps": 72479, "tokens": 21878538240, "loss": 4.607247471809387, "l_mix": 2.8568193912506104, "l_a": 2.93720743060112, "l_c": 2.8975526094436646, "z_loss": 34.391979694366455, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1451234668493271, "tokens_per_second": 1447532.635000245, "eta_seconds": 11137.11105518341, "data_wait_seconds": 2.1517276763916016e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 41740, "total_steps": 72479, "tokens": 21883781120, "loss": 4.814048230648041, "l_mix": 2.9862831234931946, "l_a": 3.0691283345222473, "l_c": 3.023421734571457, "z_loss": 34.52090930938721, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15276429057121277, "tokens_per_second": 1389006.2208073332, "eta_seconds": 11602.603782892227, "data_wait_seconds": 1.7404556274414062e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 41750, "total_steps": 72479, "tokens": 21889024000, "loss": 4.776085257530212, "l_mix": 2.9637502133846283, "l_a": 3.0375193655490875, "l_c": 3.003597229719162, "z_loss": 34.84689426422119, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11298162490129471, "tokens_per_second": 1438210.2963989924, "eta_seconds": 11202.009881544112, "data_wait_seconds": 2.0563602447509766e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 41760, "total_steps": 72479, "tokens": 21894266880, "loss": 4.287662386894226, "l_mix": 2.656645894050598, "l_a": 2.743774563074112, "l_c": 2.692947030067444, "z_loss": 35.064674854278564, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12651672959327698, "tokens_per_second": 1320107.3000873518, "eta_seconds": 12200.22271745205, "data_wait_seconds": 2.4050474166870117e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 41770, "total_steps": 72479, "tokens": 21899509760, "loss": 4.412280201911926, "l_mix": 2.73386150598526, "l_a": 2.823781043291092, "l_c": 2.7709476351737976, "z_loss": 35.33365535736084, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11141020804643631, "tokens_per_second": 1455981.076254597, "eta_seconds": 11058.083415079116, "data_wait_seconds": 1.6480684280395508e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 41780, "total_steps": 72479, "tokens": 21904752640, "loss": 4.4458171129226685, "l_mix": 2.7553457021713257, "l_a": 2.8411724865436554, "l_c": 2.793731778860092, "z_loss": 35.07564973831177, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1620846390724182, "tokens_per_second": 1424678.0890338589, "eta_seconds": 11297.371269965171, "data_wait_seconds": 2.1845102310180664e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 41790, "total_steps": 72479, "tokens": 21909995520, "loss": 4.398015260696411, "l_mix": 2.725169539451599, "l_a": 2.81541508436203, "l_c": 2.7607370615005493, "z_loss": 35.09581899642944, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1704929918050766, "tokens_per_second": 1466570.160536048, "eta_seconds": 10971.09082467556, "data_wait_seconds": 2.962350845336914e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 41800, "total_steps": 72479, "tokens": 21915238400, "loss": 4.3191863894462585, "l_mix": 2.676281124353409, "l_a": 2.763724833726883, "l_c": 2.7126259803771973, "z_loss": 34.69031524658203, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13137656450271606, "tokens_per_second": 1448855.7392772466, "eta_seconds": 11101.61012995243, "data_wait_seconds": 2.0325183868408203e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.20550898, "gate_a2b/L1_absmax": 0.74222147, "gate_a2b/L1_rms": 0.21838912, "gate_a2b/L3_absmean": 0.20520799, "gate_a2b/L3_absmax": 0.58839303, "gate_a2b/L3_rms": 0.21138006, "gate_a2b/L5_absmean": 0.31820482, "gate_a2b/L5_absmax": 0.72527164, "gate_a2b/L5_rms": 0.32410082, "gate_c2b/L1_absmean": 0.01800681, "gate_c2b/L1_absmax": 0.6382277, "gate_c2b/L1_rms": 0.06793433, "gate_c2b/L3_absmean": 0.19996986, "gate_c2b/L3_absmax": 0.44566587, "gate_c2b/L3_rms": 0.20417081, "gate_c2b/L5_absmean": 0.5662505, "gate_c2b/L5_absmax": 0.78722721, "gate_c2b/L5_rms": 0.57859147, "gate_b2a/L7_absmean": 0.05298378, "gate_b2a/L7_absmax": 0.31632003, "gate_b2a/L7_rms": 0.05826656, "gate_b2a/L11_absmean": 0.08437832, "gate_b2a/L11_absmax": 0.58687019, "gate_b2a/L11_rms": 0.09279523, "gate_b2a/L14_absmean": 0.21784556, "gate_b2a/L14_absmax": 0.78295344, "gate_b2a/L14_rms": 0.22597836, "gate_b2c/L3_absmean": 0.03887176, "gate_b2c/L3_absmax": 0.59035236, "gate_b2c/L3_rms": 0.05053747, "gate_b2c/L5_absmean": 0.04262826, "gate_b2c/L5_absmax": 0.60674506, "gate_b2c/L5_rms": 0.05325985, "gate_b2c/L6_absmean": 0.0607751, "gate_b2c/L6_absmax": 0.41742533, "gate_b2c/L6_rms": 0.06824638}, "streams": {"cos_a2b/round0": 0.371713, "cos_c2b/round0": 0.305981, "cos_b2a/round0": 0.148257, "cos_b2c/round0": 0.129754, "rms_a/round0": 4.892962, "rms_b/round0": 9.184531, "rms_c/round0": 3.642183, "cos_a2b/round1": 0.421564, "cos_c2b/round1": 0.304974, "cos_b2a/round1": 0.251561, "cos_b2c/round1": 0.121807, "rms_a/round1": 9.330841, "rms_b/round1": 20.661448, "rms_c/round1": 7.34517, "cos_a2b/round2": 0.557242, "cos_c2b/round2": 0.519795, "cos_b2a/round2": 0.10639, "cos_b2c/round2": 0.180822, "rms_a/round2": 25.014915, "rms_b/round2": 53.688488, "rms_c/round2": 13.740415}, "alpha_mean": 0.43728594989853775, "alpha_hist": [0.00665, 0.06099, 0.17706, 0.20689, 0.19109, 0.15701, 0.12192, 0.06999, 0.00838, 2e-05], "alpha_by_type": {"special": {"share": 0.00127, "mean": 0.57472, "std": 0.09371}, "whitespace": {"share": 0.02382, "mean": 0.52188, "std": 0.10532}, "digit": {"share": 0.02786, "mean": 0.44582, "std": 0.10125}, "word_start": {"share": 0.59943, "mean": 0.4566, "std": 0.16486}, "word_cont": {"share": 0.24768, "mean": 0.32375, "std": 0.12376}, "punct": {"share": 0.09995, "mean": 0.57856, "std": 0.15957}}, "alpha_p05": 0.19072, "alpha_p50": 0.43746, "alpha_p95": 0.73106}
{"step": 41810, "total_steps": 72479, "tokens": 21920481280, "loss": 2.8471481800079346, "l_mix": 1.7617865949869156, "l_a": 1.8290302455425262, "l_c": 1.7888414859771729, "z_loss": 40.554930210113525, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16679923236370087, "tokens_per_second": 1441517.7740421514, "eta_seconds": 11154.485197162629, "data_wait_seconds": 2.7418136596679688e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 41820, "total_steps": 72479, "tokens": 21925724160, "loss": 3.030419886112213, "l_mix": 1.8755532801151276, "l_a": 1.9443383067846298, "l_c": 1.9052168875932693, "z_loss": 43.502938747406006, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18397526443004608, "tokens_per_second": 1430726.0728246833, "eta_seconds": 11234.956919646263, "data_wait_seconds": 2.2649765014648438e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 41830, "total_steps": 72479, "tokens": 21930967040, "loss": 4.810485780239105, "l_mix": 2.984406590461731, "l_a": 3.063531517982483, "l_c": 3.0233988165855408, "z_loss": 34.82929563522339, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1397211104631424, "tokens_per_second": 1443458.5765259622, "eta_seconds": 11132.223101735115, "data_wait_seconds": 2.1219253540039062e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 41840, "total_steps": 72479, "tokens": 21936209920, "loss": 4.699846506118774, "l_mix": 2.914694994688034, "l_a": 2.997868835926056, "l_c": 2.952636033296585, "z_loss": 34.31450366973877, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12660253047943115, "tokens_per_second": 1430496.1879871986, "eta_seconds": 11229.432253575325, "data_wait_seconds": 3.764033317565918e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 41850, "total_steps": 72479, "tokens": 21941452800, "loss": 4.451488554477692, "l_mix": 2.761347621679306, "l_a": 2.835888534784317, "l_c": 2.7979142665863037, "z_loss": 34.22571659088135, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16913098096847534, "tokens_per_second": 1363641.4830061311, "eta_seconds": 11776.128368139267, "data_wait_seconds": 3.120303153991699e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 41860, "total_steps": 72479, "tokens": 21946695680, "loss": 4.634545981884003, "l_mix": 2.8745467364788055, "l_a": 2.9548992216587067, "l_c": 2.9117646515369415, "z_loss": 34.428677558898926, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15828406810760498, "tokens_per_second": 1432271.9406578038, "eta_seconds": 11208.188763809205, "data_wait_seconds": 1.6570091247558594e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 41870, "total_steps": 72479, "tokens": 21951938560, "loss": 4.351851403713226, "l_mix": 2.696683019399643, "l_a": 2.7858740985393524, "l_c": 2.7313537001609802, "z_loss": 35.042542934417725, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11623788625001907, "tokens_per_second": 1446227.6386581569, "eta_seconds": 11096.407621479035, "data_wait_seconds": 2.2649765014648438e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 41880, "total_steps": 72479, "tokens": 21957181440, "loss": 4.377063572406769, "l_mix": 2.712236851453781, "l_a": 2.8005602657794952, "l_c": 2.7488617599010468, "z_loss": 35.01281452178955, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12727482616901398, "tokens_per_second": 1418548.7809524522, "eta_seconds": 11309.225828123093, "data_wait_seconds": 1.7434358596801758e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 41890, "total_steps": 72479, "tokens": 21962424320, "loss": 4.520561456680298, "l_mix": 2.8017571568489075, "l_a": 2.8893409073352814, "l_c": 2.840006500482559, "z_loss": 35.584306716918945, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13141192495822906, "tokens_per_second": 1455650.4958241344, "eta_seconds": 11017.373798179626, "data_wait_seconds": 1.5676021575927734e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 41900, "total_steps": 72479, "tokens": 21967667200, "loss": 4.427300035953522, "l_mix": 2.7430669367313385, "l_a": 2.831507384777069, "l_c": 2.7826027572155, "z_loss": 35.89589595794678, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16000063717365265, "tokens_per_second": 1449445.5389299884, "eta_seconds": 11060.921104931831, "data_wait_seconds": 2.250075340270996e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.20516537, "gate_a2b/L1_absmax": 0.74006462, "gate_a2b/L1_rms": 0.21801655, "gate_a2b/L3_absmean": 0.20427962, "gate_a2b/L3_absmax": 0.58608955, "gate_a2b/L3_rms": 0.21050897, "gate_a2b/L5_absmean": 0.3173793, "gate_a2b/L5_absmax": 0.72568136, "gate_a2b/L5_rms": 0.32326627, "gate_c2b/L1_absmean": 0.017902, "gate_c2b/L1_absmax": 0.63710964, "gate_c2b/L1_rms": 0.06769605, "gate_c2b/L3_absmean": 0.19935329, "gate_c2b/L3_absmax": 0.44279456, "gate_c2b/L3_rms": 0.20341264, "gate_c2b/L5_absmean": 0.56559926, "gate_c2b/L5_absmax": 0.79400557, "gate_c2b/L5_rms": 0.57799506, "gate_b2a/L7_absmean": 0.0519906, "gate_b2a/L7_absmax": 0.31675637, "gate_b2a/L7_rms": 0.05743935, "gate_b2a/L11_absmean": 0.08426379, "gate_b2a/L11_absmax": 0.58843338, "gate_b2a/L11_rms": 0.09271351, "gate_b2a/L14_absmean": 0.21800643, "gate_b2a/L14_absmax": 0.78064388, "gate_b2a/L14_rms": 0.2260602, "gate_b2c/L3_absmean": 0.03903831, "gate_b2c/L3_absmax": 0.58927882, "gate_b2c/L3_rms": 0.05063789, "gate_b2c/L5_absmean": 0.0421744, "gate_b2c/L5_absmax": 0.60789913, "gate_b2c/L5_rms": 0.05291674, "gate_b2c/L6_absmean": 0.06067795, "gate_b2c/L6_absmax": 0.41288468, "gate_b2c/L6_rms": 0.06811016}, "streams": {"cos_a2b/round0": 0.372595, "cos_c2b/round0": 0.311596, "cos_b2a/round0": 0.148083, "cos_b2c/round0": 0.13034, "rms_a/round0": 4.924921, "rms_b/round0": 9.209602, "rms_c/round0": 3.683931, "cos_a2b/round1": 0.420376, "cos_c2b/round1": 0.299877, "cos_b2a/round1": 0.25252, "cos_b2c/round1": 0.120304, "rms_a/round1": 9.389846, "rms_b/round1": 20.785311, "rms_c/round1": 7.449167, "cos_a2b/round2": 0.545623, "cos_c2b/round2": 0.529004, "cos_b2a/round2": 0.101155, "cos_b2c/round2": 0.179958, "rms_a/round2": 25.03783, "rms_b/round2": 54.117443, "rms_c/round2": 13.988627}, "alpha_mean": 0.41141818193938207, "alpha_hist": [0.01526, 0.09912, 0.19867, 0.19797, 0.17081, 0.14125, 0.11108, 0.05931, 0.0065, 3e-05], "alpha_by_type": {"special": {"share": 0.00139, "mean": 0.53675, "std": 0.10032}, "whitespace": {"share": 0.02725, "mean": 0.49373, "std": 0.12083}, "digit": {"share": 0.03268, "mean": 0.42642, "std": 0.09994}, "word_start": {"share": 0.57892, "mean": 0.43167, "std": 0.17292}, "word_cont": {"share": 0.25739, "mean": 0.29536, "std": 0.12706}, "punct": {"share": 0.10237, "mean": 0.56031, "std": 0.16503}}, "alpha_p05": 0.14609, "alpha_p50": 0.39071, "alpha_p95": 0.71522}
{"step": 41910, "total_steps": 72479, "tokens": 21972910080, "loss": 4.532608270645142, "l_mix": 2.8094707429409027, "l_a": 2.8969306647777557, "l_c": 2.8468607664108276, "z_loss": 34.6415376663208, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1265818476676941, "tokens_per_second": 1426274.180281744, "eta_seconds": 11236.941742038727, "data_wait_seconds": 2.2649765014648438e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 41920, "total_steps": 72479, "tokens": 21978152960, "loss": 4.435756623744965, "l_mix": 2.748943328857422, "l_a": 2.8339477479457855, "l_c": 2.7887629568576813, "z_loss": 34.502180099487305, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18052701652050018, "tokens_per_second": 1429456.301126249, "eta_seconds": 11208.259377622604, "data_wait_seconds": 2.0802021026611328e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 41930, "total_steps": 72479, "tokens": 21983395840, "loss": 3.004243940114975, "l_mix": 1.859009400010109, "l_a": 1.9307086169719696, "l_c": 1.8867397457361221, "z_loss": 42.831735610961914, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1273571252822876, "tokens_per_second": 1446376.7924635028, "eta_seconds": 11073.51431207657, "data_wait_seconds": 3.701448440551758e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 41940, "total_steps": 72479, "tokens": 21988638720, "loss": 4.4508795738220215, "l_mix": 2.7580438554286957, "l_a": 2.8454408049583435, "l_c": 2.797344923019409, "z_loss": 35.39570331573486, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11969507485628128, "tokens_per_second": 1351500.6049434652, "eta_seconds": 11847.002637982368, "data_wait_seconds": 1.6957521438598633e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 41950, "total_steps": 72479, "tokens": 21993881600, "loss": 4.453677177429199, "l_mix": 2.7600586116313934, "l_a": 2.8491427898406982, "l_c": 2.796252280473709, "z_loss": 34.07069110870361, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15352490544319153, "tokens_per_second": 1294817.6490623183, "eta_seconds": 12361.577217912672, "data_wait_seconds": 2.1785497665405273e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 41960, "total_steps": 72479, "tokens": 21999124480, "loss": 4.347839951515198, "l_mix": 2.6943873465061188, "l_a": 2.782051742076874, "l_c": 2.7294569611549377, "z_loss": 34.4289755821228, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18084844946861267, "tokens_per_second": 1468479.2202384346, "eta_seconds": 10896.13339533806, "data_wait_seconds": 2.2202730178833008e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 41970, "total_steps": 72479, "tokens": 22004367360, "loss": 4.353278577327728, "l_mix": 2.6966151297092438, "l_a": 2.7875392138957977, "l_c": 2.7346719205379486, "z_loss": 35.57033920288086, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12638841569423676, "tokens_per_second": 1454450.3929203474, "eta_seconds": 10997.626780438424, "data_wait_seconds": 1.9252300262451172e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 41980, "total_steps": 72479, "tokens": 22009610240, "loss": 4.481564044952393, "l_mix": 2.7764905393123627, "l_a": 2.8671716153621674, "l_c": 2.816406786441803, "z_loss": 35.43825340270996, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16240383684635162, "tokens_per_second": 1424129.3898547625, "eta_seconds": 11228.094740486144, "data_wait_seconds": 1.7404556274414062e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 41990, "total_steps": 72479, "tokens": 22014853120, "loss": 4.464456677436829, "l_mix": 2.767167776823044, "l_a": 2.855481445789337, "l_c": 2.8021480441093445, "z_loss": 34.70850467681885, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15059399604797363, "tokens_per_second": 1460375.6303751997, "eta_seconds": 10945.825511956215, "data_wait_seconds": 1.77919864654541e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 42000, "total_steps": 72479, "tokens": 22020096000, "loss": 4.3346903920173645, "l_mix": 2.6848567128181458, "l_a": 2.776997059583664, "l_c": 2.722448021173477, "z_loss": 36.23563194274902, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12619653344154358, "tokens_per_second": 1388095.8534448873, "eta_seconds": 11512.010436701774, "data_wait_seconds": 1.5914440155029297e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.20473039, "gate_a2b/L1_absmax": 0.73833174, "gate_a2b/L1_rms": 0.21758868, "gate_a2b/L3_absmean": 0.20402156, "gate_a2b/L3_absmax": 0.5871104, "gate_a2b/L3_rms": 0.21021305, "gate_a2b/L5_absmean": 0.31658593, "gate_a2b/L5_absmax": 0.72436357, "gate_a2b/L5_rms": 0.32246673, "gate_c2b/L1_absmean": 0.01795007, "gate_c2b/L1_absmax": 0.63581634, "gate_c2b/L1_rms": 0.06758321, "gate_c2b/L3_absmean": 0.19941171, "gate_c2b/L3_absmax": 0.44935209, "gate_c2b/L3_rms": 0.20349005, "gate_c2b/L5_absmean": 0.56528604, "gate_c2b/L5_absmax": 0.79817402, "gate_c2b/L5_rms": 0.57770318, "gate_b2a/L7_absmean": 0.05217089, "gate_b2a/L7_absmax": 0.31708059, "gate_b2a/L7_rms": 0.05753817, "gate_b2a/L11_absmean": 0.0843823, "gate_b2a/L11_absmax": 0.58795828, "gate_b2a/L11_rms": 0.09280007, "gate_b2a/L14_absmean": 0.21764553, "gate_b2a/L14_absmax": 0.78104156, "gate_b2a/L14_rms": 0.22569479, "gate_b2c/L3_absmean": 0.03864151, "gate_b2c/L3_absmax": 0.58794016, "gate_b2c/L3_rms": 0.05030077, "gate_b2c/L5_absmean": 0.04223264, "gate_b2c/L5_absmax": 0.60894895, "gate_b2c/L5_rms": 0.05300567, "gate_b2c/L6_absmean": 0.06076756, "gate_b2c/L6_absmax": 0.41157767, "gate_b2c/L6_rms": 0.06805273}, "streams": {"cos_a2b/round0": 0.374831, "cos_c2b/round0": 0.314188, "cos_b2a/round0": 0.141335, "cos_b2c/round0": 0.130404, "rms_a/round0": 4.977338, "rms_b/round0": 9.220261, "rms_c/round0": 3.66867, "cos_a2b/round1": 0.416679, "cos_c2b/round1": 0.295883, "cos_b2a/round1": 0.244349, "cos_b2c/round1": 0.11461, "rms_a/round1": 9.500026, "rms_b/round1": 20.942038, "rms_c/round1": 7.451397, "cos_a2b/round2": 0.545834, "cos_c2b/round2": 0.526106, "cos_b2a/round2": 0.106208, "cos_b2c/round2": 0.173744, "rms_a/round2": 25.533979, "rms_b/round2": 55.082996, "rms_c/round2": 13.991321}, "alpha_mean": 0.4432953415942933, "alpha_hist": [0.00868, 0.07617, 0.17313, 0.18301, 0.16518, 0.17097, 0.13016, 0.07666, 0.01578, 0.00026], "alpha_by_type": {"special": {"share": 0.00121, "mean": 0.55795, "std": 0.12052}, "whitespace": {"share": 0.0302, "mean": 0.53775, "std": 0.13614}, "digit": {"share": 0.05099, "mean": 0.45664, "std": 0.12144}, "word_start": {"share": 0.54994, "mean": 0.45913, "std": 0.17365}, "word_cont": {"share": 0.254, "mean": 0.32345, "std": 0.1349}, "punct": {"share": 0.11366, "mean": 0.60222, "std": 0.16181}}, "alpha_p05": 0.16776, "alpha_p50": 0.42967, "alpha_p95": 0.74498}
{"step": 42010, "total_steps": 72479, "tokens": 22025338880, "loss": 4.368853747844696, "l_mix": 2.7060429453849792, "l_a": 2.7957431972026825, "l_c": 2.7469590604305267, "z_loss": 35.29262065887451, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2154349535703659, "tokens_per_second": 1433466.636858177, "eta_seconds": 11143.985260105132, "data_wait_seconds": 2.053380012512207e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 42020, "total_steps": 72479, "tokens": 22030581760, "loss": 4.501189172267914, "l_mix": 2.7900214791297913, "l_a": 2.878103792667389, "l_c": 2.825788140296936, "z_loss": 35.21134042739868, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18116670846939087, "tokens_per_second": 1359214.7515284051, "eta_seconds": 11748.907355546951, "data_wait_seconds": 2.2411346435546875e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 42030, "total_steps": 72479, "tokens": 22035824640, "loss": 4.428004145622253, "l_mix": 2.7438598573207855, "l_a": 2.832079291343689, "l_c": 2.781734734773636, "z_loss": 36.01449155807495, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13144007325172424, "tokens_per_second": 1471277.084812038, "eta_seconds": 10850.46826107502, "data_wait_seconds": 1.913309097290039e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 42040, "total_steps": 72479, "tokens": 22041067520, "loss": 3.1748624742031097, "l_mix": 1.9656473100185394, "l_a": 2.0359881818294525, "l_c": 1.9947289824485779, "z_loss": 42.66087770462036, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15923596918582916, "tokens_per_second": 1454704.2086760711, "eta_seconds": 10970.47931587696, "data_wait_seconds": 1.919269561767578e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 42050, "total_steps": 72479, "tokens": 22046310400, "loss": 4.402243196964264, "l_mix": 2.7278983891010284, "l_a": 2.8182593882083893, "l_c": 2.762890011072159, "z_loss": 35.832173347473145, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15253569185733795, "tokens_per_second": 1455869.3560431544, "eta_seconds": 10958.09832508564, "data_wait_seconds": 1.7315149307250977e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 42060, "total_steps": 72479, "tokens": 22051553280, "loss": 4.362163186073303, "l_mix": 2.7020503878593445, "l_a": 2.7929490208625793, "l_c": 2.740760028362274, "z_loss": 35.535279273986816, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14744777977466583, "tokens_per_second": 1439298.5719363834, "eta_seconds": 11080.617311072348, "data_wait_seconds": 3.7789344787597656e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 42070, "total_steps": 72479, "tokens": 22056796160, "loss": 4.478434920310974, "l_mix": 2.776268094778061, "l_a": 2.8580383956432343, "l_c": 2.8158512115478516, "z_loss": 33.91387176513672, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.22378522157669067, "tokens_per_second": 1369427.3292034233, "eta_seconds": 11642.146649193763, "data_wait_seconds": 3.269314765930176e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 42080, "total_steps": 72479, "tokens": 22062039040, "loss": 4.431610822677612, "l_mix": 2.7464264631271362, "l_a": 2.8340645134449005, "l_c": 2.783216655254364, "z_loss": 35.32777261734009, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12894384562969208, "tokens_per_second": 1416020.7209175667, "eta_seconds": 11255.36560063362, "data_wait_seconds": 1.8596649169921875e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 42090, "total_steps": 72479, "tokens": 22067281920, "loss": 4.393395960330963, "l_mix": 2.7220377027988434, "l_a": 2.812741845846176, "l_c": 2.7584519386291504, "z_loss": 35.557809829711914, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1428237408399582, "tokens_per_second": 1441376.4229440053, "eta_seconds": 11053.731543254853, "data_wait_seconds": 1.767277717590332e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 42100, "total_steps": 72479, "tokens": 22072524800, "loss": 4.372510492801666, "l_mix": 2.7089316844940186, "l_a": 2.798490345478058, "l_c": 2.746772050857544, "z_loss": 34.96968984603882, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13358676433563232, "tokens_per_second": 1397686.8784535022, "eta_seconds": 11395.503096961975, "data_wait_seconds": 2.321600914001465e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.20428477, "gate_a2b/L1_absmax": 0.73872083, "gate_a2b/L1_rms": 0.21718808, "gate_a2b/L3_absmean": 0.20362461, "gate_a2b/L3_absmax": 0.58440322, "gate_a2b/L3_rms": 0.20976107, "gate_a2b/L5_absmean": 0.31562415, "gate_a2b/L5_absmax": 0.72485411, "gate_a2b/L5_rms": 0.32151091, "gate_c2b/L1_absmean": 0.01766429, "gate_c2b/L1_absmax": 0.63732171, "gate_c2b/L1_rms": 0.06761492, "gate_c2b/L3_absmean": 0.19935955, "gate_c2b/L3_absmax": 0.45605192, "gate_c2b/L3_rms": 0.20357576, "gate_c2b/L5_absmean": 0.56431949, "gate_c2b/L5_absmax": 0.79942441, "gate_c2b/L5_rms": 0.5767259, "gate_b2a/L7_absmean": 0.05221352, "gate_b2a/L7_absmax": 0.31762034, "gate_b2a/L7_rms": 0.05759527, "gate_b2a/L11_absmean": 0.08473613, "gate_b2a/L11_absmax": 0.58998817, "gate_b2a/L11_rms": 0.09315815, "gate_b2a/L14_absmean": 0.21708831, "gate_b2a/L14_absmax": 0.78203565, "gate_b2a/L14_rms": 0.22519834, "gate_b2c/L3_absmean": 0.03877009, "gate_b2c/L3_absmax": 0.58731711, "gate_b2c/L3_rms": 0.05038437, "gate_b2c/L5_absmean": 0.0421094, "gate_b2c/L5_absmax": 0.60900104, "gate_b2c/L5_rms": 0.05288903, "gate_b2c/L6_absmean": 0.06071286, "gate_b2c/L6_absmax": 0.41065121, "gate_b2c/L6_rms": 0.06802617}, "streams": {"cos_a2b/round0": 0.3697, "cos_c2b/round0": 0.304848, "cos_b2a/round0": 0.144236, "cos_b2c/round0": 0.125813, "rms_a/round0": 4.907181, "rms_b/round0": 9.204324, "rms_c/round0": 3.672341, "cos_a2b/round1": 0.416441, "cos_c2b/round1": 0.298874, "cos_b2a/round1": 0.246846, "cos_b2c/round1": 0.116083, "rms_a/round1": 9.329101, "rms_b/round1": 20.789574, "rms_c/round1": 7.384865, "cos_a2b/round2": 0.541911, "cos_c2b/round2": 0.531039, "cos_b2a/round2": 0.099803, "cos_b2c/round2": 0.172985, "rms_a/round2": 24.972799, "rms_b/round2": 54.565361, "rms_c/round2": 13.72146}, "alpha_mean": 0.414217430294471, "alpha_hist": [0.00937, 0.09084, 0.19637, 0.21538, 0.17444, 0.14113, 0.10669, 0.05855, 0.00723, 2e-05], "alpha_by_type": {"special": {"share": 0.00078, "mean": 0.52671, "std": 0.126}, "whitespace": {"share": 0.02707, "mean": 0.50379, "std": 0.11626}, "digit": {"share": 0.03503, "mean": 0.44017, "std": 0.12529}, "word_start": {"share": 0.53979, "mean": 0.43398, "std": 0.16864}, "word_cont": {"share": 0.28403, "mean": 0.305, "std": 0.12028}, "punct": {"share": 0.1133, "mean": 0.56365, "std": 0.15876}}, "alpha_p05": 0.16132, "alpha_p50": 0.38908, "alpha_p95": 0.7074}
{"step": 42110, "total_steps": 72479, "tokens": 22077767680, "loss": 4.665970802307129, "l_mix": 2.894580155611038, "l_a": 2.9735686779022217, "l_c": 2.931066781282425, "z_loss": 34.32930612564087, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11192385852336884, "tokens_per_second": 1408236.8597021562, "eta_seconds": 11306.409260845185, "data_wait_seconds": 1.755356788635254e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 42120, "total_steps": 72479, "tokens": 22083010560, "loss": 4.829006671905518, "l_mix": 2.9962345361709595, "l_a": 3.0732966363430023, "l_c": 3.035943627357483, "z_loss": 34.20820093154907, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1424471139907837, "tokens_per_second": 1338088.1238553885, "eta_seconds": 11895.225066447258, "data_wait_seconds": 2.294778823852539e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 42130, "total_steps": 72479, "tokens": 22088253440, "loss": 4.748363137245178, "l_mix": 2.944729119539261, "l_a": 3.0290949940681458, "l_c": 2.9830183386802673, "z_loss": 34.757675647735596, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13483573496341705, "tokens_per_second": 1397407.2780522574, "eta_seconds": 11386.527579975127, "data_wait_seconds": 2.0831823348999023e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 42140, "total_steps": 72479, "tokens": 22093496320, "loss": 4.738035500049591, "l_mix": 2.940238356590271, "l_a": 3.0157733261585236, "l_c": 2.9768838584423065, "z_loss": 33.613945960998535, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14510752260684967, "tokens_per_second": 1464794.060760534, "eta_seconds": 10859.119420337676, "data_wait_seconds": 1.901388168334961e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 42150, "total_steps": 72479, "tokens": 22098739200, "loss": 4.890708744525909, "l_mix": 3.0355765521526337, "l_a": 3.1092412173748016, "l_c": 3.074532389640808, "z_loss": 34.41075921058655, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13611960411071777, "tokens_per_second": 1396444.188737057, "eta_seconds": 11386.871656060219, "data_wait_seconds": 1.8090009689331055e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 42160, "total_steps": 72479, "tokens": 22103982080, "loss": 4.383793830871582, "l_mix": 2.7163285315036774, "l_a": 2.8043320775032043, "l_c": 2.7538853883743286, "z_loss": 35.82975912094116, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.10919719934463501, "tokens_per_second": 1452371.1544782158, "eta_seconds": 10944.783516931535, "data_wait_seconds": 1.6927719116210938e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 42170, "total_steps": 72479, "tokens": 22109224960, "loss": 4.474190354347229, "l_mix": 2.773067533969879, "l_a": 2.860092371702194, "l_c": 2.8103165924549103, "z_loss": 35.056116580963135, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1145973801612854, "tokens_per_second": 1482167.9726448203, "eta_seconds": 10721.217355442046, "data_wait_seconds": 2.1636486053466797e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 42180, "total_steps": 72479, "tokens": 22114467840, "loss": 4.365640461444855, "l_mix": 2.705052763223648, "l_a": 2.7921760380268097, "l_c": 2.7431162297725677, "z_loss": 35.74717044830322, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12440278381109238, "tokens_per_second": 1452269.96214562, "eta_seconds": 10938.325880217551, "data_wait_seconds": 1.9431114196777344e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 42190, "total_steps": 72479, "tokens": 22119710720, "loss": 4.470094263553619, "l_mix": 2.769636809825897, "l_a": 2.8600485026836395, "l_c": 2.8081426918506622, "z_loss": 35.40963268280029, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13781625032424927, "tokens_per_second": 1378001.7941732008, "eta_seconds": 11524.048298883437, "data_wait_seconds": 2.2739171981811523e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 42200, "total_steps": 72479, "tokens": 22124953600, "loss": 4.410527050495148, "l_mix": 2.7340484261512756, "l_a": 2.818498879671097, "l_c": 2.769763171672821, "z_loss": 34.39023685455322, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.21859951317310333, "tokens_per_second": 1411645.5064710968, "eta_seconds": 11245.681921720505, "data_wait_seconds": 3.2395124435424805e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.20487355, "gate_a2b/L1_absmax": 0.7349211, "gate_a2b/L1_rms": 0.2175241, "gate_a2b/L3_absmean": 0.20353451, "gate_a2b/L3_absmax": 0.58284527, "gate_a2b/L3_rms": 0.20962226, "gate_a2b/L5_absmean": 0.31457666, "gate_a2b/L5_absmax": 0.72650087, "gate_a2b/L5_rms": 0.32047063, "gate_c2b/L1_absmean": 0.01802721, "gate_c2b/L1_absmax": 0.63480937, "gate_c2b/L1_rms": 0.06758874, "gate_c2b/L3_absmean": 0.19836698, "gate_c2b/L3_absmax": 0.44875807, "gate_c2b/L3_rms": 0.20250665, "gate_c2b/L5_absmean": 0.56429613, "gate_c2b/L5_absmax": 0.79482949, "gate_c2b/L5_rms": 0.57663429, "gate_b2a/L7_absmean": 0.05203371, "gate_b2a/L7_absmax": 0.31845391, "gate_b2a/L7_rms": 0.05752294, "gate_b2a/L11_absmean": 0.08425759, "gate_b2a/L11_absmax": 0.59218323, "gate_b2a/L11_rms": 0.09281301, "gate_b2a/L14_absmean": 0.21690288, "gate_b2a/L14_absmax": 0.78119028, "gate_b2a/L14_rms": 0.22504774, "gate_b2c/L3_absmean": 0.03846892, "gate_b2c/L3_absmax": 0.58704883, "gate_b2c/L3_rms": 0.05006629, "gate_b2c/L5_absmean": 0.04177091, "gate_b2c/L5_absmax": 0.60858971, "gate_b2c/L5_rms": 0.05260575, "gate_b2c/L6_absmean": 0.0603273, "gate_b2c/L6_absmax": 0.40740275, "gate_b2c/L6_rms": 0.0675945}, "streams": {"cos_a2b/round0": 0.369492, "cos_c2b/round0": 0.305604, "cos_b2a/round0": 0.143392, "cos_b2c/round0": 0.127025, "rms_a/round0": 4.960629, "rms_b/round0": 9.303471, "rms_c/round0": 3.716353, "cos_a2b/round1": 0.416506, "cos_c2b/round1": 0.293955, "cos_b2a/round1": 0.248109, "cos_b2c/round1": 0.111736, "rms_a/round1": 9.404131, "rms_b/round1": 21.047541, "rms_c/round1": 7.437763, "cos_a2b/round2": 0.532004, "cos_c2b/round2": 0.535088, "cos_b2a/round2": 0.098074, "cos_b2c/round2": 0.176088, "rms_a/round2": 25.078554, "rms_b/round2": 55.305912, "rms_c/round2": 13.890708}, "alpha_mean": 0.41483471233873104, "alpha_hist": [0.00877, 0.0815, 0.19846, 0.22397, 0.17851, 0.13477, 0.11406, 0.05444, 0.00548, 5e-05], "alpha_by_type": {"special": {"share": 0.00093, "mean": 0.53563, "std": 0.11526}, "whitespace": {"share": 0.02977, "mean": 0.51997, "std": 0.12659}, "digit": {"share": 0.04182, "mean": 0.42893, "std": 0.10998}, "word_start": {"share": 0.53809, "mean": 0.43854, "std": 0.16411}, "word_cont": {"share": 0.28111, "mean": 0.30011, "std": 0.11647}, "punct": {"share": 0.10828, "mean": 0.55948, "std": 0.1557}}, "alpha_p05": 0.16996, "alpha_p50": 0.3935, "alpha_p95": 0.71183}
{"step": 42210, "total_steps": 72479, "tokens": 22130196480, "loss": 4.4667675495147705, "l_mix": 2.7687660455703735, "l_a": 2.8549036383628845, "l_c": 2.8051014840602875, "z_loss": 34.575520515441895, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12052522599697113, "tokens_per_second": 1396747.7118268728, "eta_seconds": 11361.875403571128, "data_wait_seconds": 1.8328428268432617e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 42220, "total_steps": 72479, "tokens": 22135439360, "loss": 4.721948444843292, "l_mix": 2.9285732805728912, "l_a": 3.0110349655151367, "l_c": 2.9668816924095154, "z_loss": 35.071168422698975, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13261224329471588, "tokens_per_second": 1430710.6207585393, "eta_seconds": 11088.497115921973, "data_wait_seconds": 1.7076730728149414e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 42230, "total_steps": 72479, "tokens": 22140682240, "loss": 4.799040913581848, "l_mix": 2.9773696660995483, "l_a": 3.055785298347473, "l_c": 3.0164517760276794, "z_loss": 33.79721117019653, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15915854275226593, "tokens_per_second": 1321415.5543279627, "eta_seconds": 12001.665683484078, "data_wait_seconds": 2.1725893020629883e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 42240, "total_steps": 72479, "tokens": 22145925120, "loss": 4.623131573200226, "l_mix": 2.8683257400989532, "l_a": 2.946491301059723, "l_c": 2.9028609693050385, "z_loss": 34.64566516876221, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15513074398040771, "tokens_per_second": 1383477.0768338176, "eta_seconds": 11459.492244195937, "data_wait_seconds": 1.856684684753418e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 42250, "total_steps": 72479, "tokens": 22151168000, "loss": 4.686431646347046, "l_mix": 2.906231641769409, "l_a": 2.9875302016735077, "l_c": 2.9464694261550903, "z_loss": 34.77308940887451, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19243648648262024, "tokens_per_second": 1479942.6520224558, "eta_seconds": 10708.997359013558, "data_wait_seconds": 2.753734588623047e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 42260, "total_steps": 72479, "tokens": 22156410880, "loss": 4.82661759853363, "l_mix": 2.9948441982269287, "l_a": 3.075339287519455, "l_c": 3.030571788549423, "z_loss": 33.726526737213135, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16828739643096924, "tokens_per_second": 1420624.560130771, "eta_seconds": 11152.46034500599, "data_wait_seconds": 2.446770668029785e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 42270, "total_steps": 72479, "tokens": 22161653760, "loss": 4.416296422481537, "l_mix": 2.7370060980319977, "l_a": 2.8202608227729797, "l_c": 2.7773733139038086, "z_loss": 35.89082622528076, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14086130261421204, "tokens_per_second": 1414593.8843308012, "eta_seconds": 11196.299070310593, "data_wait_seconds": 2.294778823852539e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 42280, "total_steps": 72479, "tokens": 22166896640, "loss": 4.283906877040863, "l_mix": 2.6539171636104584, "l_a": 2.7422337234020233, "l_c": 2.691065013408661, "z_loss": 35.34213876724243, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12537860870361328, "tokens_per_second": 1455819.4296889352, "eta_seconds": 10875.64363348484, "data_wait_seconds": 1.990795135498047e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 42290, "total_steps": 72479, "tokens": 22172139520, "loss": 4.340302765369415, "l_mix": 2.688674509525299, "l_a": 2.779238998889923, "l_c": 2.726188153028488, "z_loss": 35.33926725387573, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1351446956396103, "tokens_per_second": 1470183.7687224152, "eta_seconds": 10765.817694854735, "data_wait_seconds": 2.8312206268310547e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 42300, "total_steps": 72479, "tokens": 22177382400, "loss": 4.444323658943176, "l_mix": 2.7542034089565277, "l_a": 2.8428273499011993, "l_c": 2.790906399488449, "z_loss": 35.36915159225464, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11621042340993881, "tokens_per_second": 1418961.7849382574, "eta_seconds": 11150.74959731102, "data_wait_seconds": 1.9073486328125e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.20344992, "gate_a2b/L1_absmax": 0.73657203, "gate_a2b/L1_rms": 0.2161331, "gate_a2b/L3_absmean": 0.202843, "gate_a2b/L3_absmax": 0.58325362, "gate_a2b/L3_rms": 0.20903942, "gate_a2b/L5_absmean": 0.31431016, "gate_a2b/L5_absmax": 0.72738874, "gate_a2b/L5_rms": 0.32020858, "gate_c2b/L1_absmean": 0.01882289, "gate_c2b/L1_absmax": 0.63521445, "gate_c2b/L1_rms": 0.06765229, "gate_c2b/L3_absmean": 0.19841163, "gate_c2b/L3_absmax": 0.44109988, "gate_c2b/L3_rms": 0.20253946, "gate_c2b/L5_absmean": 0.56337392, "gate_c2b/L5_absmax": 0.78884834, "gate_c2b/L5_rms": 0.57558388, "gate_b2a/L7_absmean": 0.05214065, "gate_b2a/L7_absmax": 0.31805065, "gate_b2a/L7_rms": 0.05761056, "gate_b2a/L11_absmean": 0.08444498, "gate_b2a/L11_absmax": 0.59207296, "gate_b2a/L11_rms": 0.09295912, "gate_b2a/L14_absmean": 0.2168745, "gate_b2a/L14_absmax": 0.78214085, "gate_b2a/L14_rms": 0.22503518, "gate_b2c/L3_absmean": 0.03871289, "gate_b2c/L3_absmax": 0.58560622, "gate_b2c/L3_rms": 0.05029354, "gate_b2c/L5_absmean": 0.04216104, "gate_b2c/L5_absmax": 0.6081931, "gate_b2c/L5_rms": 0.05291751, "gate_b2c/L6_absmean": 0.06036676, "gate_b2c/L6_absmax": 0.4074716, "gate_b2c/L6_rms": 0.06765222}, "streams": {"cos_a2b/round0": 0.373255, "cos_c2b/round0": 0.322625, "cos_b2a/round0": 0.153029, "cos_b2c/round0": 0.135298, "rms_a/round0": 4.976352, "rms_b/round0": 9.218368, "rms_c/round0": 3.656706, "cos_a2b/round1": 0.424932, "cos_c2b/round1": 0.306018, "cos_b2a/round1": 0.249142, "cos_b2c/round1": 0.118862, "rms_a/round1": 9.532926, "rms_b/round1": 20.872177, "rms_c/round1": 7.437834, "cos_a2b/round2": 0.55794, "cos_c2b/round2": 0.528148, "cos_b2a/round2": 0.108006, "cos_b2c/round2": 0.177172, "rms_a/round2": 25.545649, "rms_b/round2": 54.495831, "rms_c/round2": 13.96695}, "alpha_mean": 0.4329858753128093, "alpha_hist": [0.00801, 0.06622, 0.16927, 0.20329, 0.20351, 0.16693, 0.11864, 0.05821, 0.00589, 3e-05], "alpha_by_type": {"special": {"share": 0.00111, "mean": 0.52613, "std": 0.08291}, "whitespace": {"share": 0.02307, "mean": 0.50155, "std": 0.10808}, "digit": {"share": 0.03714, "mean": 0.4377, "std": 0.11346}, "word_start": {"share": 0.61685, "mean": 0.44698, "std": 0.15804}, "word_cont": {"share": 0.22125, "mean": 0.31836, "std": 0.12766}, "punct": {"share": 0.10057, "mean": 0.58084, "std": 0.15353}}, "alpha_p05": 0.17526, "alpha_p50": 0.42417, "alpha_p95": 0.72194}
{"step": 42310, "total_steps": 72479, "tokens": 22182625280, "loss": 4.401036739349365, "l_mix": 2.7272729873657227, "l_a": 2.8118708431720734, "l_c": 2.7673412561416626, "z_loss": 34.973989486694336, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1687316745519638, "tokens_per_second": 1436688.2898611778, "eta_seconds": 11009.517362689972, "data_wait_seconds": 2.440810203552246e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 42320, "total_steps": 72479, "tokens": 22187868160, "loss": 4.447853684425354, "l_mix": 2.7572337687015533, "l_a": 2.846351385116577, "l_c": 2.789047986268997, "z_loss": 34.9403977394104, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2188216596841812, "tokens_per_second": 1403462.882342741, "eta_seconds": 11266.41964738369, "data_wait_seconds": 1.683831214904785e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 42330, "total_steps": 72479, "tokens": 22193111040, "loss": 2.877912253141403, "l_mix": 1.7800382524728775, "l_a": 1.8489200174808502, "l_c": 1.810659721493721, "z_loss": 44.63604116439819, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16594836115837097, "tokens_per_second": 1449412.7703000999, "eta_seconds": 10905.629670095443, "data_wait_seconds": 3.1888484954833984e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 42340, "total_steps": 72479, "tokens": 22198353920, "loss": 4.675917387008667, "l_mix": 2.901166170835495, "l_a": 2.975792795419693, "l_c": 2.9400444626808167, "z_loss": 34.22649002075195, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15108360350131989, "tokens_per_second": 1464104.2619977216, "eta_seconds": 10792.616647696494, "data_wait_seconds": 1.5467405319213867e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20768, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 42350, "total_steps": 72479, "tokens": 22203596800, "loss": 4.611144632101059, "l_mix": 2.8610360622406006, "l_a": 2.937429904937744, "l_c": 2.896265059709549, "z_loss": 34.59765577316284, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1390848010778427, "tokens_per_second": 1350530.804306594, "eta_seconds": 11696.344208979606, "data_wait_seconds": 1.77919864654541e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 42360, "total_steps": 72479, "tokens": 22208839680, "loss": 4.763529360294342, "l_mix": 2.954468786716461, "l_a": 3.033311367034912, "l_c": 2.9968902468681335, "z_loss": 34.73565340042114, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.22250336408615112, "tokens_per_second": 1350178.803504951, "eta_seconds": 11695.510424995422, "data_wait_seconds": 1.96993350982666e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 42370, "total_steps": 72479, "tokens": 22214082560, "loss": 4.6952515840530396, "l_mix": 2.9124039113521576, "l_a": 2.9936271607875824, "l_c": 2.9491982758045197, "z_loss": 35.50195550918579, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19209495186805725, "tokens_per_second": 1443268.912120817, "eta_seconds": 10937.52332599163, "data_wait_seconds": 1.9073486328125e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 42380, "total_steps": 72479, "tokens": 22219325440, "loss": 4.658640742301941, "l_mix": 2.8892429172992706, "l_a": 2.968491643667221, "l_c": 2.929500639438629, "z_loss": 35.14211368560791, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13999047875404358, "tokens_per_second": 1469888.4638183662, "eta_seconds": 10735.878878188134, "data_wait_seconds": 1.5854835510253906e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 42390, "total_steps": 72479, "tokens": 22224568320, "loss": 4.423576176166534, "l_mix": 2.7416412234306335, "l_a": 2.826943814754486, "l_c": 2.7795054614543915, "z_loss": 35.14887237548828, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1302470713853836, "tokens_per_second": 1439450.1630521729, "eta_seconds": 10959.25516347885, "data_wait_seconds": 1.6391277313232422e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 42400, "total_steps": 72479, "tokens": 22229811200, "loss": 4.261902034282684, "l_mix": 2.640219032764435, "l_a": 2.7302441895008087, "l_c": 2.6753656566143036, "z_loss": 35.696624755859375, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.10405950248241425, "tokens_per_second": 1469864.6873535959, "eta_seconds": 10728.918714547157, "data_wait_seconds": 1.424551010131836e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.20299359, "gate_a2b/L1_absmax": 0.7359699, "gate_a2b/L1_rms": 0.21562931, "gate_a2b/L3_absmean": 0.20233415, "gate_a2b/L3_absmax": 0.57944006, "gate_a2b/L3_rms": 0.20855738, "gate_a2b/L5_absmean": 0.3130261, "gate_a2b/L5_absmax": 0.7255981, "gate_a2b/L5_rms": 0.31891036, "gate_c2b/L1_absmean": 0.01778282, "gate_c2b/L1_absmax": 0.63604164, "gate_c2b/L1_rms": 0.06729881, "gate_c2b/L3_absmean": 0.19790122, "gate_c2b/L3_absmax": 0.44568852, "gate_c2b/L3_rms": 0.20200872, "gate_c2b/L5_absmean": 0.56303543, "gate_c2b/L5_absmax": 0.79440445, "gate_c2b/L5_rms": 0.57530999, "gate_b2a/L7_absmean": 0.05215812, "gate_b2a/L7_absmax": 0.31705266, "gate_b2a/L7_rms": 0.05753471, "gate_b2a/L11_absmean": 0.08426365, "gate_b2a/L11_absmax": 0.58987904, "gate_b2a/L11_rms": 0.09276816, "gate_b2a/L14_absmean": 0.21676628, "gate_b2a/L14_absmax": 0.78157854, "gate_b2a/L14_rms": 0.22495021, "gate_b2c/L3_absmean": 0.03855811, "gate_b2c/L3_absmax": 0.58542567, "gate_b2c/L3_rms": 0.05004898, "gate_b2c/L5_absmean": 0.04171833, "gate_b2c/L5_absmax": 0.60192782, "gate_b2c/L5_rms": 0.05238762, "gate_b2c/L6_absmean": 0.06000198, "gate_b2c/L6_absmax": 0.40697652, "gate_b2c/L6_rms": 0.06734584}, "streams": {"cos_a2b/round0": 0.377207, "cos_c2b/round0": 0.316725, "cos_b2a/round0": 0.14141, "cos_b2c/round0": 0.132271, "rms_a/round0": 5.008433, "rms_b/round0": 9.163524, "rms_c/round0": 3.65758, "cos_a2b/round1": 0.414128, "cos_c2b/round1": 0.293666, "cos_b2a/round1": 0.240967, "cos_b2c/round1": 0.109089, "rms_a/round1": 9.467255, "rms_b/round1": 20.790112, "rms_c/round1": 7.413459, "cos_a2b/round2": 0.534178, "cos_c2b/round2": 0.534157, "cos_b2a/round2": 0.102743, "cos_b2c/round2": 0.173417, "rms_a/round2": 25.220472, "rms_b/round2": 54.872578, "rms_c/round2": 14.001399}, "alpha_mean": 0.41744379624022443, "alpha_hist": [0.00832, 0.06978, 0.18134, 0.22389, 0.22084, 0.14542, 0.09801, 0.04756, 0.00481, 5e-05], "alpha_by_type": {"special": {"share": 0.00096, "mean": 0.52395, "std": 0.11562}, "whitespace": {"share": 0.0349, "mean": 0.4829, "std": 0.11937}, "digit": {"share": 0.07565, "mean": 0.42988, "std": 0.08173}, "word_start": {"share": 0.53046, "mean": 0.43345, "std": 0.16204}, "word_cont": {"share": 0.23022, "mean": 0.30955, "std": 0.12136}, "punct": {"share": 0.12781, "mean": 0.51932, "std": 0.15149}}, "alpha_p05": 0.16749, "alpha_p50": 0.42155, "alpha_p95": 0.69513}
{"step": 42410, "total_steps": 72479, "tokens": 22235054080, "loss": 4.43368262052536, "l_mix": 2.746182680130005, "l_a": 2.8383549451828003, "l_c": 2.786644995212555, "z_loss": 35.78006649017334, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.10830219835042953, "tokens_per_second": 1428414.5847654408, "eta_seconds": 11036.582824158668, "data_wait_seconds": 1.722574234008789e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 42420, "total_steps": 72479, "tokens": 22240296960, "loss": 4.371076047420502, "l_mix": 2.7086336612701416, "l_a": 2.793921619653702, "l_c": 2.7475529313087463, "z_loss": 35.63908386230469, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1357516199350357, "tokens_per_second": 1312796.1740736184, "eta_seconds": 12004.584796357154, "data_wait_seconds": 1.5228986740112305e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 42430, "total_steps": 72479, "tokens": 22245539840, "loss": 4.457356154918671, "l_mix": 2.762870728969574, "l_a": 2.850646823644638, "l_c": 2.7976375818252563, "z_loss": 35.053030014038086, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13080184161663055, "tokens_per_second": 1470337.4134945592, "eta_seconds": 10714.771975064277, "data_wait_seconds": 2.3990869522094727e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 42440, "total_steps": 72479, "tokens": 22250782720, "loss": 3.1545919477939606, "l_mix": 1.9522401839494705, "l_a": 2.026098817586899, "l_c": 1.981740117073059, "z_loss": 42.60572528839111, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18558397889137268, "tokens_per_second": 1443960.831024487, "eta_seconds": 10906.865957593916, "data_wait_seconds": 1.773238182067871e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 42450, "total_steps": 72479, "tokens": 22256025600, "loss": 2.890285938978195, "l_mix": 1.7894039899110794, "l_a": 1.853671818971634, "l_c": 1.81593456864357, "z_loss": 42.30559253692627, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2353464514017105, "tokens_per_second": 1396194.6925937512, "eta_seconds": 11276.252828860283, "data_wait_seconds": 2.1219253540039062e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08236}}
{"step": 42460, "total_steps": 72479, "tokens": 22261268480, "loss": 4.410867512226105, "l_mix": 2.732232242822647, "l_a": 2.8237531185150146, "l_c": 2.7716974318027496, "z_loss": 35.964781761169434, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1358715444803238, "tokens_per_second": 1464571.143639476, "eta_seconds": 10746.218468356134, "data_wait_seconds": 1.5348196029663086e-05, "mix": {"fineweb_hq": 0.50959, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 42470, "total_steps": 72479, "tokens": 22266511360, "loss": 4.464848816394806, "l_mix": 2.7661598324775696, "l_a": 2.8585771024227142, "l_c": 2.8037191033363342, "z_loss": 35.08305311203003, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15640589594841003, "tokens_per_second": 1360167.622063702, "eta_seconds": 11567.220346069336, "data_wait_seconds": 1.817941665649414e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 42480, "total_steps": 72479, "tokens": 22271754240, "loss": 4.412169873714447, "l_mix": 2.734099954366684, "l_a": 2.8237337470054626, "l_c": 2.7698323130607605, "z_loss": 34.85955572128296, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12305991351604462, "tokens_per_second": 1442702.3948029147, "eta_seconds": 10901.843490839005, "data_wait_seconds": 3.489851951599121e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 42490, "total_steps": 72479, "tokens": 22276997120, "loss": 4.435626983642578, "l_mix": 2.7485449612140656, "l_a": 2.8389316499233246, "l_c": 2.7846746742725372, "z_loss": 35.00356721878052, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11815448850393295, "tokens_per_second": 1468259.0034586182, "eta_seconds": 10708.51450252533, "data_wait_seconds": 1.9997358322143555e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 42500, "total_steps": 72479, "tokens": 22282240000, "loss": 4.463877201080322, "l_mix": 2.7663141191005707, "l_a": 2.8537562787532806, "l_c": 2.8047870695590973, "z_loss": 34.982155323028564, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11247977614402771, "tokens_per_second": 1402511.3780175918, "eta_seconds": 11206.775359082223, "data_wait_seconds": 1.430511474609375e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.20234016, "gate_a2b/L1_absmax": 0.73916197, "gate_a2b/L1_rms": 0.21501783, "gate_a2b/L3_absmean": 0.20137873, "gate_a2b/L3_absmax": 0.58230519, "gate_a2b/L3_rms": 0.20762344, "gate_a2b/L5_absmean": 0.3116799, "gate_a2b/L5_absmax": 0.72679037, "gate_a2b/L5_rms": 0.3176592, "gate_c2b/L1_absmean": 0.02012189, "gate_c2b/L1_absmax": 0.63877183, "gate_c2b/L1_rms": 0.06814429, "gate_c2b/L3_absmean": 0.19782268, "gate_c2b/L3_absmax": 0.4481028, "gate_c2b/L3_rms": 0.2019296, "gate_c2b/L5_absmean": 0.56271279, "gate_c2b/L5_absmax": 0.79199404, "gate_c2b/L5_rms": 0.57491535, "gate_b2a/L7_absmean": 0.05205408, "gate_b2a/L7_absmax": 0.31894311, "gate_b2a/L7_rms": 0.05750467, "gate_b2a/L11_absmean": 0.08426444, "gate_b2a/L11_absmax": 0.59346366, "gate_b2a/L11_rms": 0.09284315, "gate_b2a/L14_absmean": 0.2159811, "gate_b2a/L14_absmax": 0.78884411, "gate_b2a/L14_rms": 0.22430135, "gate_b2c/L3_absmean": 0.03792674, "gate_b2c/L3_absmax": 0.58419788, "gate_b2c/L3_rms": 0.04962918, "gate_b2c/L5_absmean": 0.04155218, "gate_b2c/L5_absmax": 0.60823673, "gate_b2c/L5_rms": 0.05245697, "gate_b2c/L6_absmean": 0.05965606, "gate_b2c/L6_absmax": 0.41326731, "gate_b2c/L6_rms": 0.06711742}, "streams": {"cos_a2b/round0": 0.372475, "cos_c2b/round0": 0.312804, "cos_b2a/round0": 0.148167, "cos_b2c/round0": 0.133242, "rms_a/round0": 4.98717, "rms_b/round0": 9.435429, "rms_c/round0": 3.715407, "cos_a2b/round1": 0.422858, "cos_c2b/round1": 0.304182, "cos_b2a/round1": 0.252404, "cos_b2c/round1": 0.112845, "rms_a/round1": 9.51767, "rms_b/round1": 21.162092, "rms_c/round1": 7.417929, "cos_a2b/round2": 0.542205, "cos_c2b/round2": 0.529229, "cos_b2a/round2": 0.107119, "cos_b2c/round2": 0.181704, "rms_a/round2": 25.399471, "rms_b/round2": 55.441612, "rms_c/round2": 13.771426}, "alpha_mean": 0.4208157519793261, "alpha_hist": [0.00906, 0.0907, 0.20296, 0.19711, 0.16122, 0.14281, 0.12196, 0.0658, 0.00836, 2e-05], "alpha_by_type": {"special": {"share": 0.00067, "mean": 0.56017, "std": 0.12203}, "whitespace": {"share": 0.01857, "mean": 0.49831, "std": 0.11474}, "digit": {"share": 0.02441, "mean": 0.41617, "std": 0.12048}, "word_start": {"share": 0.59586, "mean": 0.4417, "std": 0.17445}, "word_cont": {"share": 0.26707, "mean": 0.31059, "std": 0.12335}, "punct": {"share": 0.09341, "mean": 0.58759, "std": 0.16116}}, "alpha_p05": 0.16212, "alpha_p50": 0.39981, "alpha_p95": 0.73048}
{"step": 42510, "total_steps": 72479, "tokens": 22287482880, "loss": 4.304429233074188, "l_mix": 2.6668293178081512, "l_a": 2.7542525231838226, "l_c": 2.7044135332107544, "z_loss": 35.82617235183716, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11699189990758896, "tokens_per_second": 931475.142203913, "eta_seconds": 16868.28382217884, "data_wait_seconds": 2.446770668029785e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 42520, "total_steps": 72479, "tokens": 22292725760, "loss": 4.455006539821625, "l_mix": 2.7603993713855743, "l_a": 2.850383311510086, "l_c": 2.798306882381439, "z_loss": 35.4598331451416, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.10661289840936661, "tokens_per_second": 1462236.402460388, "eta_seconds": 10741.863740754126, "data_wait_seconds": 1.6480684280395508e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 42530, "total_steps": 72479, "tokens": 22297968640, "loss": 4.3561707735061646, "l_mix": 2.698824554681778, "l_a": 2.788355678319931, "l_c": 2.73613178730011, "z_loss": 36.11612796783447, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1450948417186737, "tokens_per_second": 1429802.5141449147, "eta_seconds": 10981.867185616493, "data_wait_seconds": 2.0176172256469727e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 42540, "total_steps": 72479, "tokens": 22303211520, "loss": 4.44908994436264, "l_mix": 2.7565430402755737, "l_a": 2.847191095352173, "l_c": 2.7946315705776215, "z_loss": 35.68289136886597, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1795843541622162, "tokens_per_second": 1321471.7756096148, "eta_seconds": 11878.163969683646, "data_wait_seconds": 1.9937753677368164e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 42550, "total_steps": 72479, "tokens": 22308454400, "loss": 3.0114975571632385, "l_mix": 1.8643986284732819, "l_a": 1.9314487427473068, "l_c": 1.8922141343355179, "z_loss": 41.97697734832764, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15634982287883759, "tokens_per_second": 1414634.379775587, "eta_seconds": 11092.20571501255, "data_wait_seconds": 2.1904706954956055e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 42560, "total_steps": 72479, "tokens": 22313697280, "loss": 3.0158695578575134, "l_mix": 1.8655633479356766, "l_a": 1.9377300143241882, "l_c": 1.8966239243745804, "z_loss": 42.993911266326904, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19842082262039185, "tokens_per_second": 1400042.2462908663, "eta_seconds": 11204.070958256721, "data_wait_seconds": 2.6494264602661133e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 42570, "total_steps": 72479, "tokens": 22318940160, "loss": 4.428993046283722, "l_mix": 2.7445854544639587, "l_a": 2.832284063100815, "l_c": 2.7824078500270844, "z_loss": 35.448912143707275, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1142980083823204, "tokens_per_second": 1144072.0916746354, "eta_seconds": 13706.242732524872, "data_wait_seconds": 1.952052116394043e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 42580, "total_steps": 72479, "tokens": 22324183040, "loss": 4.36305433511734, "l_mix": 2.7030690908432007, "l_a": 2.7933923602104187, "l_c": 2.7398916482925415, "z_loss": 35.34904432296753, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11546865850687027, "tokens_per_second": 1326330.608293779, "eta_seconds": 11818.838239860535, "data_wait_seconds": 1.9490718841552734e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 42590, "total_steps": 72479, "tokens": 22329425920, "loss": 4.369298279285431, "l_mix": 2.7080611288547516, "l_a": 2.792900562286377, "l_c": 2.74455663561821, "z_loss": 35.10342836380005, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13183757662773132, "tokens_per_second": 1320306.8768720285, "eta_seconds": 11868.78922355175, "data_wait_seconds": 1.7017126083374023e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 42600, "total_steps": 72479, "tokens": 22334668800, "loss": 4.364727139472961, "l_mix": 2.704452097415924, "l_a": 2.7930099070072174, "l_c": 2.7412401735782623, "z_loss": 35.98990774154663, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15460141003131866, "tokens_per_second": 1263075.0466293197, "eta_seconds": 12402.431030368805, "data_wait_seconds": 2.086162567138672e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.20203848, "gate_a2b/L1_absmax": 0.73823231, "gate_a2b/L1_rms": 0.21474822, "gate_a2b/L3_absmean": 0.20068021, "gate_a2b/L3_absmax": 0.5807879, "gate_a2b/L3_rms": 0.20685291, "gate_a2b/L5_absmean": 0.31097153, "gate_a2b/L5_absmax": 0.72609395, "gate_a2b/L5_rms": 0.31686884, "gate_c2b/L1_absmean": 0.01844611, "gate_c2b/L1_absmax": 0.63688564, "gate_c2b/L1_rms": 0.06762005, "gate_c2b/L3_absmean": 0.19811909, "gate_c2b/L3_absmax": 0.44760767, "gate_c2b/L3_rms": 0.20219643, "gate_c2b/L5_absmean": 0.56248397, "gate_c2b/L5_absmax": 0.79479986, "gate_c2b/L5_rms": 0.57467985, "gate_b2a/L7_absmean": 0.05239866, "gate_b2a/L7_absmax": 0.31791422, "gate_b2a/L7_rms": 0.05773481, "gate_b2a/L11_absmean": 0.08416946, "gate_b2a/L11_absmax": 0.5932954, "gate_b2a/L11_rms": 0.09274684, "gate_b2a/L14_absmean": 0.2160684, "gate_b2a/L14_absmax": 0.78523785, "gate_b2a/L14_rms": 0.22433673, "gate_b2c/L3_absmean": 0.03828982, "gate_b2c/L3_absmax": 0.58293116, "gate_b2c/L3_rms": 0.04981993, "gate_b2c/L5_absmean": 0.04175892, "gate_b2c/L5_absmax": 0.60746342, "gate_b2c/L5_rms": 0.05251411, "gate_b2c/L6_absmean": 0.05984986, "gate_b2c/L6_absmax": 0.40814146, "gate_b2c/L6_rms": 0.06717426}, "streams": {"cos_a2b/round0": 0.370645, "cos_c2b/round0": 0.310928, "cos_b2a/round0": 0.151308, "cos_b2c/round0": 0.130893, "rms_a/round0": 5.057008, "rms_b/round0": 9.409659, "rms_c/round0": 3.715952, "cos_a2b/round1": 0.420507, "cos_c2b/round1": 0.303668, "cos_b2a/round1": 0.252602, "cos_b2c/round1": 0.11769, "rms_a/round1": 9.621912, "rms_b/round1": 21.172295, "rms_c/round1": 7.499749, "cos_a2b/round2": 0.554358, "cos_c2b/round2": 0.52269, "cos_b2a/round2": 0.103482, "cos_b2c/round2": 0.179336, "rms_a/round2": 25.874193, "rms_b/round2": 55.224812, "rms_c/round2": 14.090091}, "alpha_mean": 0.4465790701332395, "alpha_hist": [0.00533, 0.06082, 0.17065, 0.19255, 0.18356, 0.1678, 0.13445, 0.07364, 0.01106, 0.00014], "alpha_by_type": {"special": {"share": 0.00127, "mean": 0.5972, "std": 0.1127}, "whitespace": {"share": 0.02249, "mean": 0.55675, "std": 0.11523}, "digit": {"share": 0.02473, "mean": 0.49331, "std": 0.10285}, "word_start": {"share": 0.58305, "mean": 0.46128, "std": 0.16694}, "word_cont": {"share": 0.25832, "mean": 0.33854, "std": 0.12884}, "punct": {"share": 0.11014, "mean": 0.58743, "std": 0.16244}}, "alpha_p05": 0.18892, "alpha_p50": 0.43494, "alpha_p95": 0.74017}
{"step": 42610, "total_steps": 72479, "tokens": 22339911680, "loss": 4.417765140533447, "l_mix": 2.7374366521835327, "l_a": 2.824479430913925, "l_c": 2.7766156792640686, "z_loss": 35.681931495666504, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11770959943532944, "tokens_per_second": 1400312.736739341, "eta_seconds": 11183.18634197712, "data_wait_seconds": 1.895427703857422e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 42620, "total_steps": 72479, "tokens": 22345154560, "loss": 4.638333559036255, "l_mix": 2.876923084259033, "l_a": 2.95515239238739, "l_c": 2.916215866804123, "z_loss": 34.59543800354004, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1425408571958542, "tokens_per_second": 1439186.5714200074, "eta_seconds": 10877.474611616133, "data_wait_seconds": 2.1576881408691406e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 42630, "total_steps": 72479, "tokens": 22350397440, "loss": 4.654251337051392, "l_mix": 2.887470543384552, "l_a": 2.9647907316684723, "l_c": 2.9244785606861115, "z_loss": 34.31726598739624, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12678571045398712, "tokens_per_second": 1299030.9624223085, "eta_seconds": 12047.03580183983, "data_wait_seconds": 1.683831214904785e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 42640, "total_steps": 72479, "tokens": 22355640320, "loss": 4.7885419726371765, "l_mix": 2.9706145226955414, "l_a": 3.0517771244049072, "l_c": 3.007980763912201, "z_loss": 34.83993482589722, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15047629177570343, "tokens_per_second": 1446195.7762401688, "eta_seconds": 10817.504717564583, "data_wait_seconds": 6.091594696044922e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 42650, "total_steps": 72479, "tokens": 22360883200, "loss": 4.653741538524628, "l_mix": 2.8863602578639984, "l_a": 2.966783970594406, "l_c": 2.9244864881038666, "z_loss": 35.28714323043823, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12388618290424347, "tokens_per_second": 1421418.68207376, "eta_seconds": 11002.378784823417, "data_wait_seconds": 2.0951032638549805e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 42660, "total_steps": 72479, "tokens": 22366126080, "loss": 4.682511687278748, "l_mix": 2.904750108718872, "l_a": 2.983854740858078, "l_c": 2.942016899585724, "z_loss": 35.02200937271118, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1194382756948471, "tokens_per_second": 1479298.917642266, "eta_seconds": 10568.346725296973, "data_wait_seconds": 2.059340476989746e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 42670, "total_steps": 72479, "tokens": 22371368960, "loss": 3.0432835817337036, "l_mix": 1.8842252641916275, "l_a": 1.9521262794733047, "l_c": 1.9114012718200684, "z_loss": 42.63329601287842, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13280640542507172, "tokens_per_second": 1344787.1553160145, "eta_seconds": 11621.542435336114, "data_wait_seconds": 1.9758939743041992e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 42680, "total_steps": 72479, "tokens": 22376611840, "loss": 4.35997748374939, "l_mix": 2.701666533946991, "l_a": 2.789953798055649, "l_c": 2.737749010324478, "z_loss": 36.40117597579956, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16929389536380768, "tokens_per_second": 1310455.444033805, "eta_seconds": 11922.00633995533, "data_wait_seconds": 2.6017427444458008e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 42690, "total_steps": 72479, "tokens": 22381854720, "loss": 4.351350367069244, "l_mix": 2.696805328130722, "l_a": 2.778904438018799, "l_c": 2.7362453043460846, "z_loss": 36.19421148300171, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13564421236515045, "tokens_per_second": 1320418.8977458964, "eta_seconds": 11828.07611937523, "data_wait_seconds": 1.6182661056518555e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 42700, "total_steps": 72479, "tokens": 22387097600, "loss": 4.452267587184906, "l_mix": 2.7587830126285553, "l_a": 2.844537913799286, "l_c": 2.800410211086273, "z_loss": 35.60829734802246, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12045055627822876, "tokens_per_second": 1305228.0302750345, "eta_seconds": 11961.720090174676, "data_wait_seconds": 2.586841583251953e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.2024031, "gate_a2b/L1_absmax": 0.73772442, "gate_a2b/L1_rms": 0.21500601, "gate_a2b/L3_absmean": 0.20078817, "gate_a2b/L3_absmax": 0.58076417, "gate_a2b/L3_rms": 0.20686841, "gate_a2b/L5_absmean": 0.31023714, "gate_a2b/L5_absmax": 0.7242502, "gate_a2b/L5_rms": 0.31616017, "gate_c2b/L1_absmean": 0.01795319, "gate_c2b/L1_absmax": 0.6360352, "gate_c2b/L1_rms": 0.0672901, "gate_c2b/L3_absmean": 0.19744428, "gate_c2b/L3_absmax": 0.45190379, "gate_c2b/L3_rms": 0.20156102, "gate_c2b/L5_absmean": 0.56150979, "gate_c2b/L5_absmax": 0.79196626, "gate_c2b/L5_rms": 0.57368588, "gate_b2a/L7_absmean": 0.05197774, "gate_b2a/L7_absmax": 0.3192507, "gate_b2a/L7_rms": 0.05742689, "gate_b2a/L11_absmean": 0.08394749, "gate_b2a/L11_absmax": 0.59470063, "gate_b2a/L11_rms": 0.09255358, "gate_b2a/L14_absmean": 0.2162272, "gate_b2a/L14_absmax": 0.78341001, "gate_b2a/L14_rms": 0.22440371, "gate_b2c/L3_absmean": 0.03812931, "gate_b2c/L3_absmax": 0.58271003, "gate_b2c/L3_rms": 0.04964355, "gate_b2c/L5_absmean": 0.04168268, "gate_b2c/L5_absmax": 0.60793525, "gate_b2c/L5_rms": 0.05246197, "gate_b2c/L6_absmean": 0.05973431, "gate_b2c/L6_absmax": 0.40505436, "gate_b2c/L6_rms": 0.06697036}, "streams": {"cos_a2b/round0": 0.374279, "cos_c2b/round0": 0.31123, "cos_b2a/round0": 0.147446, "cos_b2c/round0": 0.132855, "rms_a/round0": 5.104929, "rms_b/round0": 9.439303, "rms_c/round0": 3.713576, "cos_a2b/round1": 0.416374, "cos_c2b/round1": 0.295034, "cos_b2a/round1": 0.247226, "cos_b2c/round1": 0.113409, "rms_a/round1": 9.729881, "rms_b/round1": 21.370312, "rms_c/round1": 7.603175, "cos_a2b/round2": 0.547854, "cos_c2b/round2": 0.523217, "cos_b2a/round2": 0.102208, "cos_b2c/round2": 0.17393, "rms_a/round2": 26.117722, "rms_b/round2": 55.813778, "rms_c/round2": 14.353536}, "alpha_mean": 0.4293732092798237, "alpha_hist": [0.00914, 0.06873, 0.17737, 0.20746, 0.19196, 0.16586, 0.11317, 0.05794, 0.00832, 6e-05], "alpha_by_type": {"special": {"share": 0.0013, "mean": 0.54526, "std": 0.12807}, "whitespace": {"share": 0.0303, "mean": 0.51109, "std": 0.10904}, "digit": {"share": 0.04903, "mean": 0.44885, "std": 0.107}, "word_start": {"share": 0.55635, "mean": 0.44242, "std": 0.16327}, "word_cont": {"share": 0.24706, "mean": 0.31954, "std": 0.12987}, "punct": {"share": 0.11597, "mean": 0.56989, "std": 0.15288}}, "alpha_p05": 0.16831, "alpha_p50": 0.4237, "alpha_p95": 0.72526}
{"step": 42710, "total_steps": 72479, "tokens": 22392340480, "loss": 4.475584983825684, "l_mix": 2.773604154586792, "l_a": 2.862961560487747, "l_c": 2.810308039188385, "z_loss": 35.60516309738159, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14335770905017853, "tokens_per_second": 1471880.5552123731, "eta_seconds": 10603.801658177377, "data_wait_seconds": 2.9146671295166016e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 42720, "total_steps": 72479, "tokens": 22397583360, "loss": 4.389793515205383, "l_mix": 2.720145523548126, "l_a": 2.8060303926467896, "l_c": 2.7594630420207977, "z_loss": 35.87798881530762, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1340896636247635, "tokens_per_second": 1397006.4580860883, "eta_seconds": 11168.371127915383, "data_wait_seconds": 2.041459083557129e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 42730, "total_steps": 72479, "tokens": 22402826240, "loss": 3.057145744562149, "l_mix": 1.892031118273735, "l_a": 1.9632300734519958, "l_c": 1.920485332608223, "z_loss": 43.78167390823364, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16450071334838867, "tokens_per_second": 1391086.9249662354, "eta_seconds": 11212.12731719017, "data_wait_seconds": 2.434849739074707e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 42740, "total_steps": 72479, "tokens": 22408069120, "loss": 4.724538624286652, "l_mix": 2.9318720400333405, "l_a": 3.008455276489258, "l_c": 2.9671000242233276, "z_loss": 34.856749057769775, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15118058025836945, "tokens_per_second": 1475102.0338899598, "eta_seconds": 10569.981244540215, "data_wait_seconds": 3.0100345611572266e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 42750, "total_steps": 72479, "tokens": 22413312000, "loss": 4.758675217628479, "l_mix": 2.9521512389183044, "l_a": 3.030757963657379, "l_c": 2.9909884333610535, "z_loss": 35.142866134643555, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14088696241378784, "tokens_per_second": 1432607.7593046369, "eta_seconds": 10879.85029451847, "data_wait_seconds": 2.1696090698242188e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 42760, "total_steps": 72479, "tokens": 22418554880, "loss": 4.776740729808807, "l_mix": 2.9639141261577606, "l_a": 3.0413810908794403, "l_c": 3.001374125480652, "z_loss": 34.097185134887695, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1382884830236435, "tokens_per_second": 1300159.2193687737, "eta_seconds": 11984.159201335908, "data_wait_seconds": 2.4139881134033203e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 42770, "total_steps": 72479, "tokens": 22423797760, "loss": 4.630917727947235, "l_mix": 2.8726307451725006, "l_a": 2.9532856047153473, "l_c": 2.907670795917511, "z_loss": 34.09947466850281, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12000531703233719, "tokens_per_second": 1437359.2555198357, "eta_seconds": 10836.589483237267, "data_wait_seconds": 3.68654727935791e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 42780, "total_steps": 72479, "tokens": 22429040640, "loss": 3.056546837091446, "l_mix": 1.8920659273862839, "l_a": 1.961186870932579, "l_c": 1.9204159528017044, "z_loss": 44.06197166442871, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.21685339510440826, "tokens_per_second": 1346142.0067171715, "eta_seconds": 11567.003506541252, "data_wait_seconds": 3.0219554901123047e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 42790, "total_steps": 72479, "tokens": 22434283520, "loss": 4.3767284750938416, "l_mix": 2.7122118175029755, "l_a": 2.7985758185386658, "l_c": 2.749813050031662, "z_loss": 35.97566509246826, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12905894219875336, "tokens_per_second": 1459917.7193710643, "eta_seconds": 10661.96144170761, "data_wait_seconds": 1.6421079635620117e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 42800, "total_steps": 72479, "tokens": 22439526400, "loss": 4.367318272590637, "l_mix": 2.707057148218155, "l_a": 2.7926509082317352, "l_c": 2.7415526807308197, "z_loss": 35.70183801651001, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14898304641246796, "tokens_per_second": 1132972.614374411, "eta_seconds": 13734.086203479766, "data_wait_seconds": 3.483891487121582e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.2020978, "gate_a2b/L1_absmax": 0.73653805, "gate_a2b/L1_rms": 0.21470337, "gate_a2b/L3_absmean": 0.20034026, "gate_a2b/L3_absmax": 0.57833928, "gate_a2b/L3_rms": 0.20648889, "gate_a2b/L5_absmean": 0.31014657, "gate_a2b/L5_absmax": 0.72556961, "gate_a2b/L5_rms": 0.31606373, "gate_c2b/L1_absmean": 0.01758417, "gate_c2b/L1_absmax": 0.63443369, "gate_c2b/L1_rms": 0.06717235, "gate_c2b/L3_absmean": 0.19672669, "gate_c2b/L3_absmax": 0.44422609, "gate_c2b/L3_rms": 0.20082071, "gate_c2b/L5_absmean": 0.55992025, "gate_c2b/L5_absmax": 0.79280955, "gate_c2b/L5_rms": 0.57206047, "gate_b2a/L7_absmean": 0.05233315, "gate_b2a/L7_absmax": 0.31873587, "gate_b2a/L7_rms": 0.05771204, "gate_b2a/L11_absmean": 0.08402669, "gate_b2a/L11_absmax": 0.5948602, "gate_b2a/L11_rms": 0.0926572, "gate_b2a/L14_absmean": 0.21565868, "gate_b2a/L14_absmax": 0.78234279, "gate_b2a/L14_rms": 0.22388312, "gate_b2c/L3_absmean": 0.03812988, "gate_b2c/L3_absmax": 0.58197039, "gate_b2c/L3_rms": 0.04958962, "gate_b2c/L5_absmean": 0.04167714, "gate_b2c/L5_absmax": 0.60795105, "gate_b2c/L5_rms": 0.05248226, "gate_b2c/L6_absmean": 0.05977605, "gate_b2c/L6_absmax": 0.40322119, "gate_b2c/L6_rms": 0.06703255}, "streams": {"cos_a2b/round0": 0.376895, "cos_c2b/round0": 0.330443, "cos_b2a/round0": 0.148672, "cos_b2c/round0": 0.139124, "rms_a/round0": 5.062773, "rms_b/round0": 9.334921, "rms_c/round0": 3.689649, "cos_a2b/round1": 0.423609, "cos_c2b/round1": 0.29411, "cos_b2a/round1": 0.251463, "cos_b2c/round1": 0.10711, "rms_a/round1": 9.593841, "rms_b/round1": 21.08499, "rms_c/round1": 7.450377, "cos_a2b/round2": 0.552192, "cos_c2b/round2": 0.52834, "cos_b2a/round2": 0.101522, "cos_b2c/round2": 0.171506, "rms_a/round2": 25.857296, "rms_b/round2": 55.646412, "rms_c/round2": 13.942256}, "alpha_mean": 0.4415887304119366, "alpha_hist": [0.00566, 0.05415, 0.17084, 0.2034, 0.19273, 0.17683, 0.12889, 0.05925, 0.00807, 0.00017], "alpha_by_type": {"special": {"share": 0.00067, "mean": 0.5838, "std": 0.1235}, "whitespace": {"share": 0.03038, "mean": 0.5246, "std": 0.12564}, "digit": {"share": 0.05136, "mean": 0.42572, "std": 0.12062}, "word_start": {"share": 0.58325, "mean": 0.45409, "std": 0.16025}, "word_cont": {"share": 0.2296, "mean": 0.33873, "std": 0.12835}, "punct": {"share": 0.10474, "mean": 0.58027, "std": 0.15705}}, "alpha_p05": 0.18684, "alpha_p50": 0.42131, "alpha_p95": 0.72115}
{"step": 42810, "total_steps": 72479, "tokens": 22444769280, "loss": 4.41599577665329, "l_mix": 2.7364843487739563, "l_a": 2.8210330307483673, "l_c": 2.7773381173610687, "z_loss": 36.31845140457153, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.10496332496404648, "tokens_per_second": 1414945.9537233275, "eta_seconds": 10993.423905038833, "data_wait_seconds": 1.728534698486328e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 42820, "total_steps": 72479, "tokens": 22450012160, "loss": 4.446437418460846, "l_mix": 2.755479246377945, "l_a": 2.842411160469055, "l_c": 2.7941161394119263, "z_loss": 35.466959953308105, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.10061395913362503, "tokens_per_second": 1485493.8938348535, "eta_seconds": 10467.80323805809, "data_wait_seconds": 1.850724220275879e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 42830, "total_steps": 72479, "tokens": 22455255040, "loss": 4.407240569591522, "l_mix": 2.7304535508155823, "l_a": 2.8180099427700043, "l_c": 2.771279960870743, "z_loss": 37.21634006500244, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18161512911319733, "tokens_per_second": 1466145.697439266, "eta_seconds": 10602.367103862764, "data_wait_seconds": 1.4662742614746094e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 42840, "total_steps": 72479, "tokens": 22460497920, "loss": 2.833262711763382, "l_mix": 1.7535385489463806, "l_a": 1.8176067620515823, "l_c": 1.781473696231842, "z_loss": 44.88919496536255, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18354175984859467, "tokens_per_second": 1460070.0982612136, "eta_seconds": 10642.89450931549, "data_wait_seconds": 1.7911195755004883e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 42850, "total_steps": 72479, "tokens": 22465740800, "loss": 3.1198242902755737, "l_mix": 1.9204824715852737, "l_a": 2.0307427644729614, "l_c": 1.967063069343567, "z_loss": 38.092116832733154, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2269192934036255, "tokens_per_second": 1390058.4472952175, "eta_seconds": 11175.16258559227, "data_wait_seconds": 2.5659799575805664e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 42860, "total_steps": 72479, "tokens": 22470983680, "loss": 3.173933357000351, "l_mix": 1.9531216025352478, "l_a": 2.06747068464756, "l_c": 2.0019015222787857, "z_loss": 38.72958278656006, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2098160833120346, "tokens_per_second": 1415580.9952581434, "eta_seconds": 10969.973688554765, "data_wait_seconds": 4.786252975463867e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 42870, "total_steps": 72479, "tokens": 22476226560, "loss": 3.181437611579895, "l_mix": 1.9587138891220093, "l_a": 2.0688668191432953, "l_c": 2.0068786293268204, "z_loss": 37.980355739593506, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16238220036029816, "tokens_per_second": 1381175.1619672978, "eta_seconds": 11239.445813584327, "data_wait_seconds": 1.8715858459472656e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 42880, "total_steps": 72479, "tokens": 22481469440, "loss": 4.704373419284821, "l_mix": 2.917345851659775, "l_a": 2.999680310487747, "l_c": 2.9570780098438263, "z_loss": 34.827781677246094, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15157553553581238, "tokens_per_second": 1399226.9499535728, "eta_seconds": 11090.695839238166, "data_wait_seconds": 2.047419548034668e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 42890, "total_steps": 72479, "tokens": 22486712320, "loss": 4.592524170875549, "l_mix": 2.8487228453159332, "l_a": 2.9291741847991943, "l_c": 2.8834966719150543, "z_loss": 34.826175689697266, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13913758099079132, "tokens_per_second": 1464540.4185917277, "eta_seconds": 10592.509045886993, "data_wait_seconds": 1.6391277313232422e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 42900, "total_steps": 72479, "tokens": 22491955200, "loss": 4.432759463787079, "l_mix": 2.746370941400528, "l_a": 2.835090845823288, "l_c": 2.7862039506435394, "z_loss": 36.42693567276001, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11948001384735107, "tokens_per_second": 1431681.677280357, "eta_seconds": 10831.957269620894, "data_wait_seconds": 1.481175422668457e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.20163499, "gate_a2b/L1_absmax": 0.73477906, "gate_a2b/L1_rms": 0.21415488, "gate_a2b/L3_absmean": 0.19983195, "gate_a2b/L3_absmax": 0.57823652, "gate_a2b/L3_rms": 0.20595941, "gate_a2b/L5_absmean": 0.30873647, "gate_a2b/L5_absmax": 0.72517014, "gate_a2b/L5_rms": 0.31466398, "gate_c2b/L1_absmean": 0.01802188, "gate_c2b/L1_absmax": 0.63390654, "gate_c2b/L1_rms": 0.06724492, "gate_c2b/L3_absmean": 0.19660647, "gate_c2b/L3_absmax": 0.44938418, "gate_c2b/L3_rms": 0.20075913, "gate_c2b/L5_absmean": 0.55958593, "gate_c2b/L5_absmax": 0.79805648, "gate_c2b/L5_rms": 0.57175952, "gate_b2a/L7_absmean": 0.05188923, "gate_b2a/L7_absmax": 0.31897318, "gate_b2a/L7_rms": 0.05734678, "gate_b2a/L11_absmean": 0.08389334, "gate_b2a/L11_absmax": 0.59570873, "gate_b2a/L11_rms": 0.09250307, "gate_b2a/L14_absmean": 0.21579868, "gate_b2a/L14_absmax": 0.78112406, "gate_b2a/L14_rms": 0.22400735, "gate_b2c/L3_absmean": 0.03839418, "gate_b2c/L3_absmax": 0.58133006, "gate_b2c/L3_rms": 0.04976155, "gate_b2c/L5_absmean": 0.04151339, "gate_b2c/L5_absmax": 0.60568619, "gate_b2c/L5_rms": 0.05228617, "gate_b2c/L6_absmean": 0.05979647, "gate_b2c/L6_absmax": 0.40169483, "gate_b2c/L6_rms": 0.06699285}, "streams": {"cos_a2b/round0": 0.372084, "cos_c2b/round0": 0.310729, "cos_b2a/round0": 0.147736, "cos_b2c/round0": 0.133741, "rms_a/round0": 5.087725, "rms_b/round0": 9.513378, "rms_c/round0": 3.729544, "cos_a2b/round1": 0.419118, "cos_c2b/round1": 0.303028, "cos_b2a/round1": 0.247597, "cos_b2c/round1": 0.117696, "rms_a/round1": 9.699356, "rms_b/round1": 21.348743, "rms_c/round1": 7.526414, "cos_a2b/round2": 0.555009, "cos_c2b/round2": 0.514582, "cos_b2a/round2": 0.105941, "cos_b2c/round2": 0.178642, "rms_a/round2": 26.198925, "rms_b/round2": 55.540699, "rms_c/round2": 13.922416}, "alpha_mean": 0.44197192945321717, "alpha_hist": [0.00867, 0.08092, 0.17372, 0.18137, 0.17104, 0.15431, 0.13245, 0.08386, 0.01357, 0.00011], "alpha_by_type": {"special": {"share": 0.0009, "mean": 0.56438, "std": 0.08167}, "whitespace": {"share": 0.02538, "mean": 0.52371, "std": 0.12524}, "digit": {"share": 0.03676, "mean": 0.44832, "std": 0.11212}, "word_start": {"share": 0.59129, "mean": 0.45904, "std": 0.17588}, "word_cont": {"share": 0.24574, "mean": 0.32163, "std": 0.13593}, "punct": {"share": 0.09993, "mean": 0.6127, "std": 0.16097}}, "alpha_p05": 0.17441, "alpha_p50": 0.42895, "alpha_p95": 0.7364}
{"step": 42910, "total_steps": 72479, "tokens": 22497198080, "loss": 4.469024240970612, "l_mix": 2.7699341773986816, "l_a": 2.851972281932831, "l_c": 2.8116610646247864, "z_loss": 36.21460580825806, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1370125412940979, "tokens_per_second": 1425947.149310845, "eta_seconds": 10871.84183473587, "data_wait_seconds": 1.5795230865478516e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 42920, "total_steps": 72479, "tokens": 22502440960, "loss": 4.55176842212677, "l_mix": 2.8213116824626923, "l_a": 2.909121334552765, "l_c": 2.859067738056183, "z_loss": 35.902974128723145, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11347091943025589, "tokens_per_second": 1305131.1978621476, "eta_seconds": 11874.230742001535, "data_wait_seconds": 2.0951032638549805e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 42930, "total_steps": 72479, "tokens": 22507683840, "loss": 4.490887820720673, "l_mix": 2.7831590473651886, "l_a": 2.8695168495178223, "l_c": 2.8229119777679443, "z_loss": 35.926658153533936, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12119203805923462, "tokens_per_second": 1427955.2601815413, "eta_seconds": 10849.209736466408, "data_wait_seconds": 1.7195940017700195e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 42940, "total_steps": 72479, "tokens": 22512926720, "loss": 4.393506944179535, "l_mix": 2.7224296629428864, "l_a": 2.808435618877411, "l_c": 2.7618215680122375, "z_loss": 36.27901077270508, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.10589366406202316, "tokens_per_second": 1467652.1315149234, "eta_seconds": 10552.189377474784, "data_wait_seconds": 1.6927719116210938e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 42950, "total_steps": 72479, "tokens": 22518169600, "loss": 3.001602381467819, "l_mix": 1.8580508530139923, "l_a": 1.9268998205661774, "l_c": 1.8849385678768158, "z_loss": 43.724520683288574, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17677544057369232, "tokens_per_second": 1414368.3352790608, "eta_seconds": 10946.017360424996, "data_wait_seconds": 2.828240394592285e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 42960, "total_steps": 72479, "tokens": 22523412480, "loss": 3.1243729293346405, "l_mix": 1.9209598004817963, "l_a": 2.0384591966867447, "l_c": 1.9729175716638565, "z_loss": 39.64563751220703, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.21451176702976227, "tokens_per_second": 1351840.4972654874, "eta_seconds": 11448.434562587738, "data_wait_seconds": 1.6421079635620117e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 42970, "total_steps": 72479, "tokens": 22528655360, "loss": 3.143202096223831, "l_mix": 1.9342845231294632, "l_a": 2.0464904606342316, "l_c": 1.9832345396280289, "z_loss": 38.460294246673584, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1864105463027954, "tokens_per_second": 1462285.6030707273, "eta_seconds": 10580.159279084206, "data_wait_seconds": 3.072619438171387e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 42980, "total_steps": 72479, "tokens": 22533898240, "loss": 3.19729882478714, "l_mix": 1.9681230634450912, "l_a": 2.0823117196559906, "l_c": 2.014940559864044, "z_loss": 37.136760234832764, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16255763173103333, "tokens_per_second": 1430293.3550418217, "eta_seconds": 10813.14658806324, "data_wait_seconds": 3.0487775802612305e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 42990, "total_steps": 72479, "tokens": 22539141120, "loss": 3.177663654088974, "l_mix": 1.9547833055257797, "l_a": 2.070736050605774, "l_c": 2.0055314153432846, "z_loss": 38.29520606994629, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.184400275349617, "tokens_per_second": 1445186.2272987731, "eta_seconds": 10698.087582039832, "data_wait_seconds": 2.9593706130981445e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 43000, "total_steps": 72479, "tokens": 22544384000, "loss": 4.382606148719788, "l_mix": 2.7152296900749207, "l_a": 2.805222451686859, "l_c": 2.7526990473270416, "z_loss": 36.33574438095093, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1354505717754364, "tokens_per_second": 1389060.7948350478, "eta_seconds": 11126.57272415161, "data_wait_seconds": 9.965896606445312e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.20134929, "gate_a2b/L1_absmax": 0.73432475, "gate_a2b/L1_rms": 0.21383372, "gate_a2b/L3_absmean": 0.19942532, "gate_a2b/L3_absmax": 0.5804137, "gate_a2b/L3_rms": 0.205542, "gate_a2b/L5_absmean": 0.30813381, "gate_a2b/L5_absmax": 0.72620243, "gate_a2b/L5_rms": 0.31407365, "gate_c2b/L1_absmean": 0.01789745, "gate_c2b/L1_absmax": 0.63412672, "gate_c2b/L1_rms": 0.06712127, "gate_c2b/L3_absmean": 0.1965393, "gate_c2b/L3_absmax": 0.44192448, "gate_c2b/L3_rms": 0.20059539, "gate_c2b/L5_absmean": 0.5586077, "gate_c2b/L5_absmax": 0.78376234, "gate_c2b/L5_rms": 0.57068723, "gate_b2a/L7_absmean": 0.05181002, "gate_b2a/L7_absmax": 0.319051, "gate_b2a/L7_rms": 0.05725275, "gate_b2a/L11_absmean": 0.08382308, "gate_b2a/L11_absmax": 0.59460527, "gate_b2a/L11_rms": 0.09245532, "gate_b2a/L14_absmean": 0.2152395, "gate_b2a/L14_absmax": 0.78284371, "gate_b2a/L14_rms": 0.22347414, "gate_b2c/L3_absmean": 0.03808364, "gate_b2c/L3_absmax": 0.58118421, "gate_b2c/L3_rms": 0.04951797, "gate_b2c/L5_absmean": 0.04155842, "gate_b2c/L5_absmax": 0.60449624, "gate_b2c/L5_rms": 0.05229531, "gate_b2c/L6_absmean": 0.05942638, "gate_b2c/L6_absmax": 0.40013757, "gate_b2c/L6_rms": 0.06662973}, "streams": {"cos_a2b/round0": 0.369463, "cos_c2b/round0": 0.309147, "cos_b2a/round0": 0.140891, "cos_b2c/round0": 0.129122, "rms_a/round0": 5.119579, "rms_b/round0": 9.530451, "rms_c/round0": 3.74743, "cos_a2b/round1": 0.415347, "cos_c2b/round1": 0.294865, "cos_b2a/round1": 0.245133, "cos_b2c/round1": 0.109654, "rms_a/round1": 9.688409, "rms_b/round1": 21.441572, "rms_c/round1": 7.575135, "cos_a2b/round2": 0.542304, "cos_c2b/round2": 0.527211, "cos_b2a/round2": 0.10186, "cos_b2c/round2": 0.170647, "rms_a/round2": 26.001802, "rms_b/round2": 56.439621, "rms_c/round2": 14.053645}, "alpha_mean": 0.4287985261445755, "alpha_hist": [0.00833, 0.06175, 0.17909, 0.22839, 0.19234, 0.14842, 0.10918, 0.06332, 0.00902, 0.00015], "alpha_by_type": {"special": {"share": 0.00067, "mean": 0.54728, "std": 0.12403}, "whitespace": {"share": 0.02368, "mean": 0.52299, "std": 0.13005}, "digit": {"share": 0.04805, "mean": 0.43037, "std": 0.09835}, "word_start": {"share": 0.5172, "mean": 0.44675, "std": 0.1645}, "word_cont": {"share": 0.29172, "mean": 0.32825, "std": 0.12077}, "punct": {"share": 0.11868, "mean": 0.57762, "std": 0.15976}}, "alpha_p05": 0.17983, "alpha_p50": 0.41466, "alpha_p95": 0.72952}
{"step": 43010, "total_steps": 72479, "tokens": 22549626880, "loss": 4.43080735206604, "l_mix": 2.745651215314865, "l_a": 2.832912862300873, "l_c": 2.784274160861969, "z_loss": 35.86601686477661, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14697964489459991, "tokens_per_second": 1464865.7793799574, "eta_seconds": 10547.207320618629, "data_wait_seconds": 2.3752450942993164e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 43020, "total_steps": 72479, "tokens": 22554869760, "loss": 4.4818742871284485, "l_mix": 2.7776617407798767, "l_a": 2.865834891796112, "l_c": 2.8148733973503113, "z_loss": 36.20528173446655, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17137916386127472, "tokens_per_second": 1460508.8983183736, "eta_seconds": 10575.081199288368, "data_wait_seconds": 1.558661460876465e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 43030, "total_steps": 72479, "tokens": 22560112640, "loss": 4.379518568515778, "l_mix": 2.713281214237213, "l_a": 2.801903188228607, "l_c": 2.75222110748291, "z_loss": 37.00673294067383, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1247137039899826, "tokens_per_second": 1358314.3093421573, "eta_seconds": 11366.85169684887, "data_wait_seconds": 1.6987323760986328e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 43040, "total_steps": 72479, "tokens": 22565355520, "loss": 4.365426421165466, "l_mix": 2.704791784286499, "l_a": 2.791531801223755, "l_c": 2.7439169585704803, "z_loss": 35.8414740562439, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13916932046413422, "tokens_per_second": 1435900.5528596158, "eta_seconds": 10749.013503241538, "data_wait_seconds": 1.7315149307250977e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 43050, "total_steps": 72479, "tokens": 22570598400, "loss": 4.478549122810364, "l_mix": 2.77591136097908, "l_a": 2.863193541765213, "l_c": 2.812265306711197, "z_loss": 36.205824851989746, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.191995307803154, "tokens_per_second": 1402826.9410556087, "eta_seconds": 10998.69919834137, "data_wait_seconds": 1.671910285949707e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 43060, "total_steps": 72479, "tokens": 22575841280, "loss": 2.938455492258072, "l_mix": 1.8181252479553223, "l_a": 1.8883363902568817, "l_c": 1.8460975587368011, "z_loss": 45.31948947906494, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20317231118679047, "tokens_per_second": 1432933.931205172, "eta_seconds": 10763.949639344215, "data_wait_seconds": 3.0159950256347656e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 43070, "total_steps": 72479, "tokens": 22581084160, "loss": 3.137629956007004, "l_mix": 1.930263414978981, "l_a": 2.0442242324352264, "l_c": 1.9803306609392166, "z_loss": 37.96678924560547, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1702972799539566, "tokens_per_second": 1259271.931802356, "eta_seconds": 12244.206674194334, "data_wait_seconds": 2.2917985916137695e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 43080, "total_steps": 72479, "tokens": 22586327040, "loss": 3.166508048772812, "l_mix": 1.9469821751117706, "l_a": 2.066231817007065, "l_c": 1.9988542348146439, "z_loss": 40.43938112258911, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2158498764038086, "tokens_per_second": 1473435.9234700517, "eta_seconds": 10460.952299642562, "data_wait_seconds": 1.8894672393798828e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 43090, "total_steps": 72479, "tokens": 22591569920, "loss": 3.131675273180008, "l_mix": 1.9272627979516983, "l_a": 2.0401033461093903, "l_c": 1.9746047407388687, "z_loss": 39.188579082489014, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19249732792377472, "tokens_per_second": 1457144.1188767876, "eta_seconds": 10574.31439511776, "data_wait_seconds": 2.911686897277832e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 43100, "total_steps": 72479, "tokens": 22596812800, "loss": 3.122376888990402, "l_mix": 1.9204798340797424, "l_a": 2.035104900598526, "l_c": 1.9712185561656952, "z_loss": 37.74670743942261, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1600489616394043, "tokens_per_second": 1429556.3836051445, "eta_seconds": 10774.711182188988, "data_wait_seconds": 2.6434659957885742e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.20132728, "gate_a2b/L1_absmax": 0.73252076, "gate_a2b/L1_rms": 0.21380737, "gate_a2b/L3_absmean": 0.19880818, "gate_a2b/L3_absmax": 0.57951111, "gate_a2b/L3_rms": 0.20490685, "gate_a2b/L5_absmean": 0.30757198, "gate_a2b/L5_absmax": 0.726601, "gate_a2b/L5_rms": 0.3134678, "gate_c2b/L1_absmean": 0.01806846, "gate_c2b/L1_absmax": 0.63349867, "gate_c2b/L1_rms": 0.06702543, "gate_c2b/L3_absmean": 0.19658236, "gate_c2b/L3_absmax": 0.44152278, "gate_c2b/L3_rms": 0.20063651, "gate_c2b/L5_absmean": 0.55824852, "gate_c2b/L5_absmax": 0.78874034, "gate_c2b/L5_rms": 0.57033688, "gate_b2a/L7_absmean": 0.05182589, "gate_b2a/L7_absmax": 0.31968379, "gate_b2a/L7_rms": 0.05732663, "gate_b2a/L11_absmean": 0.08381545, "gate_b2a/L11_absmax": 0.59639442, "gate_b2a/L11_rms": 0.09252754, "gate_b2a/L14_absmean": 0.21541977, "gate_b2a/L14_absmax": 0.78304464, "gate_b2a/L14_rms": 0.22363496, "gate_b2c/L3_absmean": 0.03789394, "gate_b2c/L3_absmax": 0.58145857, "gate_b2c/L3_rms": 0.04939365, "gate_b2c/L5_absmean": 0.04144233, "gate_b2c/L5_absmax": 0.60653317, "gate_b2c/L5_rms": 0.05224126, "gate_b2c/L6_absmean": 0.05965554, "gate_b2c/L6_absmax": 0.39938527, "gate_b2c/L6_rms": 0.0667827}, "streams": {"cos_a2b/round0": 0.368751, "cos_c2b/round0": 0.27259, "cos_b2a/round0": 0.133864, "cos_b2c/round0": 0.106234, "rms_a/round0": 5.327319, "rms_b/round0": 9.713581, "rms_c/round0": 3.8465, "cos_a2b/round1": 0.415364, "cos_c2b/round1": 0.301559, "cos_b2a/round1": 0.240003, "cos_b2c/round1": 0.119038, "rms_a/round1": 10.258052, "rms_b/round1": 22.644388, "rms_c/round1": 8.032206, "cos_a2b/round2": 0.520732, "cos_c2b/round2": 0.509569, "cos_b2a/round2": 0.082968, "cos_b2c/round2": 0.1661, "rms_a/round2": 26.738558, "rms_b/round2": 57.517815, "rms_c/round2": 15.079342}, "alpha_mean": 0.40846793281969695, "alpha_hist": [0.03107, 0.16537, 0.19589, 0.15274, 0.11737, 0.10634, 0.10081, 0.08488, 0.04175, 0.00377], "alpha_by_type": {"special": {"share": 0.00102, "mean": 0.59292, "std": 0.12661}, "whitespace": {"share": 0.0403, "mean": 0.62133, "std": 0.16625}, "digit": {"share": 0.01474, "mean": 0.41184, "std": 0.16944}, "word_start": {"share": 0.55212, "mean": 0.41348, "std": 0.20968}, "word_cont": {"share": 0.27367, "mean": 0.27678, "std": 0.13979}, "punct": {"share": 0.11815, "mean": 0.61543, "std": 0.18917}}, "alpha_p05": 0.11757, "alpha_p50": 0.36207, "alpha_p95": 0.78332}
{"step": 43110, "total_steps": 72479, "tokens": 22602055680, "loss": 4.434171378612518, "l_mix": 2.747704565525055, "l_a": 2.8348370492458344, "l_c": 2.7867188155651093, "z_loss": 36.29046392440796, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11871365457773209, "tokens_per_second": 1422997.0690401238, "eta_seconds": 10820.692893195153, "data_wait_seconds": 2.6434659957885742e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 43120, "total_steps": 72479, "tokens": 22607298560, "loss": 4.362499952316284, "l_mix": 2.7028084695339203, "l_a": 2.790727823972702, "l_c": 2.7415770888328552, "z_loss": 37.42917251586914, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15081697702407837, "tokens_per_second": 1318452.4586911749, "eta_seconds": 11674.72614619732, "data_wait_seconds": 1.952052116394043e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 43130, "total_steps": 72479, "tokens": 22612541440, "loss": 4.438193202018738, "l_mix": 2.7504881024360657, "l_a": 2.834773004055023, "l_c": 2.790910542011261, "z_loss": 35.72212553024292, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1639392077922821, "tokens_per_second": 1441142.4419903704, "eta_seconds": 10677.17393066883, "data_wait_seconds": 1.8090009689331055e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 43140, "total_steps": 72479, "tokens": 22617784320, "loss": 4.40223240852356, "l_mix": 2.7278691828250885, "l_a": 2.816497266292572, "l_c": 2.7647132575511932, "z_loss": 35.83503723144531, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16288577020168304, "tokens_per_second": 1310021.2320991722, "eta_seconds": 11741.859792113304, "data_wait_seconds": 2.0444393157958984e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 43150, "total_steps": 72479, "tokens": 22623027200, "loss": 3.1684928238391876, "l_mix": 1.9489548802375793, "l_a": 2.0654936134815216, "l_c": 1.9996327310800552, "z_loss": 37.33778095245361, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19505715370178223, "tokens_per_second": 1386817.1499045824, "eta_seconds": 11087.8660197258, "data_wait_seconds": 2.4080276489257812e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 43160, "total_steps": 72479, "tokens": 22628270080, "loss": 4.942621529102325, "l_mix": 3.067584276199341, "l_a": 3.145790785551071, "l_c": 3.104332983493805, "z_loss": 33.55340790748596, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1402764767408371, "tokens_per_second": 1311871.1663980312, "eta_seconds": 11717.309035921096, "data_wait_seconds": 2.0444393157958984e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 43170, "total_steps": 72479, "tokens": 22633512960, "loss": 4.723962068557739, "l_mix": 2.9294963777065277, "l_a": 3.013518810272217, "l_c": 2.9680332839488983, "z_loss": 35.69583797454834, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17258651554584503, "tokens_per_second": 1450734.4343264282, "eta_seconds": 10592.122602462769, "data_wait_seconds": 4.348158836364746e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 43180, "total_steps": 72479, "tokens": 22638755840, "loss": 3.1475750505924225, "l_mix": 1.9371886253356934, "l_a": 2.0502171367406845, "l_c": 1.9844040721654892, "z_loss": 36.518115520477295, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18130812048912048, "tokens_per_second": 1405201.1161179082, "eta_seconds": 10931.612518525124, "data_wait_seconds": 3.6269426345825195e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 43190, "total_steps": 72479, "tokens": 22643998720, "loss": 3.1563864648342133, "l_mix": 1.9420811086893082, "l_a": 2.0549114644527435, "l_c": 1.9927729368209839, "z_loss": 37.71559429168701, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16479018330574036, "tokens_per_second": 1464879.2456997521, "eta_seconds": 10482.687414050102, "data_wait_seconds": 2.2292137145996094e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 43200, "total_steps": 72479, "tokens": 22649241600, "loss": 3.17078560590744, "l_mix": 1.9506949931383133, "l_a": 2.0676871240139008, "l_c": 1.9992814362049103, "z_loss": 38.56332588195801, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15437830984592438, "tokens_per_second": 1440883.894981839, "eta_seconds": 10653.619216275216, "data_wait_seconds": 1.5914440155029297e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.20048366, "gate_a2b/L1_absmax": 0.73245049, "gate_a2b/L1_rms": 0.21309705, "gate_a2b/L3_absmean": 0.19823742, "gate_a2b/L3_absmax": 0.5810976, "gate_a2b/L3_rms": 0.20443147, "gate_a2b/L5_absmean": 0.30674759, "gate_a2b/L5_absmax": 0.72837281, "gate_a2b/L5_rms": 0.31269103, "gate_c2b/L1_absmean": 0.01823869, "gate_c2b/L1_absmax": 0.63222873, "gate_c2b/L1_rms": 0.06697221, "gate_c2b/L3_absmean": 0.19580963, "gate_c2b/L3_absmax": 0.44487163, "gate_c2b/L3_rms": 0.1999, "gate_c2b/L5_absmean": 0.55768204, "gate_c2b/L5_absmax": 0.78120148, "gate_c2b/L5_rms": 0.56978345, "gate_b2a/L7_absmean": 0.0514601, "gate_b2a/L7_absmax": 0.32013983, "gate_b2a/L7_rms": 0.0570129, "gate_b2a/L11_absmean": 0.08378008, "gate_b2a/L11_absmax": 0.59679651, "gate_b2a/L11_rms": 0.09250847, "gate_b2a/L14_absmean": 0.21534029, "gate_b2a/L14_absmax": 0.7817682, "gate_b2a/L14_rms": 0.22355595, "gate_b2c/L3_absmean": 0.03816813, "gate_b2c/L3_absmax": 0.58170325, "gate_b2c/L3_rms": 0.0496061, "gate_b2c/L5_absmean": 0.04137378, "gate_b2c/L5_absmax": 0.60677731, "gate_b2c/L5_rms": 0.05221552, "gate_b2c/L6_absmean": 0.05941237, "gate_b2c/L6_absmax": 0.39943412, "gate_b2c/L6_rms": 0.06660562}, "streams": {"cos_a2b/round0": 0.364623, "cos_c2b/round0": 0.270747, "cos_b2a/round0": 0.138793, "cos_b2c/round0": 0.103538, "rms_a/round0": 5.325947, "rms_b/round0": 9.695748, "rms_c/round0": 3.859095, "cos_a2b/round1": 0.411823, "cos_c2b/round1": 0.300065, "cos_b2a/round1": 0.236958, "cos_b2c/round1": 0.118088, "rms_a/round1": 10.292857, "rms_b/round1": 22.517, "rms_c/round1": 7.9838, "cos_a2b/round2": 0.527139, "cos_c2b/round2": 0.509766, "cos_b2a/round2": 0.088168, "cos_b2c/round2": 0.165499, "rms_a/round2": 26.847124, "rms_b/round2": 57.145691, "rms_c/round2": 14.995879}, "alpha_mean": 0.4283036880381488, "alpha_hist": [0.0303, 0.13306, 0.18279, 0.15721, 0.136, 0.1068, 0.10341, 0.09375, 0.05324, 0.00345], "alpha_by_type": {"special": {"share": 0.00104, "mean": 0.65312, "std": 0.11189}, "whitespace": {"share": 0.03484, "mean": 0.62567, "std": 0.15975}, "digit": {"share": 0.00923, "mean": 0.39909, "std": 0.17847}, "word_start": {"share": 0.55696, "mean": 0.4445, "std": 0.20923}, "word_cont": {"share": 0.24385, "mean": 0.27463, "std": 0.14831}, "punct": {"share": 0.15408, "mean": 0.56859, "std": 0.19902}}, "alpha_p05": 0.1192, "alpha_p50": 0.39327, "alpha_p95": 0.80225}
{"step": 43210, "total_steps": 72479, "tokens": 22654484480, "loss": 3.1433723270893097, "l_mix": 1.9337337017059326, "l_a": 2.0484984517097473, "l_c": 1.9836302399635315, "z_loss": 39.31547212600708, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1552177369594574, "tokens_per_second": 1471482.4534110765, "eta_seconds": 10428.520867800713, "data_wait_seconds": 3.013014793395996e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 43220, "total_steps": 72479, "tokens": 22659727360, "loss": 4.4061625599861145, "l_mix": 2.7303726077079773, "l_a": 2.8183961510658264, "l_c": 2.767570376396179, "z_loss": 36.174535274505615, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13107658922672272, "tokens_per_second": 1457336.6751775376, "eta_seconds": 10526.148729586603, "data_wait_seconds": 2.1696090698242188e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 43230, "total_steps": 72479, "tokens": 22664970240, "loss": 4.491214632987976, "l_mix": 2.7828530967235565, "l_a": 2.8699265122413635, "l_c": 2.8246116936206818, "z_loss": 36.93857526779175, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16033118963241577, "tokens_per_second": 1436057.712290872, "eta_seconds": 10678.470357251166, "data_wait_seconds": 1.6957521438598633e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 43240, "total_steps": 72479, "tokens": 22670213120, "loss": 4.4132121205329895, "l_mix": 2.734653264284134, "l_a": 2.8222950100898743, "l_c": 2.77290078997612, "z_loss": 36.19632530212402, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11118572950363159, "tokens_per_second": 1381030.3052572235, "eta_seconds": 11100.159622597694, "data_wait_seconds": 2.2143125534057617e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 43250, "total_steps": 72479, "tokens": 22675456000, "loss": 4.432901382446289, "l_mix": 2.7460777163505554, "l_a": 2.8368672728538513, "l_c": 2.78587806224823, "z_loss": 35.714778900146484, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14930948615074158, "tokens_per_second": 1364779.9485050822, "eta_seconds": 11228.487031030654, "data_wait_seconds": 1.621246337890625e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 43260, "total_steps": 72479, "tokens": 22680698880, "loss": 3.184841424226761, "l_mix": 1.9592637419700623, "l_a": 2.0773281157016754, "l_c": 2.007930666208267, "z_loss": 38.379446506500244, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17353761196136475, "tokens_per_second": 1477601.6739470873, "eta_seconds": 10367.591849756242, "data_wait_seconds": 1.6033649444580078e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 43270, "total_steps": 72479, "tokens": 22685941760, "loss": 3.212693840265274, "l_mix": 1.9771502017974854, "l_a": 2.091838300228119, "l_c": 2.0266403406858444, "z_loss": 38.2564058303833, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13899417221546173, "tokens_per_second": 1451345.7842956015, "eta_seconds": 10551.536620497704, "data_wait_seconds": 1.7434358596801758e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 43280, "total_steps": 72479, "tokens": 22691184640, "loss": 3.1856532394886017, "l_mix": 1.9586467891931534, "l_a": 2.078236401081085, "l_c": 2.0117849558591843, "z_loss": 38.951908588409424, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19883230328559875, "tokens_per_second": 1389962.764405067, "eta_seconds": 11013.737708687782, "data_wait_seconds": 3.1620264053344727e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 43290, "total_steps": 72479, "tokens": 22696427520, "loss": 3.1444280445575714, "l_mix": 1.9355334490537643, "l_a": 2.047047972679138, "l_c": 1.982600450515747, "z_loss": 37.133601665496826, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16661784052848816, "tokens_per_second": 1432849.1533032688, "eta_seconds": 10680.428150248526, "data_wait_seconds": 1.8328428268432617e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 43300, "total_steps": 72479, "tokens": 22701670400, "loss": 3.157709628343582, "l_mix": 1.942260131239891, "l_a": 2.0570382475852966, "l_c": 1.9944598227739334, "z_loss": 38.200932025909424, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1699693351984024, "tokens_per_second": 1401281.1713592627, "eta_seconds": 10917.294733333589, "data_wait_seconds": 1.7076730728149414e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.20021562, "gate_a2b/L1_absmax": 0.73308694, "gate_a2b/L1_rms": 0.21278575, "gate_a2b/L3_absmean": 0.19847003, "gate_a2b/L3_absmax": 0.58147293, "gate_a2b/L3_rms": 0.20461853, "gate_a2b/L5_absmean": 0.30630061, "gate_a2b/L5_absmax": 0.72765255, "gate_a2b/L5_rms": 0.31224531, "gate_c2b/L1_absmean": 0.01772222, "gate_c2b/L1_absmax": 0.6321364, "gate_c2b/L1_rms": 0.06676839, "gate_c2b/L3_absmean": 0.19590428, "gate_c2b/L3_absmax": 0.44088617, "gate_c2b/L3_rms": 0.20001055, "gate_c2b/L5_absmean": 0.55690402, "gate_c2b/L5_absmax": 0.78850871, "gate_c2b/L5_rms": 0.56894284, "gate_b2a/L7_absmean": 0.05214205, "gate_b2a/L7_absmax": 0.31928372, "gate_b2a/L7_rms": 0.05750487, "gate_b2a/L11_absmean": 0.08387193, "gate_b2a/L11_absmax": 0.59581316, "gate_b2a/L11_rms": 0.09254519, "gate_b2a/L14_absmean": 0.21516213, "gate_b2a/L14_absmax": 0.78096527, "gate_b2a/L14_rms": 0.22335079, "gate_b2c/L3_absmean": 0.03820986, "gate_b2c/L3_absmax": 0.58023739, "gate_b2c/L3_rms": 0.04956561, "gate_b2c/L5_absmean": 0.04136391, "gate_b2c/L5_absmax": 0.6053378, "gate_b2c/L5_rms": 0.05213768, "gate_b2c/L6_absmean": 0.05920657, "gate_b2c/L6_absmax": 0.39837095, "gate_b2c/L6_rms": 0.06638494}, "streams": {"cos_a2b/round0": 0.365699, "cos_c2b/round0": 0.269325, "cos_b2a/round0": 0.135426, "cos_b2c/round0": 0.105737, "rms_a/round0": 5.368134, "rms_b/round0": 9.709332, "rms_c/round0": 3.850859, "cos_a2b/round1": 0.41425, "cos_c2b/round1": 0.302368, "cos_b2a/round1": 0.237131, "cos_b2c/round1": 0.120981, "rms_a/round1": 10.447552, "rms_b/round1": 22.926863, "rms_c/round1": 8.133471, "cos_a2b/round2": 0.520331, "cos_c2b/round2": 0.512087, "cos_b2a/round2": 0.085761, "cos_b2c/round2": 0.168023, "rms_a/round2": 27.203915, "rms_b/round2": 58.362198, "rms_c/round2": 15.431981}, "alpha_mean": 0.41219256107913793, "alpha_hist": [0.04158, 0.16786, 0.19096, 0.14462, 0.11063, 0.09509, 0.09505, 0.09132, 0.05667, 0.00621], "alpha_by_type": {"special": {"share": 0.00107, "mean": 0.61817, "std": 0.10157}, "whitespace": {"share": 0.03291, "mean": 0.64895, "std": 0.17728}, "digit": {"share": 0.01109, "mean": 0.39497, "std": 0.17062}, "word_start": {"share": 0.57417, "mean": 0.42084, "std": 0.21781}, "word_cont": {"share": 0.26418, "mean": 0.26311, "std": 0.14255}, "punct": {"share": 0.11658, "mean": 0.64034, "std": 0.19117}}, "alpha_p05": 0.11757, "alpha_p50": 0.34953, "alpha_p95": 0.81227}
{"step": 43310, "total_steps": 72479, "tokens": 22706913280, "loss": 3.1274127066135406, "l_mix": 1.9255236834287643, "l_a": 2.0355731546878815, "l_c": 1.970723569393158, "z_loss": 38.43264627456665, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20609618723392487, "tokens_per_second": 1485356.0274637332, "eta_seconds": 10295.818907546996, "data_wait_seconds": 2.5838613510131836e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 43320, "total_steps": 72479, "tokens": 22712156160, "loss": 3.164354622364044, "l_mix": 1.947016417980194, "l_a": 2.060990869998932, "l_c": 1.9968029260635376, "z_loss": 39.03978490829468, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15583808720111847, "tokens_per_second": 1463413.7491771297, "eta_seconds": 10446.610741901397, "data_wait_seconds": 1.8864870071411133e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 43330, "total_steps": 72479, "tokens": 22717399040, "loss": 4.434886157512665, "l_mix": 2.747788965702057, "l_a": 2.83979931473732, "l_c": 2.783857583999634, "z_loss": 36.26288557052612, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13279980421066284, "tokens_per_second": 1452251.0680780893, "eta_seconds": 10523.298104524612, "data_wait_seconds": 1.728534698486328e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 43340, "total_steps": 72479, "tokens": 22722641920, "loss": 4.392105340957642, "l_mix": 2.721770703792572, "l_a": 2.8063720166683197, "l_c": 2.76141020655632, "z_loss": 35.86962699890137, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11304884403944016, "tokens_per_second": 1347656.328810939, "eta_seconds": 11336.145355010032, "data_wait_seconds": 2.1010637283325195e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 43350, "total_steps": 72479, "tokens": 22727884800, "loss": 4.586603820323944, "l_mix": 2.843312293291092, "l_a": 2.929478883743286, "l_c": 2.8814927339553833, "z_loss": 35.107093334198, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15665815770626068, "tokens_per_second": 1337007.4742438784, "eta_seconds": 11422.512922477721, "data_wait_seconds": 2.2113323211669922e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 43360, "total_steps": 72479, "tokens": 22733127680, "loss": 3.21467262506485, "l_mix": 1.9791333377361298, "l_a": 2.0894463658332825, "l_c": 2.029017686843872, "z_loss": 37.96830081939697, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1475432813167572, "tokens_per_second": 1414344.7746025212, "eta_seconds": 10794.215488433838, "data_wait_seconds": 1.7851591110229492e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 43370, "total_steps": 72479, "tokens": 22738370560, "loss": 3.1914005875587463, "l_mix": 1.9635330438613892, "l_a": 2.07958647608757, "l_c": 2.0133051574230194, "z_loss": 39.17307233810425, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16715039312839508, "tokens_per_second": 1443070.963988115, "eta_seconds": 10575.709561657906, "data_wait_seconds": 1.665949821472168e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 43380, "total_steps": 72479, "tokens": 22743613440, "loss": 3.1883470118045807, "l_mix": 1.961658388376236, "l_a": 2.0778338611125946, "l_c": 2.0111281275749207, "z_loss": 38.609896659851074, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1698920875787735, "tokens_per_second": 1468464.903159804, "eta_seconds": 10389.255118846893, "data_wait_seconds": 1.6123056411743164e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 43390, "total_steps": 72479, "tokens": 22748856320, "loss": 3.168008506298065, "l_mix": 1.9506770819425583, "l_a": 2.058760941028595, "l_c": 1.9990103244781494, "z_loss": 38.272135734558105, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2129514068365097, "tokens_per_second": 1374759.3449838005, "eta_seconds": 11093.587897872925, "data_wait_seconds": 3.153085708618164e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 43400, "total_steps": 72479, "tokens": 22754099200, "loss": 3.1733424365520477, "l_mix": 1.951921969652176, "l_a": 2.0678353011608124, "l_c": 2.0035659968852997, "z_loss": 38.07487344741821, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18540465831756592, "tokens_per_second": 1420559.4022059846, "eta_seconds": 10732.230365252495, "data_wait_seconds": 3.1322240829467773e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.2000424, "gate_a2b/L1_absmax": 0.73150879, "gate_a2b/L1_rms": 0.21251592, "gate_a2b/L3_absmean": 0.19812505, "gate_a2b/L3_absmax": 0.58051181, "gate_a2b/L3_rms": 0.20421062, "gate_a2b/L5_absmean": 0.30583307, "gate_a2b/L5_absmax": 0.72796541, "gate_a2b/L5_rms": 0.31173158, "gate_c2b/L1_absmean": 0.01849016, "gate_c2b/L1_absmax": 0.63088661, "gate_c2b/L1_rms": 0.06679685, "gate_c2b/L3_absmean": 0.19540548, "gate_c2b/L3_absmax": 0.44247544, "gate_c2b/L3_rms": 0.1994922, "gate_c2b/L5_absmean": 0.55571699, "gate_c2b/L5_absmax": 0.78050035, "gate_c2b/L5_rms": 0.56768018, "gate_b2a/L7_absmean": 0.05185044, "gate_b2a/L7_absmax": 0.31870446, "gate_b2a/L7_rms": 0.05726748, "gate_b2a/L11_absmean": 0.08356958, "gate_b2a/L11_absmax": 0.59696901, "gate_b2a/L11_rms": 0.09230088, "gate_b2a/L14_absmean": 0.21462572, "gate_b2a/L14_absmax": 0.7801044, "gate_b2a/L14_rms": 0.2228536, "gate_b2c/L3_absmean": 0.03832728, "gate_b2c/L3_absmax": 0.58070803, "gate_b2c/L3_rms": 0.04966201, "gate_b2c/L5_absmean": 0.04127138, "gate_b2c/L5_absmax": 0.6053347, "gate_b2c/L5_rms": 0.05207805, "gate_b2c/L6_absmean": 0.05945469, "gate_b2c/L6_absmax": 0.39599293, "gate_b2c/L6_rms": 0.06651179}, "streams": {"cos_a2b/round0": 0.365775, "cos_c2b/round0": 0.273662, "cos_b2a/round0": 0.135319, "cos_b2c/round0": 0.109486, "rms_a/round0": 5.373954, "rms_b/round0": 9.72593, "rms_c/round0": 3.845087, "cos_a2b/round1": 0.416125, "cos_c2b/round1": 0.294944, "cos_b2a/round1": 0.236245, "cos_b2c/round1": 0.119224, "rms_a/round1": 10.456813, "rms_b/round1": 22.894161, "rms_c/round1": 8.045805, "cos_a2b/round2": 0.520587, "cos_c2b/round2": 0.510488, "cos_b2a/round2": 0.086506, "cos_b2c/round2": 0.164413, "rms_a/round2": 27.175961, "rms_b/round2": 58.432316, "rms_c/round2": 15.330667}, "alpha_mean": 0.4102788814587015, "alpha_hist": [0.03978, 0.16948, 0.18837, 0.14359, 0.11191, 0.10428, 0.1002, 0.08871, 0.04913, 0.00455], "alpha_by_type": {"special": {"share": 0.00105, "mean": 0.59884, "std": 0.16157}, "whitespace": {"share": 0.03496, "mean": 0.63871, "std": 0.17159}, "digit": {"share": 0.01074, "mean": 0.36798, "std": 0.17908}, "word_start": {"share": 0.56653, "mean": 0.41497, "std": 0.21513}, "word_cont": {"share": 0.26219, "mean": 0.27045, "std": 0.1457}, "punct": {"share": 0.12453, "mean": 0.62126, "std": 0.18663}}, "alpha_p05": 0.1101, "alpha_p50": 0.36478, "alpha_p95": 0.80502}
{"step": 43410, "total_steps": 72479, "tokens": 22759342080, "loss": 4.730204999446869, "l_mix": 2.9343165159225464, "l_a": 3.0121925473213196, "l_c": 2.9741021394729614, "z_loss": 36.000648975372314, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16189226508140564, "tokens_per_second": 1339305.506212235, "eta_seconds": 11379.425979590416, "data_wait_seconds": 1.767277717590332e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 43420, "total_steps": 72479, "tokens": 22764584960, "loss": 4.667257189750671, "l_mix": 2.8954683542251587, "l_a": 2.971971422433853, "l_c": 2.9339912831783295, "z_loss": 34.38079357147217, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16202013194561005, "tokens_per_second": 1391503.5499113882, "eta_seconds": 10948.793478083611, "data_wait_seconds": 2.512335777282715e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 43430, "total_steps": 72479, "tokens": 22769827840, "loss": 4.6612823605537415, "l_mix": 2.891466438770294, "l_a": 2.971414804458618, "l_c": 2.927971303462982, "z_loss": 35.3416223526001, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14406225085258484, "tokens_per_second": 1402211.6933092573, "eta_seconds": 10861.442808294296, "data_wait_seconds": 1.9073486328125e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 43440, "total_steps": 72479, "tokens": 22775070720, "loss": 4.454702973365784, "l_mix": 2.7604537904262543, "l_a": 2.845923811197281, "l_c": 2.8015732765197754, "z_loss": 36.57841682434082, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12069123983383179, "tokens_per_second": 1470799.9177805455, "eta_seconds": 10351.373458719254, "data_wait_seconds": 1.6361474990844727e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08234}}
{"step": 43450, "total_steps": 72479, "tokens": 22780313600, "loss": 4.478035807609558, "l_mix": 2.7755720913410187, "l_a": 2.860523670911789, "l_c": 2.8143550157546997, "z_loss": 35.862736225128174, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14282366633415222, "tokens_per_second": 1451532.2120582305, "eta_seconds": 10485.166106247902, "data_wait_seconds": 1.7434358596801758e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 43460, "total_steps": 72479, "tokens": 22785556480, "loss": 4.448645532131195, "l_mix": 2.7570691108703613, "l_a": 2.844511955976486, "l_c": 2.794075608253479, "z_loss": 35.692402839660645, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11468230932950974, "tokens_per_second": 1406008.737975044, "eta_seconds": 10820.923839998246, "data_wait_seconds": 2.658367156982422e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 43470, "total_steps": 72479, "tokens": 22790799360, "loss": 3.191189229488373, "l_mix": 1.9634526669979095, "l_a": 2.07761949300766, "l_c": 2.0148355215787888, "z_loss": 38.63209104537964, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14847703278064728, "tokens_per_second": 1447193.3075434235, "eta_seconds": 10509.356637239456, "data_wait_seconds": 2.5391578674316406e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 43480, "total_steps": 72479, "tokens": 22796042240, "loss": 3.167692393064499, "l_mix": 1.949752688407898, "l_a": 2.061992347240448, "l_c": 1.9978066384792328, "z_loss": 39.62056112289429, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.21662573516368866, "tokens_per_second": 1397178.3874558979, "eta_seconds": 10881.808542490005, "data_wait_seconds": 1.8388032913208008e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 43490, "total_steps": 72479, "tokens": 22801285120, "loss": 3.2020013630390167, "l_mix": 1.970605880022049, "l_a": 2.08445143699646, "l_c": 2.020200029015541, "z_loss": 37.466336727142334, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16271625459194183, "tokens_per_second": 1467455.5667641845, "eta_seconds": 10357.100532531738, "data_wait_seconds": 1.9848346710205078e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 43500, "total_steps": 72479, "tokens": 22806528000, "loss": 3.1531279385089874, "l_mix": 1.939293995499611, "l_a": 2.055933356285095, "l_c": 1.9901796281337738, "z_loss": 38.528348445892334, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1747397929430008, "tokens_per_second": 1351615.0741484254, "eta_seconds": 11240.879332137109, "data_wait_seconds": 1.963973045349121e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.19998531, "gate_a2b/L1_absmax": 0.73012072, "gate_a2b/L1_rms": 0.21243341, "gate_a2b/L3_absmean": 0.197375, "gate_a2b/L3_absmax": 0.57414138, "gate_a2b/L3_rms": 0.20345443, "gate_a2b/L5_absmean": 0.30475971, "gate_a2b/L5_absmax": 0.72655439, "gate_a2b/L5_rms": 0.31065491, "gate_c2b/L1_absmean": 0.01887831, "gate_c2b/L1_absmax": 0.63280338, "gate_c2b/L1_rms": 0.06671632, "gate_c2b/L3_absmean": 0.19542727, "gate_c2b/L3_absmax": 0.44040403, "gate_c2b/L3_rms": 0.19949971, "gate_c2b/L5_absmean": 0.55549634, "gate_c2b/L5_absmax": 0.77851635, "gate_c2b/L5_rms": 0.56741822, "gate_b2a/L7_absmean": 0.05094191, "gate_b2a/L7_absmax": 0.32225287, "gate_b2a/L7_rms": 0.0565639, "gate_b2a/L11_absmean": 0.08359259, "gate_b2a/L11_absmax": 0.59918737, "gate_b2a/L11_rms": 0.09237464, "gate_b2a/L14_absmean": 0.21474087, "gate_b2a/L14_absmax": 0.78082603, "gate_b2a/L14_rms": 0.22296336, "gate_b2c/L3_absmean": 0.03798089, "gate_b2c/L3_absmax": 0.5814361, "gate_b2c/L3_rms": 0.04940712, "gate_b2c/L5_absmean": 0.04117613, "gate_b2c/L5_absmax": 0.60302693, "gate_b2c/L5_rms": 0.05186842, "gate_b2c/L6_absmean": 0.05944526, "gate_b2c/L6_absmax": 0.39240304, "gate_b2c/L6_rms": 0.06648909}, "streams": {"cos_a2b/round0": 0.366616, "cos_c2b/round0": 0.278286, "cos_b2a/round0": 0.136467, "cos_b2c/round0": 0.109596, "rms_a/round0": 5.390521, "rms_b/round0": 9.690947, "rms_c/round0": 3.851705, "cos_a2b/round1": 0.413094, "cos_c2b/round1": 0.30135, "cos_b2a/round1": 0.234453, "cos_b2c/round1": 0.118485, "rms_a/round1": 10.443404, "rms_b/round1": 22.791428, "rms_c/round1": 8.062322, "cos_a2b/round2": 0.523728, "cos_c2b/round2": 0.512297, "cos_b2a/round2": 0.086156, "cos_b2c/round2": 0.166376, "rms_a/round2": 27.237881, "rms_b/round2": 58.461052, "rms_c/round2": 15.136873}, "alpha_mean": 0.4184607632971762, "alpha_hist": [0.03426, 0.15987, 0.18414, 0.14485, 0.11876, 0.10951, 0.10446, 0.08881, 0.04578, 0.00957], "alpha_by_type": {"special": {"share": 0.00111, "mean": 0.65711, "std": 0.10713}, "whitespace": {"share": 0.04193, "mean": 0.65388, "std": 0.17718}, "digit": {"share": 0.01161, "mean": 0.43439, "std": 0.17327}, "word_start": {"share": 0.5639, "mean": 0.41296, "std": 0.21326}, "word_cont": {"share": 0.25903, "mean": 0.28724, "std": 0.14734}, "punct": {"share": 0.12241, "mean": 0.63715, "std": 0.18331}}, "alpha_p05": 0.11596, "alpha_p50": 0.41775, "alpha_p95": 0.82361}
{"step": 43510, "total_steps": 72479, "tokens": 22811770880, "loss": 3.1458071768283844, "l_mix": 1.9353204518556595, "l_a": 2.0506561994552612, "l_c": 1.9842993766069412, "z_loss": 38.0860276222229, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16557316482067108, "tokens_per_second": 1477942.996329377, "eta_seconds": 10276.51209127903, "data_wait_seconds": 3.409385681152344e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 43520, "total_steps": 72479, "tokens": 22817013760, "loss": 4.426940500736237, "l_mix": 2.7429760694503784, "l_a": 2.8313008546829224, "l_c": 2.7819139063358307, "z_loss": 36.99123430252075, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14307336509227753, "tokens_per_second": 1385631.7714960475, "eta_seconds": 10957.352815032005, "data_wait_seconds": 3.0100345611572266e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 43530, "total_steps": 72479, "tokens": 22822256640, "loss": 4.313914656639099, "l_mix": 2.673084706068039, "l_a": 2.757918745279312, "l_c": 2.711514174938202, "z_loss": 36.00362300872803, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12470229715108871, "tokens_per_second": 1392485.8456178224, "eta_seconds": 10899.653565430643, "data_wait_seconds": 3.224611282348633e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 43540, "total_steps": 72479, "tokens": 22827499520, "loss": 4.40014910697937, "l_mix": 2.7257260382175446, "l_a": 2.8167650997638702, "l_c": 2.764644891023636, "z_loss": 36.68331861495972, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.117470882833004, "tokens_per_second": 1428299.9114593323, "eta_seconds": 10622.678269648552, "data_wait_seconds": 4.363059997558594e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 43550, "total_steps": 72479, "tokens": 22832742400, "loss": 4.500184118747711, "l_mix": 2.7890802919864655, "l_a": 2.874722868204117, "l_c": 2.8289566040039062, "z_loss": 35.50941514968872, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16998986899852753, "tokens_per_second": 1394500.9401221327, "eta_seconds": 10876.383884453773, "data_wait_seconds": 3.123283386230469e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 43560, "total_steps": 72479, "tokens": 22837985280, "loss": 4.391373693943024, "l_mix": 2.7216586768627167, "l_a": 2.8094460368156433, "l_c": 2.756270170211792, "z_loss": 37.355199337005615, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.24976995587348938, "tokens_per_second": 1351351.608046547, "eta_seconds": 11219.79252603054, "data_wait_seconds": 3.316998481750488e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 43570, "total_steps": 72479, "tokens": 22843228160, "loss": 4.3580615520477295, "l_mix": 2.699354827404022, "l_a": 2.7882555723190308, "l_c": 2.7407665252685547, "z_loss": 37.678929805755615, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1734938770532608, "tokens_per_second": 1411667.708588389, "eta_seconds": 10736.692282319069, "data_wait_seconds": 2.6464462280273438e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20041, "finemath": 0.08235}}
{"step": 43580, "total_steps": 72479, "tokens": 22848471040, "loss": 3.171329915523529, "l_mix": 1.9516203701496124, "l_a": 2.065371587872505, "l_c": 2.0003268271684647, "z_loss": 38.76626110076904, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14722004532814026, "tokens_per_second": 1440254.251997554, "eta_seconds": 10519.947357201576, "data_wait_seconds": 1.564621925354004e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 43590, "total_steps": 72479, "tokens": 22853713920, "loss": 3.181386113166809, "l_mix": 1.9588660597801208, "l_a": 2.0681795477867126, "l_c": 2.006887137889862, "z_loss": 37.36846733093262, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18258605897426605, "tokens_per_second": 1463413.651789362, "eta_seconds": 10349.880236172676, "data_wait_seconds": 1.7702579498291016e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 43600, "total_steps": 72479, "tokens": 22858956800, "loss": 3.166731357574463, "l_mix": 1.9485742896795273, "l_a": 2.064149335026741, "l_c": 1.996374025940895, "z_loss": 38.41884136199951, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1520935595035553, "tokens_per_second": 1390227.0007121123, "eta_seconds": 10890.964672851564, "data_wait_seconds": 3.120303153991699e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.20025773, "gate_a2b/L1_absmax": 0.73097688, "gate_a2b/L1_rms": 0.21266034, "gate_a2b/L3_absmean": 0.19701815, "gate_a2b/L3_absmax": 0.57826298, "gate_a2b/L3_rms": 0.20319112, "gate_a2b/L5_absmean": 0.30434105, "gate_a2b/L5_absmax": 0.72860098, "gate_a2b/L5_rms": 0.31028223, "gate_c2b/L1_absmean": 0.0185311, "gate_c2b/L1_absmax": 0.6322363, "gate_c2b/L1_rms": 0.06673384, "gate_c2b/L3_absmean": 0.19487861, "gate_c2b/L3_absmax": 0.43798772, "gate_c2b/L3_rms": 0.19889729, "gate_c2b/L5_absmean": 0.55465275, "gate_c2b/L5_absmax": 0.78621513, "gate_c2b/L5_rms": 0.56662142, "gate_b2a/L7_absmean": 0.05165497, "gate_b2a/L7_absmax": 0.32065484, "gate_b2a/L7_rms": 0.05708876, "gate_b2a/L11_absmean": 0.08360542, "gate_b2a/L11_absmax": 0.59716958, "gate_b2a/L11_rms": 0.09233215, "gate_b2a/L14_absmean": 0.21518114, "gate_b2a/L14_absmax": 0.78083259, "gate_b2a/L14_rms": 0.22337091, "gate_b2c/L3_absmean": 0.03811146, "gate_b2c/L3_absmax": 0.58131528, "gate_b2c/L3_rms": 0.04952144, "gate_b2c/L5_absmean": 0.04118697, "gate_b2c/L5_absmax": 0.60329044, "gate_b2c/L5_rms": 0.05186528, "gate_b2c/L6_absmean": 0.05927762, "gate_b2c/L6_absmax": 0.3921316, "gate_b2c/L6_rms": 0.06629377}, "streams": {"cos_a2b/round0": 0.366524, "cos_c2b/round0": 0.274058, "cos_b2a/round0": 0.13325, "cos_b2c/round0": 0.108325, "rms_a/round0": 5.44329, "rms_b/round0": 9.83225, "rms_c/round0": 3.870522, "cos_a2b/round1": 0.414005, "cos_c2b/round1": 0.298239, "cos_b2a/round1": 0.235236, "cos_b2c/round1": 0.12119, "rms_a/round1": 10.530219, "rms_b/round1": 23.098654, "rms_c/round1": 8.144554, "cos_a2b/round2": 0.524397, "cos_c2b/round2": 0.506341, "cos_b2a/round2": 0.088128, "cos_b2c/round2": 0.167354, "rms_a/round2": 27.558069, "rms_b/round2": 59.080402, "rms_c/round2": 15.481809}, "alpha_mean": 0.43118909998248256, "alpha_hist": [0.02806, 0.13513, 0.18803, 0.15506, 0.12303, 0.10878, 0.10435, 0.09532, 0.05798, 0.00424], "alpha_by_type": {"special": {"share": 0.00111, "mean": 0.6623, "std": 0.10054}, "whitespace": {"share": 0.03326, "mean": 0.63912, "std": 0.16824}, "digit": {"share": 0.00963, "mean": 0.41981, "std": 0.16536}, "word_start": {"share": 0.57126, "mean": 0.44326, "std": 0.20881}, "word_cont": {"share": 0.26337, "mean": 0.28302, "std": 0.14681}, "punct": {"share": 0.12137, "mean": 0.63768, "std": 0.189}}, "alpha_p05": 0.14271, "alpha_p50": 0.39747, "alpha_p95": 0.81316}
{"step": 43610, "total_steps": 72479, "tokens": 22864199680, "loss": 3.1331491470336914, "l_mix": 1.9271468967199326, "l_a": 2.0418991446495056, "l_c": 1.9781082719564438, "z_loss": 38.68038320541382, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1595231145620346, "tokens_per_second": 1378012.3291332915, "eta_seconds": 10983.697280502318, "data_wait_seconds": 2.1845102310180664e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 43620, "total_steps": 72479, "tokens": 22869442560, "loss": 3.1683454513549805, "l_mix": 1.9495674669742584, "l_a": 2.066916435956955, "l_c": 1.9956768900156021, "z_loss": 36.6312370300293, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20653678476810455, "tokens_per_second": 1435584.9313825078, "eta_seconds": 10539.555731773376, "data_wait_seconds": 2.2411346435546875e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 43630, "total_steps": 72479, "tokens": 22874685440, "loss": 4.442902445793152, "l_mix": 2.7538667619228363, "l_a": 2.837032973766327, "l_c": 2.793085753917694, "z_loss": 36.357959270477295, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18139111995697021, "tokens_per_second": 1219982.049173134, "eta_seconds": 12397.874642705918, "data_wait_seconds": 2.1159648895263672e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 43640, "total_steps": 72479, "tokens": 22879928320, "loss": 4.396763205528259, "l_mix": 2.7250580489635468, "l_a": 2.808949440717697, "l_c": 2.7634005546569824, "z_loss": 36.16614580154419, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13486042618751526, "tokens_per_second": 1382160.5596968255, "eta_seconds": 10939.35254187584, "data_wait_seconds": 1.9848346710205078e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 43650, "total_steps": 72479, "tokens": 22885171200, "loss": 4.4778271317481995, "l_mix": 2.775005429983139, "l_a": 2.863809049129486, "l_c": 2.8122628331184387, "z_loss": 36.28354740142822, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13638441264629364, "tokens_per_second": 1448301.9978400178, "eta_seconds": 10436.151282358169, "data_wait_seconds": 1.665949821472168e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 43660, "total_steps": 72479, "tokens": 22890414080, "loss": 4.464331805706024, "l_mix": 2.7665427923202515, "l_a": 2.8528347611427307, "l_c": 2.806461662054062, "z_loss": 36.46210718154907, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13810847699642181, "tokens_per_second": 1410316.736570563, "eta_seconds": 10713.519509625434, "data_wait_seconds": 1.755356788635254e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 43670, "total_steps": 72479, "tokens": 22895656960, "loss": 4.374701619148254, "l_mix": 2.710139274597168, "l_a": 2.797962099313736, "l_c": 2.7505785822868347, "z_loss": 37.21026945114136, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11809973418712616, "tokens_per_second": 1369308.2884070256, "eta_seconds": 11030.54229633808, "data_wait_seconds": 2.0712614059448242e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 43680, "total_steps": 72479, "tokens": 22900899840, "loss": 4.470467209815979, "l_mix": 2.771112233400345, "l_a": 2.8582205772399902, "l_c": 2.806295871734619, "z_loss": 37.33807325363159, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2579859495162964, "tokens_per_second": 1434646.8545827577, "eta_seconds": 10524.52041683197, "data_wait_seconds": 2.2709369659423828e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 43690, "total_steps": 72479, "tokens": 22906142720, "loss": 4.633054971694946, "l_mix": 2.8728668093681335, "l_a": 2.953127086162567, "l_c": 2.914166659116745, "z_loss": 36.33906126022339, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15038126707077026, "tokens_per_second": 1400651.6659151805, "eta_seconds": 10776.217670178414, "data_wait_seconds": 1.7970800399780273e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 43700, "total_steps": 72479, "tokens": 22911385600, "loss": 4.754651963710785, "l_mix": 2.950061619281769, "l_a": 3.0280663669109344, "l_c": 2.987234264612198, "z_loss": 34.527541637420654, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12932035326957703, "tokens_per_second": 1394984.475206237, "eta_seconds": 10816.23818771839, "data_wait_seconds": 1.659989356994629e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.19945803, "gate_a2b/L1_absmax": 0.72985101, "gate_a2b/L1_rms": 0.21185935, "gate_a2b/L3_absmean": 0.19695532, "gate_a2b/L3_absmax": 0.57666361, "gate_a2b/L3_rms": 0.20304492, "gate_a2b/L5_absmean": 0.30305934, "gate_a2b/L5_absmax": 0.72948235, "gate_a2b/L5_rms": 0.30904451, "gate_c2b/L1_absmean": 0.01783185, "gate_c2b/L1_absmax": 0.63182747, "gate_c2b/L1_rms": 0.06636731, "gate_c2b/L3_absmean": 0.19480295, "gate_c2b/L3_absmax": 0.44097406, "gate_c2b/L3_rms": 0.1988883, "gate_c2b/L5_absmean": 0.55422992, "gate_c2b/L5_absmax": 0.78038394, "gate_c2b/L5_rms": 0.56613666, "gate_b2a/L7_absmean": 0.05142271, "gate_b2a/L7_absmax": 0.32078665, "gate_b2a/L7_rms": 0.05690371, "gate_b2a/L11_absmean": 0.08320282, "gate_b2a/L11_absmax": 0.59704256, "gate_b2a/L11_rms": 0.09198029, "gate_b2a/L14_absmean": 0.21462883, "gate_b2a/L14_absmax": 0.7810331, "gate_b2a/L14_rms": 0.22284643, "gate_b2c/L3_absmean": 0.03781033, "gate_b2c/L3_absmax": 0.58096552, "gate_b2c/L3_rms": 0.04925191, "gate_b2c/L5_absmean": 0.040922, "gate_b2c/L5_absmax": 0.60242641, "gate_b2c/L5_rms": 0.05169951, "gate_b2c/L6_absmean": 0.05893784, "gate_b2c/L6_absmax": 0.3920767, "gate_b2c/L6_rms": 0.06598881}, "streams": {"cos_a2b/round0": 0.371258, "cos_c2b/round0": 0.311061, "cos_b2a/round0": 0.149328, "cos_b2c/round0": 0.131615, "rms_a/round0": 5.210693, "rms_b/round0": 9.610052, "rms_c/round0": 3.756345, "cos_a2b/round1": 0.417761, "cos_c2b/round1": 0.302138, "cos_b2a/round1": 0.254515, "cos_b2c/round1": 0.116338, "rms_a/round1": 9.818457, "rms_b/round1": 21.5856, "rms_c/round1": 7.702334, "cos_a2b/round2": 0.560527, "cos_c2b/round2": 0.52127, "cos_b2a/round2": 0.100937, "cos_b2c/round2": 0.17672, "rms_a/round2": 26.719995, "rms_b/round2": 56.44696, "rms_c/round2": 14.437391}, "alpha_mean": 0.45239817160963014, "alpha_hist": [0.00407, 0.05817, 0.15669, 0.19035, 0.18478, 0.18607, 0.13623, 0.07486, 0.00874, 3e-05], "alpha_by_type": {"special": {"share": 0.00089, "mean": 0.58502, "std": 0.09695}, "whitespace": {"share": 0.04681, "mean": 0.55432, "std": 0.11799}, "digit": {"share": 0.03314, "mean": 0.45957, "std": 0.13005}, "word_start": {"share": 0.55939, "mean": 0.46802, "std": 0.16455}, "word_cont": {"share": 0.2477, "mean": 0.34289, "std": 0.12816}, "punct": {"share": 0.11208, "mean": 0.57069, "std": 0.1635}}, "alpha_p05": 0.17753, "alpha_p50": 0.43518, "alpha_p95": 0.73979}
{"step": 43710, "total_steps": 72479, "tokens": 22916628480, "loss": 3.169051706790924, "l_mix": 1.9505907148122787, "l_a": 2.0612897872924805, "l_c": 2.000246822834015, "z_loss": 38.60079479217529, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15475492179393768, "tokens_per_second": 1403982.6816845469, "eta_seconds": 10743.181998443604, "data_wait_seconds": 3.463029861450195e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 43720, "total_steps": 72479, "tokens": 22921871360, "loss": 3.1834428906440735, "l_mix": 1.9589310884475708, "l_a": 2.0747088193893433, "l_c": 2.0069970190525055, "z_loss": 38.71932649612427, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15867723524570465, "tokens_per_second": 1455638.3549117246, "eta_seconds": 10358.341095590591, "data_wait_seconds": 2.5898218154907227e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 43730, "total_steps": 72479, "tokens": 22927114240, "loss": 3.1981009542942047, "l_mix": 1.9680384695529938, "l_a": 2.08378466963768, "l_c": 2.0164234787225723, "z_loss": 39.30757236480713, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1743156462907791, "tokens_per_second": 1419502.5684087784, "eta_seconds": 10618.336343622208, "data_wait_seconds": 2.9832124710083008e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 43740, "total_steps": 72479, "tokens": 22932357120, "loss": 4.415771901607513, "l_mix": 2.736676514148712, "l_a": 2.8223074078559875, "l_c": 2.7746767103672028, "z_loss": 36.27327823638916, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17540068924427032, "tokens_per_second": 1304129.632288556, "eta_seconds": 11553.692561650276, "data_wait_seconds": 1.9282102584838867e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 43750, "total_steps": 72479, "tokens": 22937600000, "loss": 4.414461016654968, "l_mix": 2.734902322292328, "l_a": 2.8288297057151794, "l_c": 2.7696990966796875, "z_loss": 36.1521577835083, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1874595582485199, "tokens_per_second": 1466385.8149991478, "eta_seconds": 10271.696437549592, "data_wait_seconds": 1.9758939743041992e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 43760, "total_steps": 72479, "tokens": 22942842880, "loss": 4.436626613140106, "l_mix": 2.750079333782196, "l_a": 2.830662429332733, "l_c": 2.7911616563796997, "z_loss": 36.61706829071045, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.24240314960479736, "tokens_per_second": 1381877.149688264, "eta_seconds": 10896.067769408226, "data_wait_seconds": 1.7374753952026367e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 43770, "total_steps": 72479, "tokens": 22948085760, "loss": 4.668317973613739, "l_mix": 2.890846073627472, "l_a": 2.9928641617298126, "l_c": 2.932042181491852, "z_loss": 34.73847460746765, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.5163572430610657, "tokens_per_second": 1308028.8181766379, "eta_seconds": 11507.2267390728, "data_wait_seconds": 2.0384788513183594e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 43780, "total_steps": 72479, "tokens": 22953328640, "loss": 4.466604709625244, "l_mix": 2.768725335597992, "l_a": 2.856418788433075, "l_c": 2.8031788766384125, "z_loss": 36.83769702911377, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2260884791612625, "tokens_per_second": 1387152.4644543575, "eta_seconds": 10847.071030449866, "data_wait_seconds": 1.7702579498291016e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 43790, "total_steps": 72479, "tokens": 22958571520, "loss": 4.205889195203781, "l_mix": 2.6061474084854126, "l_a": 2.6887369453907013, "l_c": 2.64373517036438, "z_loss": 35.95890808105469, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17787426710128784, "tokens_per_second": 1440280.2874687836, "eta_seconds": 10443.313404250146, "data_wait_seconds": 2.8818845748901367e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 43800, "total_steps": 72479, "tokens": 22963814400, "loss": 4.60227769613266, "l_mix": 2.854658752679825, "l_a": 2.9324876070022583, "l_c": 2.8929086327552795, "z_loss": 35.36305236816406, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1142401471734047, "tokens_per_second": 1445859.9271449472, "eta_seconds": 10399.38604681492, "data_wait_seconds": 3.528594970703125e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.19864142, "gate_a2b/L1_absmax": 0.73028469, "gate_a2b/L1_rms": 0.21099068, "gate_a2b/L3_absmean": 0.1960278, "gate_a2b/L3_absmax": 0.5784288, "gate_a2b/L3_rms": 0.20223407, "gate_a2b/L5_absmean": 0.30196616, "gate_a2b/L5_absmax": 0.72964287, "gate_a2b/L5_rms": 0.3079606, "gate_c2b/L1_absmean": 0.01870686, "gate_c2b/L1_absmax": 0.63231838, "gate_c2b/L1_rms": 0.06678455, "gate_c2b/L3_absmean": 0.19473784, "gate_c2b/L3_absmax": 0.44132191, "gate_c2b/L3_rms": 0.19875081, "gate_c2b/L5_absmean": 0.55364388, "gate_c2b/L5_absmax": 0.7761426, "gate_c2b/L5_rms": 0.56546444, "gate_b2a/L7_absmean": 0.05169775, "gate_b2a/L7_absmax": 0.32322264, "gate_b2a/L7_rms": 0.05726181, "gate_b2a/L11_absmean": 0.08317931, "gate_b2a/L11_absmax": 0.59757978, "gate_b2a/L11_rms": 0.09202683, "gate_b2a/L14_absmean": 0.21422577, "gate_b2a/L14_absmax": 0.78370148, "gate_b2a/L14_rms": 0.22257039, "gate_b2c/L3_absmean": 0.03801535, "gate_b2c/L3_absmax": 0.58035368, "gate_b2c/L3_rms": 0.04934223, "gate_b2c/L5_absmean": 0.04101075, "gate_b2c/L5_absmax": 0.5993222, "gate_b2c/L5_rms": 0.05165933, "gate_b2c/L6_absmean": 0.05881208, "gate_b2c/L6_absmax": 0.39311689, "gate_b2c/L6_rms": 0.06590863}, "streams": {"cos_a2b/round0": 0.369393, "cos_c2b/round0": 0.316189, "cos_b2a/round0": 0.15658, "cos_b2c/round0": 0.133407, "rms_a/round0": 5.203319, "rms_b/round0": 9.667941, "rms_c/round0": 3.798517, "cos_a2b/round1": 0.421308, "cos_c2b/round1": 0.306379, "cos_b2a/round1": 0.25613, "cos_b2c/round1": 0.120164, "rms_a/round1": 9.899756, "rms_b/round1": 21.725941, "rms_c/round1": 7.759524, "cos_a2b/round2": 0.558274, "cos_c2b/round2": 0.524953, "cos_b2a/round2": 0.104381, "cos_b2c/round2": 0.183404, "rms_a/round2": 26.898674, "rms_b/round2": 56.772804, "rms_c/round2": 14.593527}, "alpha_mean": 0.4359598351491769, "alpha_hist": [0.00584, 0.06535, 0.17697, 0.19881, 0.18918, 0.17058, 0.12473, 0.0629, 0.00562, 3e-05], "alpha_by_type": {"special": {"share": 0.00063, "mean": 0.53702, "std": 0.13408}, "whitespace": {"share": 0.05017, "mean": 0.53673, "std": 0.10655}, "digit": {"share": 0.02483, "mean": 0.4519, "std": 0.11352}, "word_start": {"share": 0.58142, "mean": 0.45213, "std": 0.16178}, "word_cont": {"share": 0.24065, "mean": 0.31682, "std": 0.12039}, "punct": {"share": 0.10231, "mean": 0.57042, "std": 0.15902}}, "alpha_p05": 0.17078, "alpha_p50": 0.42752, "alpha_p95": 0.73144}
{"step": 43810, "total_steps": 72479, "tokens": 22969057280, "loss": 4.521216094493866, "l_mix": 2.804381310939789, "l_a": 2.882475793361664, "l_c": 2.8403069376945496, "z_loss": 36.55867862701416, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2212950736284256, "tokens_per_second": 1386245.9228171145, "eta_seconds": 10842.818308496475, "data_wait_seconds": 1.9848346710205078e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 43820, "total_steps": 72479, "tokens": 22974300160, "loss": 4.608989775180817, "l_mix": 2.857724189758301, "l_a": 2.9416547417640686, "l_c": 2.895897001028061, "z_loss": 36.2492618560791, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15215015411376953, "tokens_per_second": 1431251.2691250474, "eta_seconds": 10498.205392813683, "data_wait_seconds": 2.059340476989746e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 43830, "total_steps": 72479, "tokens": 22979543040, "loss": 4.74901282787323, "l_mix": 2.9469471871852875, "l_a": 3.0218363106250763, "l_c": 2.985048770904541, "z_loss": 34.245434284210205, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18432380259037018, "tokens_per_second": 1331536.286312515, "eta_seconds": 11280.448806691169, "data_wait_seconds": 1.8268823623657227e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 43840, "total_steps": 72479, "tokens": 22984785920, "loss": 4.65165650844574, "l_mix": 2.8851875364780426, "l_a": 2.9653347432613373, "l_c": 2.9228947460651398, "z_loss": 35.587942123413086, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1603844165802002, "tokens_per_second": 1427227.8281638464, "eta_seconds": 10520.453522348404, "data_wait_seconds": 1.6391277313232422e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 43850, "total_steps": 72479, "tokens": 22990028800, "loss": 3.1719348430633545, "l_mix": 1.9515165984630585, "l_a": 2.0658898800611496, "l_c": 2.002170830965042, "z_loss": 38.237711906433105, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15061409771442413, "tokens_per_second": 1463394.0771112014, "eta_seconds": 10256.868868589401, "data_wait_seconds": 2.0831823348999023e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 43860, "total_steps": 72479, "tokens": 22995271680, "loss": 4.455381512641907, "l_mix": 2.761353462934494, "l_a": 2.847733795642853, "l_c": 2.799026072025299, "z_loss": 36.66243314743042, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13112473487854004, "tokens_per_second": 1420222.511801317, "eta_seconds": 10564.962988066674, "data_wait_seconds": 1.7017126083374023e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 43870, "total_steps": 72479, "tokens": 23000514560, "loss": 4.3955466747283936, "l_mix": 2.723187267780304, "l_a": 2.812718480825424, "l_c": 2.7618126273155212, "z_loss": 38.18671989440918, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1796765774488449, "tokens_per_second": 1469995.3691638075, "eta_seconds": 10203.675267720222, "data_wait_seconds": 1.5497207641601562e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 43880, "total_steps": 72479, "tokens": 23005757440, "loss": 4.370906233787537, "l_mix": 2.708407908678055, "l_a": 2.795447200536728, "l_c": 2.7462135553359985, "z_loss": 36.56663799285889, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13426613807678223, "tokens_per_second": 1428339.6182547796, "eta_seconds": 10497.58217189312, "data_wait_seconds": 2.0325183868408203e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 43890, "total_steps": 72479, "tokens": 23011000320, "loss": 4.55047881603241, "l_mix": 2.820877343416214, "l_a": 2.904585599899292, "l_c": 2.860752373933792, "z_loss": 36.124176025390625, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18973024189472198, "tokens_per_second": 1292235.045579745, "eta_seconds": 11599.182117271423, "data_wait_seconds": 2.2202730178833008e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 43900, "total_steps": 72479, "tokens": 23016243200, "loss": 4.367470562458038, "l_mix": 2.706743836402893, "l_a": 2.7935231029987335, "l_c": 2.7422324120998383, "z_loss": 36.26919746398926, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15482449531555176, "tokens_per_second": 1451854.214483542, "eta_seconds": 10320.338366293907, "data_wait_seconds": 1.6123056411743164e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.19782893, "gate_a2b/L1_absmax": 0.72869778, "gate_a2b/L1_rms": 0.21021861, "gate_a2b/L3_absmean": 0.19586666, "gate_a2b/L3_absmax": 0.57478732, "gate_a2b/L3_rms": 0.20204476, "gate_a2b/L5_absmean": 0.30151954, "gate_a2b/L5_absmax": 0.72831315, "gate_a2b/L5_rms": 0.30744052, "gate_c2b/L1_absmean": 0.01840525, "gate_c2b/L1_absmax": 0.63126081, "gate_c2b/L1_rms": 0.06675203, "gate_c2b/L3_absmean": 0.19411092, "gate_c2b/L3_absmax": 0.44348589, "gate_c2b/L3_rms": 0.19818179, "gate_c2b/L5_absmean": 0.55314791, "gate_c2b/L5_absmax": 0.76912689, "gate_c2b/L5_rms": 0.56496722, "gate_b2a/L7_absmean": 0.05160946, "gate_b2a/L7_absmax": 0.32102942, "gate_b2a/L7_rms": 0.05709629, "gate_b2a/L11_absmean": 0.08340288, "gate_b2a/L11_absmax": 0.59824234, "gate_b2a/L11_rms": 0.09218357, "gate_b2a/L14_absmean": 0.21426205, "gate_b2a/L14_absmax": 0.78256458, "gate_b2a/L14_rms": 0.22255798, "gate_b2c/L3_absmean": 0.03794566, "gate_b2c/L3_absmax": 0.58059001, "gate_b2c/L3_rms": 0.04933333, "gate_b2c/L5_absmean": 0.04114358, "gate_b2c/L5_absmax": 0.59955895, "gate_b2c/L5_rms": 0.05173432, "gate_b2c/L6_absmean": 0.05879643, "gate_b2c/L6_absmax": 0.38998875, "gate_b2c/L6_rms": 0.06579189}, "streams": {"cos_a2b/round0": 0.36747, "cos_c2b/round0": 0.310695, "cos_b2a/round0": 0.152523, "cos_b2c/round0": 0.133102, "rms_a/round0": 5.251515, "rms_b/round0": 9.745504, "rms_c/round0": 3.794819, "cos_a2b/round1": 0.423048, "cos_c2b/round1": 0.300319, "cos_b2a/round1": 0.250132, "cos_b2c/round1": 0.114299, "rms_a/round1": 9.973077, "rms_b/round1": 21.933802, "rms_c/round1": 7.676814, "cos_a2b/round2": 0.559904, "cos_c2b/round2": 0.514161, "cos_b2a/round2": 0.108276, "cos_b2c/round2": 0.180576, "rms_a/round2": 27.396345, "rms_b/round2": 57.895592, "rms_c/round2": 14.31706}, "alpha_mean": 0.46341469364253385, "alpha_hist": [0.00612, 0.05299, 0.1553, 0.18163, 0.17458, 0.17651, 0.1438, 0.09038, 0.01855, 0.00014], "alpha_by_type": {"special": {"share": 0.0006, "mean": 0.56343, "std": 0.13026}, "whitespace": {"share": 0.0215, "mean": 0.52983, "std": 0.10739}, "digit": {"share": 0.03191, "mean": 0.44705, "std": 0.09742}, "word_start": {"share": 0.57527, "mean": 0.48968, "std": 0.16563}, "word_cont": {"share": 0.25911, "mean": 0.33307, "std": 0.13117}, "punct": {"share": 0.11162, "mean": 0.622, "std": 0.16065}}, "alpha_p05": 0.20276, "alpha_p50": 0.46885, "alpha_p95": 0.76524}
{"step": 43910, "total_steps": 72479, "tokens": 23021486080, "loss": 2.890187382698059, "l_mix": 1.7886398136615753, "l_a": 1.857033059000969, "l_c": 1.814791977405548, "z_loss": 43.869850158691406, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19192427396774292, "tokens_per_second": 1406923.4074595198, "eta_seconds": 10646.197079801559, "data_wait_seconds": 2.238154411315918e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 43920, "total_steps": 72479, "tokens": 23026728960, "loss": 4.691407084465027, "l_mix": 2.909543365240097, "l_a": 2.9938226342201233, "l_c": 2.9457226991653442, "z_loss": 35.09030771255493, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12554983794689178, "tokens_per_second": 1300933.157088236, "eta_seconds": 11509.539064645767, "data_wait_seconds": 2.2679567337036133e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 43930, "total_steps": 72479, "tokens": 23031971840, "loss": 4.756071448326111, "l_mix": 2.950345128774643, "l_a": 3.0317679047584534, "l_c": 2.9873195588588715, "z_loss": 35.86116647720337, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14798858761787415, "tokens_per_second": 1354028.250279654, "eta_seconds": 11054.346989369393, "data_wait_seconds": 2.3484230041503906e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 43940, "total_steps": 72479, "tokens": 23037214720, "loss": 4.578105032444, "l_mix": 2.8401538729667664, "l_a": 2.91690593957901, "l_c": 2.8762643337249756, "z_loss": 36.02731132507324, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1349431574344635, "tokens_per_second": 1469957.8326112079, "eta_seconds": 10178.969015336035, "data_wait_seconds": 1.9222497940063477e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 43950, "total_steps": 72479, "tokens": 23042457600, "loss": 4.821574449539185, "l_mix": 2.9917493760585785, "l_a": 3.06981560587883, "l_c": 3.029601216316223, "z_loss": 33.933115005493164, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17927801609039307, "tokens_per_second": 1431727.2584848476, "eta_seconds": 10447.110134530067, "data_wait_seconds": 1.823902130126953e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 43960, "total_steps": 72479, "tokens": 23047700480, "loss": 4.667237401008606, "l_mix": 2.8948847949504852, "l_a": 2.975088059902191, "l_c": 2.9327538907527924, "z_loss": 35.221248149871826, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17183440923690796, "tokens_per_second": 1421604.3010167112, "eta_seconds": 10517.813896107673, "data_wait_seconds": 2.333521842956543e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 43970, "total_steps": 72479, "tokens": 23052943360, "loss": 4.37148243188858, "l_mix": 2.7084489166736603, "l_a": 2.7973730266094208, "l_c": 2.746071517467499, "z_loss": 36.969340324401855, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12422865629196167, "tokens_per_second": 1421338.3847419547, "eta_seconds": 10516.092967343331, "data_wait_seconds": 2.1010637283325195e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 43980, "total_steps": 72479, "tokens": 23058186240, "loss": 4.31758040189743, "l_mix": 2.6748076677322388, "l_a": 2.765006422996521, "l_c": 2.7109024226665497, "z_loss": 36.78851366043091, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1637929528951645, "tokens_per_second": 1360993.4463517405, "eta_seconds": 10978.51260933876, "data_wait_seconds": 2.8789043426513672e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 43990, "total_steps": 72479, "tokens": 23063429120, "loss": 4.445297837257385, "l_mix": 2.7536891996860504, "l_a": 2.845025599002838, "l_c": 2.793669581413269, "z_loss": 36.05263376235962, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.22179637849330902, "tokens_per_second": 1438199.6674530355, "eta_seconds": 10385.51264474392, "data_wait_seconds": 2.9295682907104492e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 44000, "total_steps": 72479, "tokens": 23068672000, "loss": 4.398202151060104, "l_mix": 2.7259529531002045, "l_a": 2.8085435926914215, "l_c": 2.7656197547912598, "z_loss": 35.960646629333496, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2271268665790558, "tokens_per_second": 1391692.094609253, "eta_seconds": 10728.808484172821, "data_wait_seconds": 2.065300941467285e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.19808948, "gate_a2b/L1_absmax": 0.72852772, "gate_a2b/L1_rms": 0.21046573, "gate_a2b/L3_absmean": 0.19500001, "gate_a2b/L3_absmax": 0.57565403, "gate_a2b/L3_rms": 0.20118479, "gate_a2b/L5_absmean": 0.30064818, "gate_a2b/L5_absmax": 0.72886193, "gate_a2b/L5_rms": 0.30659601, "gate_c2b/L1_absmean": 0.01856151, "gate_c2b/L1_absmax": 0.63129354, "gate_c2b/L1_rms": 0.0665185, "gate_c2b/L3_absmean": 0.19443093, "gate_c2b/L3_absmax": 0.43743569, "gate_c2b/L3_rms": 0.19846, "gate_c2b/L5_absmean": 0.55223072, "gate_c2b/L5_absmax": 0.78187639, "gate_c2b/L5_rms": 0.56408411, "gate_b2a/L7_absmean": 0.05167004, "gate_b2a/L7_absmax": 0.31978559, "gate_b2a/L7_rms": 0.05713645, "gate_b2a/L11_absmean": 0.08327155, "gate_b2a/L11_absmax": 0.59864151, "gate_b2a/L11_rms": 0.09210476, "gate_b2a/L14_absmean": 0.21426857, "gate_b2a/L14_absmax": 0.78324366, "gate_b2a/L14_rms": 0.2225899, "gate_b2c/L3_absmean": 0.03752073, "gate_b2c/L3_absmax": 0.58031052, "gate_b2c/L3_rms": 0.04901216, "gate_b2c/L5_absmean": 0.04107951, "gate_b2c/L5_absmax": 0.60000521, "gate_b2c/L5_rms": 0.05170999, "gate_b2c/L6_absmean": 0.05879403, "gate_b2c/L6_absmax": 0.39080706, "gate_b2c/L6_rms": 0.0658017}, "streams": {"cos_a2b/round0": 0.369312, "cos_c2b/round0": 0.308227, "cos_b2a/round0": 0.153149, "cos_b2c/round0": 0.132655, "rms_a/round0": 5.33597, "rms_b/round0": 9.812201, "rms_c/round0": 3.830731, "cos_a2b/round1": 0.419684, "cos_c2b/round1": 0.304078, "cos_b2a/round1": 0.255534, "cos_b2c/round1": 0.114201, "rms_a/round1": 10.133117, "rms_b/round1": 22.063915, "rms_c/round1": 7.816984, "cos_a2b/round2": 0.555677, "cos_c2b/round2": 0.519433, "cos_b2a/round2": 0.108226, "cos_b2c/round2": 0.179455, "rms_a/round2": 27.657692, "rms_b/round2": 58.01569, "rms_c/round2": 14.751431}, "alpha_mean": 0.4392855258394164, "alpha_hist": [0.00919, 0.06433, 0.17201, 0.19933, 0.17915, 0.17245, 0.12563, 0.06908, 0.00883, 0.0], "alpha_by_type": {"special": {"share": 0.00166, "mean": 0.58064, "std": 0.10579}, "whitespace": {"share": 0.02884, "mean": 0.51925, "std": 0.12124}, "digit": {"share": 0.02602, "mean": 0.44218, "std": 0.11896}, "word_start": {"share": 0.57028, "mean": 0.46255, "std": 0.16068}, "word_cont": {"share": 0.26624, "mean": 0.3197, "std": 0.13062}, "punct": {"share": 0.10696, "mean": 0.58845, "std": 0.1572}}, "alpha_p05": 0.18684, "alpha_p50": 0.43999, "alpha_p95": 0.73564}
{"step": 44010, "total_steps": 72479, "tokens": 23073914880, "loss": 4.373391926288605, "l_mix": 2.709801286458969, "l_a": 2.79749259352684, "l_c": 2.7478094696998596, "z_loss": 37.116379737854004, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12796564400196075, "tokens_per_second": 1481877.4208219838, "eta_seconds": 10072.327752804757, "data_wait_seconds": 2.0563602447509766e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 44020, "total_steps": 72479, "tokens": 23079157760, "loss": 2.9387516379356384, "l_mix": 1.819242686033249, "l_a": 1.8871426433324814, "l_c": 1.844553679227829, "z_loss": 41.3945198059082, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20491784811019897, "tokens_per_second": 1393727.592169324, "eta_seconds": 10705.615843319893, "data_wait_seconds": 2.3096799850463867e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 44030, "total_steps": 72479, "tokens": 23084400640, "loss": 2.590028241276741, "l_mix": 1.603205144405365, "l_a": 1.6550748720765114, "l_c": 1.634335182607174, "z_loss": 43.56746006011963, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.36128708720207214, "tokens_per_second": 1367862.6073064115, "eta_seconds": 10904.215988016129, "data_wait_seconds": 2.5659799575805664e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 44040, "total_steps": 72479, "tokens": 23089643520, "loss": 4.365091443061829, "l_mix": 2.704211801290512, "l_a": 2.7931016981601715, "l_c": 2.7431634962558746, "z_loss": 37.13942003250122, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11191127449274063, "tokens_per_second": 1385102.0874017258, "eta_seconds": 10764.712989473343, "data_wait_seconds": 2.8878450393676758e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 44050, "total_steps": 72479, "tokens": 23094886400, "loss": 4.442120671272278, "l_mix": 2.753449499607086, "l_a": 2.8371826708316803, "l_c": 2.7917209565639496, "z_loss": 37.557857513427734, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17625993490219116, "tokens_per_second": 1235814.8365178693, "eta_seconds": 12060.855001544951, "data_wait_seconds": 4.076957702636719e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 44060, "total_steps": 72479, "tokens": 23100129280, "loss": 4.371471524238586, "l_mix": 2.7092920541763306, "l_a": 2.7925845980644226, "l_c": 2.7480134665966034, "z_loss": 36.65953826904297, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14355294406414032, "tokens_per_second": 1291375.1005969977, "eta_seconds": 11537.887531757355, "data_wait_seconds": 2.41696834564209e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 44070, "total_steps": 72479, "tokens": 23105372160, "loss": 4.397543132305145, "l_mix": 2.7240088284015656, "l_a": 2.8132499754428864, "l_c": 2.765197604894638, "z_loss": 37.6795449256897, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14254668354988098, "tokens_per_second": 1306663.6858839863, "eta_seconds": 11398.876354265214, "data_wait_seconds": 2.682209014892578e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 44080, "total_steps": 72479, "tokens": 23110615040, "loss": 4.372635185718536, "l_mix": 2.7099697291851044, "l_a": 2.7956445813179016, "l_c": 2.746573507785797, "z_loss": 36.0963020324707, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17627422511577606, "tokens_per_second": 1439700.6551269235, "eta_seconds": 10341.910215139389, "data_wait_seconds": 2.03549861907959e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 44090, "total_steps": 72479, "tokens": 23115857920, "loss": 4.372313857078552, "l_mix": 2.708339422941208, "l_a": 2.7976295053958893, "l_c": 2.7489515244960785, "z_loss": 36.69624328613281, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19406959414482117, "tokens_per_second": 1389995.79943287, "eta_seconds": 10707.954684519767, "data_wait_seconds": 1.5914440155029297e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 44100, "total_steps": 72479, "tokens": 23121100800, "loss": 4.314518094062805, "l_mix": 2.6738239526748657, "l_a": 2.759454160928726, "l_c": 2.7095261812210083, "z_loss": 36.96119499206543, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2283214032649994, "tokens_per_second": 1432952.2325974223, "eta_seconds": 10383.297372746467, "data_wait_seconds": 2.0056962966918945e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.19732888, "gate_a2b/L1_absmax": 0.72829628, "gate_a2b/L1_rms": 0.2096234, "gate_a2b/L3_absmean": 0.19435953, "gate_a2b/L3_absmax": 0.57495755, "gate_a2b/L3_rms": 0.20058809, "gate_a2b/L5_absmean": 0.30005002, "gate_a2b/L5_absmax": 0.72991502, "gate_a2b/L5_rms": 0.30594537, "gate_c2b/L1_absmean": 0.0188534, "gate_c2b/L1_absmax": 0.63258493, "gate_c2b/L1_rms": 0.06676556, "gate_c2b/L3_absmean": 0.19412789, "gate_c2b/L3_absmax": 0.43826324, "gate_c2b/L3_rms": 0.19815016, "gate_c2b/L5_absmean": 0.55141878, "gate_c2b/L5_absmax": 0.7729817, "gate_c2b/L5_rms": 0.56328261, "gate_b2a/L7_absmean": 0.05146671, "gate_b2a/L7_absmax": 0.31916061, "gate_b2a/L7_rms": 0.05691739, "gate_b2a/L11_absmean": 0.08344245, "gate_b2a/L11_absmax": 0.59770805, "gate_b2a/L11_rms": 0.09217813, "gate_b2a/L14_absmean": 0.21440986, "gate_b2a/L14_absmax": 0.7830444, "gate_b2a/L14_rms": 0.22270983, "gate_b2c/L3_absmean": 0.03768971, "gate_b2c/L3_absmax": 0.58021092, "gate_b2c/L3_rms": 0.04909026, "gate_b2c/L5_absmean": 0.04068509, "gate_b2c/L5_absmax": 0.59673178, "gate_b2c/L5_rms": 0.05128274, "gate_b2c/L6_absmean": 0.05837467, "gate_b2c/L6_absmax": 0.38939795, "gate_b2c/L6_rms": 0.06536293}, "streams": {"cos_a2b/round0": 0.37119, "cos_c2b/round0": 0.313773, "cos_b2a/round0": 0.143821, "cos_b2c/round0": 0.132619, "rms_a/round0": 5.308525, "rms_b/round0": 9.766124, "rms_c/round0": 3.83712, "cos_a2b/round1": 0.413288, "cos_c2b/round1": 0.294347, "cos_b2a/round1": 0.246949, "cos_b2c/round1": 0.11278, "rms_a/round1": 10.04098, "rms_b/round1": 22.155081, "rms_c/round1": 7.773198, "cos_a2b/round2": 0.55363, "cos_c2b/round2": 0.514426, "cos_b2a/round2": 0.100247, "cos_b2c/round2": 0.177723, "rms_a/round2": 27.436491, "rms_b/round2": 58.053314, "rms_c/round2": 14.454804}, "alpha_mean": 0.4453788710466924, "alpha_hist": [0.00948, 0.05904, 0.16402, 0.19719, 0.18459, 0.17282, 0.13101, 0.07086, 0.01048, 0.00052], "alpha_by_type": {"special": {"share": 0.00104, "mean": 0.58682, "std": 0.12575}, "whitespace": {"share": 0.02802, "mean": 0.56891, "std": 0.12149}, "digit": {"share": 0.04922, "mean": 0.46013, "std": 0.10505}, "word_start": {"share": 0.53337, "mean": 0.46496, "std": 0.16376}, "word_cont": {"share": 0.26706, "mean": 0.3248, "std": 0.12949}, "punct": {"share": 0.12129, "mean": 0.58904, "std": 0.15167}}, "alpha_p05": 0.1833, "alpha_p50": 0.43842, "alpha_p95": 0.73867}
{"step": 44110, "total_steps": 72479, "tokens": 23126343680, "loss": 4.414970338344574, "l_mix": 2.7349848449230194, "l_a": 2.8196208775043488, "l_c": 2.7803304493427277, "z_loss": 36.79823398590088, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.29077446460723877, "tokens_per_second": 1378660.5389032385, "eta_seconds": 10788.389057564737, "data_wait_seconds": 2.154707908630371e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 44120, "total_steps": 72479, "tokens": 23131586560, "loss": 4.292021691799164, "l_mix": 2.659345179796219, "l_a": 2.7496361434459686, "l_c": 2.6926186978816986, "z_loss": 36.81373929977417, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19122949242591858, "tokens_per_second": 1343820.39242196, "eta_seconds": 11064.189437699317, "data_wait_seconds": 2.8073787689208984e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 44130, "total_steps": 72479, "tokens": 23136829440, "loss": 3.166682690382004, "l_mix": 1.9495459496974945, "l_a": 2.0635630786418915, "l_c": 1.9935592412948608, "z_loss": 38.21451234817505, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16268329322338104, "tokens_per_second": 1454919.3190282115, "eta_seconds": 10215.714588165283, "data_wait_seconds": 3.55839729309082e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 44140, "total_steps": 72479, "tokens": 23142072320, "loss": 2.881081134080887, "l_mix": 1.7824809700250626, "l_a": 1.8468064665794373, "l_c": 1.815193846821785, "z_loss": 43.03744840621948, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2127133458852768, "tokens_per_second": 1435457.39106322, "eta_seconds": 10350.56681201458, "data_wait_seconds": 4.175305366516113e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 44150, "total_steps": 72479, "tokens": 23147315200, "loss": 4.331818521022797, "l_mix": 2.6845632791519165, "l_a": 2.7681232690811157, "l_c": 2.7227273881435394, "z_loss": 36.93756055831909, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12352670729160309, "tokens_per_second": 1280123.854341725, "eta_seconds": 11602.43573434353, "data_wait_seconds": 1.7583370208740234e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 44160, "total_steps": 72479, "tokens": 23152558080, "loss": 4.378406524658203, "l_mix": 2.7132950127124786, "l_a": 2.8005407750606537, "l_c": 2.749830961227417, "z_loss": 37.387293338775635, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17624850571155548, "tokens_per_second": 1430336.5220060379, "eta_seconds": 10380.292779755591, "data_wait_seconds": 1.96993350982666e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 44170, "total_steps": 72479, "tokens": 23157800960, "loss": 4.403445363044739, "l_mix": 2.7291944324970245, "l_a": 2.8122443556785583, "l_c": 2.768591672182083, "z_loss": 35.97572612762451, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19554688036441803, "tokens_per_second": 1318546.6133689259, "eta_seconds": 11256.385509252548, "data_wait_seconds": 1.8268823623657227e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 44180, "total_steps": 72479, "tokens": 23163043840, "loss": 4.376142501831055, "l_mix": 2.7110873758792877, "l_a": 2.7995930314064026, "l_c": 2.750590682029724, "z_loss": 37.19286060333252, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12245568633079529, "tokens_per_second": 1330337.648436797, "eta_seconds": 11152.67701356411, "data_wait_seconds": 2.053380012512207e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 44190, "total_steps": 72479, "tokens": 23168286720, "loss": 4.352882623672485, "l_mix": 2.697609841823578, "l_a": 2.7816790342330933, "l_c": 2.7358969151973724, "z_loss": 36.69220590591431, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11875530332326889, "tokens_per_second": 1340802.9242681786, "eta_seconds": 11061.717545175554, "data_wait_seconds": 1.6629695892333984e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 44200, "total_steps": 72479, "tokens": 23173529600, "loss": 4.606895327568054, "l_mix": 2.858469158411026, "l_a": 2.932565361261368, "l_c": 2.8955216109752655, "z_loss": 34.868000984191895, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11883551627397537, "tokens_per_second": 1429571.7178134583, "eta_seconds": 10371.176323127747, "data_wait_seconds": 1.8924474716186523e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.19715765, "gate_a2b/L1_absmax": 0.72875273, "gate_a2b/L1_rms": 0.20937672, "gate_a2b/L3_absmean": 0.19415952, "gate_a2b/L3_absmax": 0.57779497, "gate_a2b/L3_rms": 0.20043603, "gate_a2b/L5_absmean": 0.29882956, "gate_a2b/L5_absmax": 0.73295438, "gate_a2b/L5_rms": 0.30481508, "gate_c2b/L1_absmean": 0.01932017, "gate_c2b/L1_absmax": 0.63137901, "gate_c2b/L1_rms": 0.06675632, "gate_c2b/L3_absmean": 0.19358741, "gate_c2b/L3_absmax": 0.43966779, "gate_c2b/L3_rms": 0.19767718, "gate_c2b/L5_absmean": 0.55146772, "gate_c2b/L5_absmax": 0.77461743, "gate_c2b/L5_rms": 0.56335127, "gate_b2a/L7_absmean": 0.0520278, "gate_b2a/L7_absmax": 0.32040381, "gate_b2a/L7_rms": 0.0574046, "gate_b2a/L11_absmean": 0.08337506, "gate_b2a/L11_absmax": 0.59972584, "gate_b2a/L11_rms": 0.09219193, "gate_b2a/L14_absmean": 0.21460927, "gate_b2a/L14_absmax": 0.79049122, "gate_b2a/L14_rms": 0.22300898, "gate_b2c/L3_absmean": 0.03784878, "gate_b2c/L3_absmax": 0.58032215, "gate_b2c/L3_rms": 0.04921353, "gate_b2c/L5_absmean": 0.04051755, "gate_b2c/L5_absmax": 0.60046995, "gate_b2c/L5_rms": 0.051291, "gate_b2c/L6_absmean": 0.05849668, "gate_b2c/L6_absmax": 0.39145353, "gate_b2c/L6_rms": 0.06548006}, "streams": {"cos_a2b/round0": 0.369066, "cos_c2b/round0": 0.325455, "cos_b2a/round0": 0.16355, "cos_b2c/round0": 0.140115, "rms_a/round0": 5.296726, "rms_b/round0": 9.748412, "rms_c/round0": 3.716845, "cos_a2b/round1": 0.424328, "cos_c2b/round1": 0.308768, "cos_b2a/round1": 0.255758, "cos_b2c/round1": 0.127986, "rms_a/round1": 10.124147, "rms_b/round1": 21.616728, "rms_c/round1": 7.758887, "cos_a2b/round2": 0.571247, "cos_c2b/round2": 0.521525, "cos_b2a/round2": 0.119064, "cos_b2c/round2": 0.196309, "rms_a/round2": 27.573322, "rms_b/round2": 57.207077, "rms_c/round2": 14.625154}, "alpha_mean": 0.47443045605787404, "alpha_hist": [0.00352, 0.03552, 0.11111, 0.16374, 0.22049, 0.25262, 0.14659, 0.06075, 0.00526, 0.00038], "alpha_by_type": {"special": {"share": 0.00046, "mean": 0.59932, "std": 0.09358}, "whitespace": {"share": 0.09935, "mean": 0.5477, "std": 0.09078}, "digit": {"share": 0.02512, "mean": 0.44685, "std": 0.13236}, "word_start": {"share": 0.52328, "mean": 0.4843, "std": 0.14612}, "word_cont": {"share": 0.23813, "mean": 0.36864, "std": 0.12784}, "punct": {"share": 0.11366, "mean": 0.59219, "std": 0.13478}}, "alpha_p05": 0.21436, "alpha_p50": 0.46995, "alpha_p95": 0.72817}
{"step": 44210, "total_steps": 72479, "tokens": 23178772480, "loss": 4.656029939651489, "l_mix": 2.8876524567604065, "l_a": 2.9682461619377136, "l_c": 2.9263453781604767, "z_loss": 36.251652240753174, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1847248375415802, "tokens_per_second": 1428780.4377219777, "eta_seconds": 10373.250557398795, "data_wait_seconds": 2.4199485778808594e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 44220, "total_steps": 72479, "tokens": 23184015360, "loss": 4.604855120182037, "l_mix": 2.855837106704712, "l_a": 2.936513125896454, "l_c": 2.8935466706752777, "z_loss": 36.41753149032593, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16678428649902344, "tokens_per_second": 1434895.5839366214, "eta_seconds": 10325.388660931587, "data_wait_seconds": 1.6570091247558594e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 44230, "total_steps": 72479, "tokens": 23189258240, "loss": 4.675130844116211, "l_mix": 2.899043321609497, "l_a": 2.9814886450767517, "l_c": 2.9388031363487244, "z_loss": 35.25006151199341, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.179719477891922, "tokens_per_second": 1430129.7349716027, "eta_seconds": 10356.131580114365, "data_wait_seconds": 2.822279930114746e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 44240, "total_steps": 72479, "tokens": 23194501120, "loss": 4.750026881694794, "l_mix": 2.946990340948105, "l_a": 3.0275502800941467, "l_c": 2.982571601867676, "z_loss": 34.99163246154785, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1417914777994156, "tokens_per_second": 1392139.3990668142, "eta_seconds": 10634.975809121133, "data_wait_seconds": 3.11434268951416e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 44250, "total_steps": 72479, "tokens": 23199744000, "loss": 2.7602853178977966, "l_mix": 1.709359034895897, "l_a": 1.7645667493343353, "l_c": 1.7385206669569016, "z_loss": 43.998849868774414, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.241421177983284, "tokens_per_second": 1470825.593667825, "eta_seconds": 10062.461528897285, "data_wait_seconds": 1.6301870346069336e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 44260, "total_steps": 72479, "tokens": 23204986880, "loss": 4.4268380999565125, "l_mix": 2.7433606684207916, "l_a": 2.8287197947502136, "l_c": 2.782871335744858, "z_loss": 36.628708839416504, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.10525944083929062, "tokens_per_second": 1331897.7510378123, "eta_seconds": 11108.122271752358, "data_wait_seconds": 2.8342008590698242e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 44270, "total_steps": 72479, "tokens": 23210229760, "loss": 4.3667266964912415, "l_mix": 2.706142634153366, "l_a": 2.7928487956523895, "l_c": 2.742431163787842, "z_loss": 37.02720928192139, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14285506308078766, "tokens_per_second": 1246219.5886827456, "eta_seconds": 11867.603692245484, "data_wait_seconds": 1.8984079360961914e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 44280, "total_steps": 72479, "tokens": 23215472640, "loss": 4.363868832588196, "l_mix": 2.7035228312015533, "l_a": 2.7908028066158295, "l_c": 2.7436834573745728, "z_loss": 36.392008781433105, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.26142367720603943, "tokens_per_second": 1422769.5689901367, "eta_seconds": 10391.280242586136, "data_wait_seconds": 1.6570091247558594e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 44290, "total_steps": 72479, "tokens": 23220715520, "loss": 4.279878616333008, "l_mix": 2.6529346704483032, "l_a": 2.731068402528763, "l_c": 2.6920776069164276, "z_loss": 36.68778610229492, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19092318415641785, "tokens_per_second": 1449710.1306254785, "eta_seconds": 10194.558291196823, "data_wait_seconds": 1.919269561767578e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 44300, "total_steps": 72479, "tokens": 23225958400, "loss": 4.319129526615143, "l_mix": 2.675893872976303, "l_a": 2.7606261670589447, "l_c": 2.7168253660202026, "z_loss": 37.40559911727905, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14562168717384338, "tokens_per_second": 1320628.7204534218, "eta_seconds": 11187.02881679535, "data_wait_seconds": 1.9550323486328125e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.19679518, "gate_a2b/L1_absmax": 0.72921103, "gate_a2b/L1_rms": 0.209011, "gate_a2b/L3_absmean": 0.19382636, "gate_a2b/L3_absmax": 0.57426566, "gate_a2b/L3_rms": 0.20002712, "gate_a2b/L5_absmean": 0.29831201, "gate_a2b/L5_absmax": 0.73376274, "gate_a2b/L5_rms": 0.30428421, "gate_c2b/L1_absmean": 0.01856714, "gate_c2b/L1_absmax": 0.63174438, "gate_c2b/L1_rms": 0.06635797, "gate_c2b/L3_absmean": 0.19388662, "gate_c2b/L3_absmax": 0.43809342, "gate_c2b/L3_rms": 0.19795915, "gate_c2b/L5_absmean": 0.55067623, "gate_c2b/L5_absmax": 0.78006512, "gate_c2b/L5_rms": 0.56251484, "gate_b2a/L7_absmean": 0.05153323, "gate_b2a/L7_absmax": 0.32006097, "gate_b2a/L7_rms": 0.0569976, "gate_b2a/L11_absmean": 0.08345592, "gate_b2a/L11_absmax": 0.59920782, "gate_b2a/L11_rms": 0.092215, "gate_b2a/L14_absmean": 0.21437289, "gate_b2a/L14_absmax": 0.78985888, "gate_b2a/L14_rms": 0.22273184, "gate_b2c/L3_absmean": 0.03780304, "gate_b2c/L3_absmax": 0.57797289, "gate_b2c/L3_rms": 0.04905617, "gate_b2c/L5_absmean": 0.04043053, "gate_b2c/L5_absmax": 0.59693891, "gate_b2c/L5_rms": 0.05109482, "gate_b2c/L6_absmean": 0.05844342, "gate_b2c/L6_absmax": 0.39085111, "gate_b2c/L6_rms": 0.06535575}, "streams": {"cos_a2b/round0": 0.368814, "cos_c2b/round0": 0.305839, "cos_b2a/round0": 0.142464, "cos_b2c/round0": 0.129509, "rms_a/round0": 5.370933, "rms_b/round0": 9.908504, "rms_c/round0": 3.87475, "cos_a2b/round1": 0.415194, "cos_c2b/round1": 0.294875, "cos_b2a/round1": 0.244857, "cos_b2c/round1": 0.110396, "rms_a/round1": 10.253159, "rms_b/round1": 22.437195, "rms_c/round1": 7.85975, "cos_a2b/round2": 0.545971, "cos_c2b/round2": 0.523199, "cos_b2a/round2": 0.094854, "cos_b2c/round2": 0.175502, "rms_a/round2": 27.765123, "rms_b/round2": 59.126308, "rms_c/round2": 14.686653}, "alpha_mean": 0.4224756280527586, "alpha_hist": [0.01154, 0.08292, 0.17851, 0.20103, 0.18565, 0.16727, 0.11603, 0.0517, 0.00519, 0.00017], "alpha_by_type": {"special": {"share": 0.00089, "mean": 0.5449, "std": 0.12304}, "whitespace": {"share": 0.02919, "mean": 0.50718, "std": 0.13484}, "digit": {"share": 0.04425, "mean": 0.4273, "std": 0.11613}, "word_start": {"share": 0.55095, "mean": 0.44061, "std": 0.16489}, "word_cont": {"share": 0.26277, "mean": 0.31758, "std": 0.13095}, "punct": {"share": 0.11195, "mean": 0.55449, "std": 0.1575}}, "alpha_p05": 0.15791, "alpha_p50": 0.41656, "alpha_p95": 0.68805}
{"step": 44310, "total_steps": 72479, "tokens": 23231201280, "loss": 2.8561470806598663, "l_mix": 1.76763416826725, "l_a": 1.8350710570812225, "l_c": 1.793305054306984, "z_loss": 43.374478816986084, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1833951771259308, "tokens_per_second": 1447904.9159585352, "eta_seconds": 10200.026610326768, "data_wait_seconds": 2.720952033996582e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 44320, "total_steps": 72479, "tokens": 23236444160, "loss": 4.747152745723724, "l_mix": 2.9447951912879944, "l_a": 3.024305582046509, "l_c": 2.983552634716034, "z_loss": 35.38515758514404, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1222384124994278, "tokens_per_second": 1485965.0802250481, "eta_seconds": 9935.244097232819, "data_wait_seconds": 1.5467405319213867e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 44330, "total_steps": 72479, "tokens": 23241687040, "loss": 4.632250428199768, "l_mix": 2.8731662333011627, "l_a": 2.9522820115089417, "l_c": 2.9113316237926483, "z_loss": 35.45404672622681, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13892658054828644, "tokens_per_second": 1399883.9589012675, "eta_seconds": 10542.43304822445, "data_wait_seconds": 2.9474496841430664e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 44340, "total_steps": 72479, "tokens": 23246929920, "loss": 4.581320226192474, "l_mix": 2.841904103755951, "l_a": 2.9225175380706787, "l_c": 2.87553608417511, "z_loss": 35.079224586486816, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1674405187368393, "tokens_per_second": 1394065.108270928, "eta_seconds": 10582.676479363441, "data_wait_seconds": 1.806020736694336e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 44350, "total_steps": 72479, "tokens": 23252172800, "loss": 4.679445862770081, "l_mix": 2.9015819132328033, "l_a": 2.9843752682209015, "l_c": 2.9418377578258514, "z_loss": 35.58983755111694, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15703053772449493, "tokens_per_second": 1395754.5236833363, "eta_seconds": 10566.11094698906, "data_wait_seconds": 1.9282102584838867e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 44360, "total_steps": 72479, "tokens": 23257415680, "loss": 3.1249964237213135, "l_mix": 1.935373142361641, "l_a": 2.004450023174286, "l_c": 1.9609607458114624, "z_loss": 43.73924922943115, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16963765025138855, "tokens_per_second": 1429011.3500589794, "eta_seconds": 10316.54106273651, "data_wait_seconds": 1.800060272216797e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 44370, "total_steps": 72479, "tokens": 23262658560, "loss": 4.432156443595886, "l_mix": 2.7462356090545654, "l_a": 2.8343333303928375, "l_c": 2.785402864217758, "z_loss": 37.19668626785278, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12195231020450592, "tokens_per_second": 1399782.7307798145, "eta_seconds": 10528.213463377953, "data_wait_seconds": 1.671910285949707e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 44380, "total_steps": 72479, "tokens": 23267901440, "loss": 4.443952918052673, "l_mix": 2.7538765966892242, "l_a": 2.84170863032341, "l_c": 2.791879117488861, "z_loss": 37.39197778701782, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.131422758102417, "tokens_per_second": 1422217.831507451, "eta_seconds": 10358.447338819504, "data_wait_seconds": 1.621246337890625e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 44390, "total_steps": 72479, "tokens": 23273144320, "loss": 4.312231957912445, "l_mix": 2.6722413897514343, "l_a": 2.756858229637146, "l_c": 2.7097764909267426, "z_loss": 36.70332384109497, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.29291120171546936, "tokens_per_second": 1441966.5773859015, "eta_seconds": 10212.945197868348, "data_wait_seconds": 1.6748905181884766e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 44400, "total_steps": 72479, "tokens": 23278387200, "loss": 4.381005346775055, "l_mix": 2.7139149010181427, "l_a": 2.804416835308075, "l_c": 2.7525508105754852, "z_loss": 36.57030630111694, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18205784261226654, "tokens_per_second": 1290573.3304356874, "eta_seconds": 11406.93241121769, "data_wait_seconds": 2.35140323638916e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.19629465, "gate_a2b/L1_absmax": 0.72665328, "gate_a2b/L1_rms": 0.20838672, "gate_a2b/L3_absmean": 0.19352774, "gate_a2b/L3_absmax": 0.5734005, "gate_a2b/L3_rms": 0.19969538, "gate_a2b/L5_absmean": 0.29778567, "gate_a2b/L5_absmax": 0.73220879, "gate_a2b/L5_rms": 0.30374131, "gate_c2b/L1_absmean": 0.01852759, "gate_c2b/L1_absmax": 0.62975329, "gate_c2b/L1_rms": 0.06629732, "gate_c2b/L3_absmean": 0.19311318, "gate_c2b/L3_absmax": 0.43805343, "gate_c2b/L3_rms": 0.19718634, "gate_c2b/L5_absmean": 0.55006915, "gate_c2b/L5_absmax": 0.7794084, "gate_c2b/L5_rms": 0.56189531, "gate_b2a/L7_absmean": 0.05089986, "gate_b2a/L7_absmax": 0.31964707, "gate_b2a/L7_rms": 0.05645873, "gate_b2a/L11_absmean": 0.08311076, "gate_b2a/L11_absmax": 0.59795022, "gate_b2a/L11_rms": 0.09191611, "gate_b2a/L14_absmean": 0.21411006, "gate_b2a/L14_absmax": 0.78603935, "gate_b2a/L14_rms": 0.22247575, "gate_b2c/L3_absmean": 0.03764282, "gate_b2c/L3_absmax": 0.57837987, "gate_b2c/L3_rms": 0.04896923, "gate_b2c/L5_absmean": 0.04048365, "gate_b2c/L5_absmax": 0.5976963, "gate_b2c/L5_rms": 0.05121083, "gate_b2c/L6_absmean": 0.05840743, "gate_b2c/L6_absmax": 0.38803658, "gate_b2c/L6_rms": 0.06531214}, "streams": {"cos_a2b/round0": 0.369078, "cos_c2b/round0": 0.307634, "cos_b2a/round0": 0.147965, "cos_b2c/round0": 0.132893, "rms_a/round0": 5.381772, "rms_b/round0": 9.927073, "rms_c/round0": 3.84473, "cos_a2b/round1": 0.415921, "cos_c2b/round1": 0.299894, "cos_b2a/round1": 0.244925, "cos_b2c/round1": 0.117002, "rms_a/round1": 10.281539, "rms_b/round1": 22.431299, "rms_c/round1": 7.841615, "cos_a2b/round2": 0.562548, "cos_c2b/round2": 0.506757, "cos_b2a/round2": 0.105456, "cos_b2c/round2": 0.181279, "rms_a/round2": 28.124325, "rms_b/round2": 58.651253, "rms_c/round2": 14.579797}, "alpha_mean": 0.44877907847772747, "alpha_hist": [0.0105, 0.06834, 0.16568, 0.1778, 0.17346, 0.17102, 0.13892, 0.08253, 0.01166, 9e-05], "alpha_by_type": {"special": {"share": 0.00096, "mean": 0.54308, "std": 0.12459}, "whitespace": {"share": 0.02748, "mean": 0.52231, "std": 0.11576}, "digit": {"share": 0.04005, "mean": 0.43806, "std": 0.11287}, "word_start": {"share": 0.57646, "mean": 0.46969, "std": 0.16908}, "word_cont": {"share": 0.24533, "mean": 0.32248, "std": 0.1392}, "punct": {"share": 0.10971, "mean": 0.606, "std": 0.15809}}, "alpha_p05": 0.18922, "alpha_p50": 0.4615, "alpha_p95": 0.74205}
{"step": 44410, "total_steps": 72479, "tokens": 23283630080, "loss": 4.398526966571808, "l_mix": 2.7250798642635345, "l_a": 2.8156690299510956, "l_c": 2.7624876499176025, "z_loss": 36.86448621749878, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1648765504360199, "tokens_per_second": 1370610.6754246869, "eta_seconds": 10736.994929242135, "data_wait_seconds": 1.6570091247558594e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 44420, "total_steps": 72479, "tokens": 23288872960, "loss": 2.8185090720653534, "l_mix": 1.7454215288162231, "l_a": 1.8067606985569, "l_c": 1.770197480916977, "z_loss": 42.34234046936035, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18271300196647644, "tokens_per_second": 1295995.7252937737, "eta_seconds": 11351.115366268157, "data_wait_seconds": 3.954768180847168e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 44430, "total_steps": 72479, "tokens": 23294115840, "loss": 2.8668845891952515, "l_mix": 1.7741361409425735, "l_a": 1.8387413173913956, "l_c": 1.803753301501274, "z_loss": 44.70976257324219, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15986526012420654, "tokens_per_second": 1339270.268978535, "eta_seconds": 10980.423035311698, "data_wait_seconds": 3.266334533691406e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 44440, "total_steps": 72479, "tokens": 23299358720, "loss": 4.713056564331055, "l_mix": 2.9237060844898224, "l_a": 3.0034303963184357, "l_c": 2.9610710442066193, "z_loss": 36.29497003555298, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1744580715894699, "tokens_per_second": 1463439.1678276784, "eta_seconds": 10045.180937600135, "data_wait_seconds": 1.6868114471435547e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 44450, "total_steps": 72479, "tokens": 23304601600, "loss": 4.656988680362701, "l_mix": 2.8888275921344757, "l_a": 2.9661256670951843, "l_c": 2.9277441799640656, "z_loss": 34.60859155654907, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17027156054973602, "tokens_per_second": 1438496.8668391188, "eta_seconds": 10215.711059761048, "data_wait_seconds": 1.7970800399780273e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 44460, "total_steps": 72479, "tokens": 23309844480, "loss": 4.581556439399719, "l_mix": 2.8418721556663513, "l_a": 2.9211886525154114, "l_c": 2.8777585327625275, "z_loss": 36.16772985458374, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1493014693260193, "tokens_per_second": 1396962.3508664758, "eta_seconds": 10515.691752815248, "data_wait_seconds": 2.3990869522094727e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 44470, "total_steps": 72479, "tokens": 23315087360, "loss": 4.714148640632629, "l_mix": 2.923051595687866, "l_a": 3.0066345036029816, "l_c": 2.963688999414444, "z_loss": 36.594411849975586, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18717537820339203, "tokens_per_second": 1395424.6012291156, "eta_seconds": 10523.52278945446, "data_wait_seconds": 2.1189451217651367e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 44480, "total_steps": 72479, "tokens": 23320330240, "loss": 4.388743281364441, "l_mix": 2.719928741455078, "l_a": 2.805515855550766, "l_c": 2.757199078798294, "z_loss": 36.997979164123535, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.10691184550523758, "tokens_per_second": 1326180.071006271, "eta_seconds": 11069.039591932298, "data_wait_seconds": 1.6570091247558594e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 44490, "total_steps": 72479, "tokens": 23325573120, "loss": 4.467284917831421, "l_mix": 2.769047260284424, "l_a": 2.855425089597702, "l_c": 2.8053669035434723, "z_loss": 36.45657777786255, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16632334887981415, "tokens_per_second": 1448599.0933749592, "eta_seconds": 10129.991727256775, "data_wait_seconds": 2.086162567138672e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 44500, "total_steps": 72479, "tokens": 23330816000, "loss": 4.335774660110474, "l_mix": 2.6861172318458557, "l_a": 2.7714803814888, "l_c": 2.7273772954940796, "z_loss": 37.05353879928589, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11956766992807388, "tokens_per_second": 1467113.5893191523, "eta_seconds": 9998.580927062036, "data_wait_seconds": 1.6808509826660156e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.19576018, "gate_a2b/L1_absmax": 0.72749257, "gate_a2b/L1_rms": 0.20793653, "gate_a2b/L3_absmean": 0.19296516, "gate_a2b/L3_absmax": 0.57670873, "gate_a2b/L3_rms": 0.19916517, "gate_a2b/L5_absmean": 0.29693162, "gate_a2b/L5_absmax": 0.73308688, "gate_a2b/L5_rms": 0.30283573, "gate_c2b/L1_absmean": 0.01874025, "gate_c2b/L1_absmax": 0.63060421, "gate_c2b/L1_rms": 0.06650364, "gate_c2b/L3_absmean": 0.19323613, "gate_c2b/L3_absmax": 0.43440667, "gate_c2b/L3_rms": 0.19726984, "gate_c2b/L5_absmean": 0.54956812, "gate_c2b/L5_absmax": 0.77380067, "gate_c2b/L5_rms": 0.56139553, "gate_b2a/L7_absmean": 0.05165286, "gate_b2a/L7_absmax": 0.31829071, "gate_b2a/L7_rms": 0.05707199, "gate_b2a/L11_absmean": 0.08282292, "gate_b2a/L11_absmax": 0.59907502, "gate_b2a/L11_rms": 0.09172086, "gate_b2a/L14_absmean": 0.21409668, "gate_b2a/L14_absmax": 0.78653526, "gate_b2a/L14_rms": 0.2224686, "gate_b2c/L3_absmean": 0.03764916, "gate_b2c/L3_absmax": 0.57796031, "gate_b2c/L3_rms": 0.048965, "gate_b2c/L5_absmean": 0.04043798, "gate_b2c/L5_absmax": 0.59880167, "gate_b2c/L5_rms": 0.0511292, "gate_b2c/L6_absmean": 0.05801758, "gate_b2c/L6_absmax": 0.38627735, "gate_b2c/L6_rms": 0.06492418}, "streams": {"cos_a2b/round0": 0.369847, "cos_c2b/round0": 0.310183, "cos_b2a/round0": 0.14801, "cos_b2c/round0": 0.13187, "rms_a/round0": 5.344426, "rms_b/round0": 9.901215, "rms_c/round0": 3.865225, "cos_a2b/round1": 0.414553, "cos_c2b/round1": 0.295019, "cos_b2a/round1": 0.248916, "cos_b2c/round1": 0.111807, "rms_a/round1": 10.234103, "rms_b/round1": 22.579706, "rms_c/round1": 7.908144, "cos_a2b/round2": 0.545851, "cos_c2b/round2": 0.519759, "cos_b2a/round2": 0.101944, "cos_b2c/round2": 0.176463, "rms_a/round2": 27.799013, "rms_b/round2": 59.345722, "rms_c/round2": 14.891228}, "alpha_mean": 0.4287792835503126, "alpha_hist": [0.00916, 0.07008, 0.18198, 0.20816, 0.18602, 0.15939, 0.11993, 0.05646, 0.00835, 0.00047], "alpha_by_type": {"special": {"share": 0.00128, "mean": 0.57926, "std": 0.12855}, "whitespace": {"share": 0.03186, "mean": 0.52674, "std": 0.13654}, "digit": {"share": 0.0507, "mean": 0.47349, "std": 0.12851}, "word_start": {"share": 0.54543, "mean": 0.43386, "std": 0.16546}, "word_cont": {"share": 0.26135, "mean": 0.33472, "std": 0.13204}, "punct": {"share": 0.10938, "mean": 0.57719, "std": 0.15635}}, "alpha_p05": 0.18213, "alpha_p50": 0.41775, "alpha_p95": 0.73182}
{"step": 44510, "total_steps": 72479, "tokens": 23336058880, "loss": 4.330411493778229, "l_mix": 2.6839474737644196, "l_a": 2.7662761509418488, "l_c": 2.7219369411468506, "z_loss": 36.874550342559814, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1203215941786766, "tokens_per_second": 1384729.831646592, "eta_seconds": 10589.654918146134, "data_wait_seconds": 1.6361474990844727e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 44520, "total_steps": 72479, "tokens": 23341301760, "loss": 4.399038374423981, "l_mix": 2.7259953916072845, "l_a": 2.8123948872089386, "l_c": 2.76441490650177, "z_loss": 36.947754859924316, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12061598151922226, "tokens_per_second": 1409790.7912798277, "eta_seconds": 10397.690410995483, "data_wait_seconds": 1.7523765563964844e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 44530, "total_steps": 72479, "tokens": 23346544640, "loss": 3.08610600233078, "l_mix": 1.910529002547264, "l_a": 1.978964239358902, "l_c": 1.9396257102489471, "z_loss": 43.856290340423584, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15252886712551117, "tokens_per_second": 1486271.8030199048, "eta_seconds": 9859.1154607296, "data_wait_seconds": 2.3931264877319336e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 44540, "total_steps": 72479, "tokens": 23351787520, "loss": 3.143507480621338, "l_mix": 1.9468185752630234, "l_a": 2.0139688551425934, "l_c": 1.974993884563446, "z_loss": 43.651429653167725, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16614075005054474, "tokens_per_second": 1463173.143552668, "eta_seconds": 10011.175025010109, "data_wait_seconds": 2.637505531311035e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 44550, "total_steps": 72479, "tokens": 23357030400, "loss": 4.451706945896149, "l_mix": 2.758915275335312, "l_a": 2.8437419831752777, "l_c": 2.798896700143814, "z_loss": 36.84113645553589, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13050147891044617, "tokens_per_second": 1430583.0143464035, "eta_seconds": 10235.574870634078, "data_wait_seconds": 2.4557113647460938e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 44560, "total_steps": 72479, "tokens": 23362273280, "loss": 4.331788599491119, "l_mix": 2.6838239133358, "l_a": 2.7731871902942657, "l_c": 2.7200282514095306, "z_loss": 38.027677059173584, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1471519023180008, "tokens_per_second": 1378213.0422524689, "eta_seconds": 10620.706830692292, "data_wait_seconds": 1.609325408935547e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 44570, "total_steps": 72479, "tokens": 23367516160, "loss": 4.347695469856262, "l_mix": 2.693955183029175, "l_a": 2.7770334780216217, "l_c": 2.7354341447353363, "z_loss": 37.379255294799805, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2524600327014923, "tokens_per_second": 1446617.995877018, "eta_seconds": 10114.870569634437, "data_wait_seconds": 3.1054019927978516e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 44580, "total_steps": 72479, "tokens": 23372759040, "loss": 4.333521246910095, "l_mix": 2.6853494942188263, "l_a": 2.7732203602790833, "l_c": 2.7206852436065674, "z_loss": 37.05080461502075, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20767170190811157, "tokens_per_second": 1439225.0961237084, "eta_seconds": 10163.185002398492, "data_wait_seconds": 1.5944242477416992e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 44590, "total_steps": 72479, "tokens": 23378001920, "loss": 4.1819665133953094, "l_mix": 2.5901005566120148, "l_a": 2.67449289560318, "l_c": 2.631726711988449, "z_loss": 37.226102352142334, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.21635662019252777, "tokens_per_second": 1464046.7512333915, "eta_seconds": 9987.295842623711, "data_wait_seconds": 2.8818845748901367e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 44600, "total_steps": 72479, "tokens": 23383244800, "loss": 4.499750077724457, "l_mix": 2.789213240146637, "l_a": 2.8761537969112396, "l_c": 2.8256356716156006, "z_loss": 36.64477062225342, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12487228214740753, "tokens_per_second": 1422275.9664499061, "eta_seconds": 10276.926206159591, "data_wait_seconds": 1.6123056411743164e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.19595772, "gate_a2b/L1_absmax": 0.72728205, "gate_a2b/L1_rms": 0.20817842, "gate_a2b/L3_absmean": 0.19279771, "gate_a2b/L3_absmax": 0.57212454, "gate_a2b/L3_rms": 0.19891211, "gate_a2b/L5_absmean": 0.29643771, "gate_a2b/L5_absmax": 0.73305273, "gate_a2b/L5_rms": 0.30234092, "gate_c2b/L1_absmean": 0.01856945, "gate_c2b/L1_absmax": 0.63106149, "gate_c2b/L1_rms": 0.06648786, "gate_c2b/L3_absmean": 0.19294636, "gate_c2b/L3_absmax": 0.43295139, "gate_c2b/L3_rms": 0.19694723, "gate_c2b/L5_absmean": 0.54880732, "gate_c2b/L5_absmax": 0.77291417, "gate_c2b/L5_rms": 0.56060773, "gate_b2a/L7_absmean": 0.05133161, "gate_b2a/L7_absmax": 0.31941742, "gate_b2a/L7_rms": 0.05680145, "gate_b2a/L11_absmean": 0.08304793, "gate_b2a/L11_absmax": 0.5957036, "gate_b2a/L11_rms": 0.0918224, "gate_b2a/L14_absmean": 0.21371064, "gate_b2a/L14_absmax": 0.78611648, "gate_b2a/L14_rms": 0.22218016, "gate_b2c/L3_absmean": 0.03740202, "gate_b2c/L3_absmax": 0.58220541, "gate_b2c/L3_rms": 0.04885537, "gate_b2c/L5_absmean": 0.04029954, "gate_b2c/L5_absmax": 0.60199821, "gate_b2c/L5_rms": 0.05111627, "gate_b2c/L6_absmean": 0.05808544, "gate_b2c/L6_absmax": 0.3834554, "gate_b2c/L6_rms": 0.06496435}, "streams": {"cos_a2b/round0": 0.369335, "cos_c2b/round0": 0.310903, "cos_b2a/round0": 0.149725, "cos_b2c/round0": 0.132522, "rms_a/round0": 5.413703, "rms_b/round0": 9.919796, "rms_c/round0": 3.880023, "cos_a2b/round1": 0.419041, "cos_c2b/round1": 0.29795, "cos_b2a/round1": 0.248358, "cos_b2c/round1": 0.112127, "rms_a/round1": 10.286237, "rms_b/round1": 22.497072, "rms_c/round1": 7.891141, "cos_a2b/round2": 0.548232, "cos_c2b/round2": 0.526478, "cos_b2a/round2": 0.102133, "cos_b2c/round2": 0.175241, "rms_a/round2": 27.762623, "rms_b/round2": 59.348866, "rms_c/round2": 14.875846}, "alpha_mean": 0.4375951733016308, "alpha_hist": [0.00665, 0.05647, 0.16716, 0.2137, 0.19621, 0.17198, 0.12308, 0.05766, 0.007, 8e-05], "alpha_by_type": {"special": {"share": 0.00114, "mean": 0.57031, "std": 0.11047}, "whitespace": {"share": 0.03133, "mean": 0.52233, "std": 0.11324}, "digit": {"share": 0.03365, "mean": 0.43784, "std": 0.12501}, "word_start": {"share": 0.5451, "mean": 0.45555, "std": 0.15919}, "word_cont": {"share": 0.27417, "mean": 0.33498, "std": 0.12228}, "punct": {"share": 0.11461, "mean": 0.57312, "std": 0.15085}}, "alpha_p05": 0.19042, "alpha_p50": 0.44336, "alpha_p95": 0.72213}
{"step": 44610, "total_steps": 72479, "tokens": 23388487680, "loss": 4.339378863573074, "l_mix": 2.6879444420337677, "l_a": 2.778972327709198, "l_c": 2.725809097290039, "z_loss": 37.76509761810303, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15748022496700287, "tokens_per_second": 1447001.9939674013, "eta_seconds": 10097.693253302574, "data_wait_seconds": 1.5944242477416992e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 44620, "total_steps": 72479, "tokens": 23393730560, "loss": 4.361418008804321, "l_mix": 2.702868700027466, "l_a": 2.786525636911392, "l_c": 2.7419722080230713, "z_loss": 37.27660322189331, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15203483402729034, "tokens_per_second": 1459980.4312586538, "eta_seconds": 10004.339153647423, "data_wait_seconds": 2.5331974029541016e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 44630, "total_steps": 72479, "tokens": 23398973440, "loss": 4.319671452045441, "l_mix": 2.6762572526931763, "l_a": 2.763030469417572, "l_c": 2.715016633272171, "z_loss": 36.989195823669434, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1103975847363472, "tokens_per_second": 1362280.9853245048, "eta_seconds": 10717.977178931236, "data_wait_seconds": 1.627206802368164e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 44640, "total_steps": 72479, "tokens": 23404216320, "loss": 2.8902341723442078, "l_mix": 1.7894786596298218, "l_a": 1.853240266442299, "l_c": 1.8159446269273758, "z_loss": 43.02991342544556, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1876009702682495, "tokens_per_second": 1324624.7888179477, "eta_seconds": 11018.707905220987, "data_wait_seconds": 1.7374753952026367e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 44650, "total_steps": 72479, "tokens": 23409459200, "loss": 2.9654552936553955, "l_mix": 1.8361785113811493, "l_a": 1.901051253080368, "l_c": 1.8632045537233353, "z_loss": 44.52043008804321, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1641717553138733, "tokens_per_second": 1471528.0439577382, "eta_seconds": 9915.14284210205, "data_wait_seconds": 2.2798776626586914e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 44660, "total_steps": 72479, "tokens": 23414702080, "loss": 4.367376506328583, "l_mix": 2.706122577190399, "l_a": 2.7938771545886993, "l_c": 2.7436356246471405, "z_loss": 37.24553060531616, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11579155921936035, "tokens_per_second": 1434353.1149247787, "eta_seconds": 10168.463902115822, "data_wait_seconds": 2.7805566787719727e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 44670, "total_steps": 72479, "tokens": 23419944960, "loss": 4.264193594455719, "l_mix": 2.6421523094177246, "l_a": 2.7283633053302765, "l_c": 2.6784409284591675, "z_loss": 36.446080684661865, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1179892048239708, "tokens_per_second": 1425312.475261951, "eta_seconds": 10229.283223891258, "data_wait_seconds": 2.35140323638916e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 44680, "total_steps": 72479, "tokens": 23425187840, "loss": 4.32203221321106, "l_mix": 2.6788421869277954, "l_a": 2.7592068016529083, "l_c": 2.71809321641922, "z_loss": 36.7310357093811, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12861661612987518, "tokens_per_second": 1433642.0502598686, "eta_seconds": 10166.193234467506, "data_wait_seconds": 2.8520822525024414e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 44690, "total_steps": 72479, "tokens": 23430430720, "loss": 4.39181923866272, "l_mix": 2.721023738384247, "l_a": 2.8071989119052887, "l_c": 2.7621192932128906, "z_loss": 37.37252378463745, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1561889499425888, "tokens_per_second": 1457517.7863355873, "eta_seconds": 9996.062736654281, "data_wait_seconds": 2.154707908630371e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 44700, "total_steps": 72479, "tokens": 23435673600, "loss": 4.444879710674286, "l_mix": 2.7546302378177643, "l_a": 2.8416110277175903, "l_c": 2.792553335428238, "z_loss": 37.202292919158936, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11913099884986877, "tokens_per_second": 1378428.156916245, "eta_seconds": 10565.800095510484, "data_wait_seconds": 1.5556812286376953e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.1955692, "gate_a2b/L1_absmax": 0.72615838, "gate_a2b/L1_rms": 0.20778576, "gate_a2b/L3_absmean": 0.19211929, "gate_a2b/L3_absmax": 0.57398129, "gate_a2b/L3_rms": 0.1982871, "gate_a2b/L5_absmean": 0.29548714, "gate_a2b/L5_absmax": 0.7311914, "gate_a2b/L5_rms": 0.30145779, "gate_c2b/L1_absmean": 0.01850289, "gate_c2b/L1_absmax": 0.62972438, "gate_c2b/L1_rms": 0.06638417, "gate_c2b/L3_absmean": 0.19311044, "gate_c2b/L3_absmax": 0.43374923, "gate_c2b/L3_rms": 0.19708674, "gate_c2b/L5_absmean": 0.54861844, "gate_c2b/L5_absmax": 0.77233762, "gate_c2b/L5_rms": 0.56042296, "gate_b2a/L7_absmean": 0.05163724, "gate_b2a/L7_absmax": 0.32091826, "gate_b2a/L7_rms": 0.05709103, "gate_b2a/L11_absmean": 0.0830328, "gate_b2a/L11_absmax": 0.59904265, "gate_b2a/L11_rms": 0.09188599, "gate_b2a/L14_absmean": 0.21423064, "gate_b2a/L14_absmax": 0.78494185, "gate_b2a/L14_rms": 0.22261298, "gate_b2c/L3_absmean": 0.03722035, "gate_b2c/L3_absmax": 0.57943475, "gate_b2c/L3_rms": 0.04865688, "gate_b2c/L5_absmean": 0.04042672, "gate_b2c/L5_absmax": 0.59873903, "gate_b2c/L5_rms": 0.05110253, "gate_b2c/L6_absmean": 0.05778693, "gate_b2c/L6_absmax": 0.3832185, "gate_b2c/L6_rms": 0.06468355}, "streams": {"cos_a2b/round0": 0.366323, "cos_c2b/round0": 0.312032, "cos_b2a/round0": 0.14724, "cos_b2c/round0": 0.134589, "rms_a/round0": 5.444394, "rms_b/round0": 9.966346, "rms_c/round0": 3.834751, "cos_a2b/round1": 0.428794, "cos_c2b/round1": 0.309645, "cos_b2a/round1": 0.250285, "cos_b2c/round1": 0.107716, "rms_a/round1": 10.332246, "rms_b/round1": 22.296871, "rms_c/round1": 7.858754, "cos_a2b/round2": 0.564155, "cos_c2b/round2": 0.5159, "cos_b2a/round2": 0.114851, "cos_b2c/round2": 0.180475, "rms_a/round2": 28.179079, "rms_b/round2": 58.838242, "rms_c/round2": 14.703678}, "alpha_mean": 0.46758439785156014, "alpha_hist": [0.00618, 0.04536, 0.13503, 0.1895, 0.1833, 0.18912, 0.15608, 0.08574, 0.00955, 0.00014], "alpha_by_type": {"special": {"share": 0.00082, "mean": 0.56189, "std": 0.12107}, "whitespace": {"share": 0.0177, "mean": 0.53516, "std": 0.11476}, "digit": {"share": 0.021, "mean": 0.4318, "std": 0.13923}, "word_start": {"share": 0.59679, "mean": 0.49014, "std": 0.15714}, "word_cont": {"share": 0.23856, "mean": 0.3415, "std": 0.13008}, "punct": {"share": 0.12514, "mean": 0.59619, "std": 0.14951}}, "alpha_p05": 0.20181, "alpha_p50": 0.46199, "alpha_p95": 0.72817}
{"step": 44710, "total_steps": 72479, "tokens": 23440916480, "loss": 4.674038946628571, "l_mix": 2.899252623319626, "l_a": 2.9791455566883087, "l_c": 2.9368084967136383, "z_loss": 35.02233839035034, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12557156383991241, "tokens_per_second": 1429158.1811499605, "eta_seconds": 10187.08332221508, "data_wait_seconds": 2.816319465637207e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 44720, "total_steps": 72479, "tokens": 23446159360, "loss": 4.590145528316498, "l_mix": 2.847994565963745, "l_a": 2.9235376715660095, "l_c": 2.8836322724819183, "z_loss": 35.14544343948364, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1681886613368988, "tokens_per_second": 1418156.5949175586, "eta_seconds": 10262.41435124874, "data_wait_seconds": 1.9431114196777344e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 44730, "total_steps": 72479, "tokens": 23451402240, "loss": 4.608365476131439, "l_mix": 2.8584034740924835, "l_a": 2.938577950000763, "l_c": 2.894628345966339, "z_loss": 36.29644012451172, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1567291021347046, "tokens_per_second": 1344140.657332231, "eta_seconds": 10823.620007801057, "data_wait_seconds": 1.7583370208740234e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 44740, "total_steps": 72479, "tokens": 23456645120, "loss": 4.7142751812934875, "l_mix": 2.924464225769043, "l_a": 3.0023093223571777, "l_c": 2.9637266993522644, "z_loss": 35.74341344833374, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11591706424951553, "tokens_per_second": 1392647.138708468, "eta_seconds": 10442.864116668701, "data_wait_seconds": 2.4616718292236328e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 44750, "total_steps": 72479, "tokens": 23461888000, "loss": 4.703295111656189, "l_mix": 2.9184220135211945, "l_a": 2.9945446252822876, "l_c": 2.955032229423523, "z_loss": 34.760690212249756, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13951566815376282, "tokens_per_second": 1445685.3130391678, "eta_seconds": 10056.117898464203, "data_wait_seconds": 2.6613473892211914e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 44760, "total_steps": 72479, "tokens": 23467130880, "loss": 2.9194566011428833, "l_mix": 1.806901603937149, "l_a": 1.8669784665107727, "l_c": 1.841537892818451, "z_loss": 41.86575984954834, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.24277621507644653, "tokens_per_second": 1410863.5314509529, "eta_seconds": 10300.598709964752, "data_wait_seconds": 2.4527311325073242e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 44770, "total_steps": 72479, "tokens": 23472373760, "loss": 4.463599443435669, "l_mix": 2.7664643228054047, "l_a": 2.85288205742836, "l_c": 2.80423504114151, "z_loss": 37.194462299346924, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13371969759464264, "tokens_per_second": 1411550.9060931238, "eta_seconds": 10291.868418836593, "data_wait_seconds": 1.683831214904785e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 44780, "total_steps": 72479, "tokens": 23477616640, "loss": 4.406325399875641, "l_mix": 2.7298903465270996, "l_a": 2.8195621967315674, "l_c": 2.7685543298721313, "z_loss": 37.0845513343811, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12563857436180115, "tokens_per_second": 1401555.713928614, "eta_seconds": 10361.524103307724, "data_wait_seconds": 1.8835067749023438e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 44790, "total_steps": 72479, "tokens": 23482859520, "loss": 4.425887763500214, "l_mix": 2.743033528327942, "l_a": 2.8293817043304443, "l_c": 2.7801323533058167, "z_loss": 36.78581142425537, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11162059009075165, "tokens_per_second": 1346727.9942300145, "eta_seconds": 10779.467341732978, "data_wait_seconds": 1.8090009689331055e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 44800, "total_steps": 72479, "tokens": 23488102400, "loss": 4.334169805049896, "l_mix": 2.6855449080467224, "l_a": 2.7723891139030457, "l_c": 2.723026782274246, "z_loss": 36.63516139984131, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14385275542736053, "tokens_per_second": 1469514.6139323881, "eta_seconds": 9875.211457180978, "data_wait_seconds": 2.6345252990722656e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.19472186, "gate_a2b/L1_absmax": 0.72588557, "gate_a2b/L1_rms": 0.20688918, "gate_a2b/L3_absmean": 0.19221441, "gate_a2b/L3_absmax": 0.57085294, "gate_a2b/L3_rms": 0.19832355, "gate_a2b/L5_absmean": 0.29501727, "gate_a2b/L5_absmax": 0.73409182, "gate_a2b/L5_rms": 0.30099231, "gate_c2b/L1_absmean": 0.01899161, "gate_c2b/L1_absmax": 0.63124514, "gate_c2b/L1_rms": 0.0664826, "gate_c2b/L3_absmean": 0.19240905, "gate_c2b/L3_absmax": 0.43511209, "gate_c2b/L3_rms": 0.19640861, "gate_c2b/L5_absmean": 0.54716271, "gate_c2b/L5_absmax": 0.77168423, "gate_c2b/L5_rms": 0.55887765, "gate_b2a/L7_absmean": 0.05171895, "gate_b2a/L7_absmax": 0.32103789, "gate_b2a/L7_rms": 0.0571186, "gate_b2a/L11_absmean": 0.08292247, "gate_b2a/L11_absmax": 0.59920233, "gate_b2a/L11_rms": 0.09177765, "gate_b2a/L14_absmean": 0.21333471, "gate_b2a/L14_absmax": 0.78917694, "gate_b2a/L14_rms": 0.22185852, "gate_b2c/L3_absmean": 0.03723664, "gate_b2c/L3_absmax": 0.58100265, "gate_b2c/L3_rms": 0.0486942, "gate_b2c/L5_absmean": 0.04018556, "gate_b2c/L5_absmax": 0.59190661, "gate_b2c/L5_rms": 0.05073639, "gate_b2c/L6_absmean": 0.05771247, "gate_b2c/L6_absmax": 0.38584265, "gate_b2c/L6_rms": 0.06463955}, "streams": {"cos_a2b/round0": 0.370784, "cos_c2b/round0": 0.316158, "cos_b2a/round0": 0.137952, "cos_b2c/round0": 0.136058, "rms_a/round0": 5.45504, "rms_b/round0": 9.95421, "rms_c/round0": 3.86796, "cos_a2b/round1": 0.410956, "cos_c2b/round1": 0.290198, "cos_b2a/round1": 0.236389, "cos_b2c/round1": 0.106639, "rms_a/round1": 10.358347, "rms_b/round1": 22.640114, "rms_c/round1": 7.857224, "cos_a2b/round2": 0.553022, "cos_c2b/round2": 0.50723, "cos_b2a/round2": 0.100926, "cos_b2c/round2": 0.171293, "rms_a/round2": 28.193993, "rms_b/round2": 59.342094, "rms_c/round2": 14.810184}, "alpha_mean": 0.4658548025174838, "alpha_hist": [0.0063, 0.04362, 0.14456, 0.18684, 0.18753, 0.1813, 0.15233, 0.08566, 0.01147, 0.00037], "alpha_by_type": {"special": {"share": 0.00056, "mean": 0.56339, "std": 0.16664}, "whitespace": {"share": 0.03067, "mean": 0.58764, "std": 0.14167}, "digit": {"share": 0.06718, "mean": 0.52456, "std": 0.12079}, "word_start": {"share": 0.55338, "mean": 0.46943, "std": 0.16102}, "word_cont": {"share": 0.22517, "mean": 0.34445, "std": 0.1333}, "punct": {"share": 0.12303, "mean": 0.6091, "std": 0.14411}}, "alpha_p05": 0.20371, "alpha_p50": 0.47116, "alpha_p95": 0.74167}
{"step": 44810, "total_steps": 72479, "tokens": 23493345280, "loss": 4.438450634479523, "l_mix": 2.7509281933307648, "l_a": 2.837568938732147, "l_c": 2.7875058948993683, "z_loss": 37.05207824707031, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18621811270713806, "tokens_per_second": 1444798.161106103, "eta_seconds": 10040.519888877869, "data_wait_seconds": 2.4646520614624023e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 44820, "total_steps": 72479, "tokens": 23498588160, "loss": 3.1112916469573975, "l_mix": 1.9257014989852905, "l_a": 1.996050775051117, "l_c": 1.9559161067008972, "z_loss": 44.02953863143921, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1396668255329132, "tokens_per_second": 1383308.9378375593, "eta_seconds": 10483.03917899132, "data_wait_seconds": 3.1441450119018555e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 44830, "total_steps": 72479, "tokens": 23503831040, "loss": 4.644348084926605, "l_mix": 2.8817123472690582, "l_a": 2.959896296262741, "l_c": 2.915556013584137, "z_loss": 35.71002388000488, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12333881855010986, "tokens_per_second": 1392541.487071665, "eta_seconds": 10409.77166323662, "data_wait_seconds": 2.4080276489257812e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 44840, "total_steps": 72479, "tokens": 23509073920, "loss": 4.686746060848236, "l_mix": 2.907502144575119, "l_a": 2.983811140060425, "l_c": 2.947002023458481, "z_loss": 35.60874843597412, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17255553603172302, "tokens_per_second": 1331776.4344391685, "eta_seconds": 10880.802255749702, "data_wait_seconds": 2.3812055587768555e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 44850, "total_steps": 72479, "tokens": 23514316800, "loss": 4.695280373096466, "l_mix": 2.9125093519687653, "l_a": 2.9902477860450745, "l_c": 2.9523220658302307, "z_loss": 36.3850998878479, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12839587032794952, "tokens_per_second": 1432036.2431919575, "eta_seconds": 10115.353728556633, "data_wait_seconds": 2.7477741241455078e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 44860, "total_steps": 72479, "tokens": 23519559680, "loss": 4.75983190536499, "l_mix": 2.9528648257255554, "l_a": 3.0343309938907623, "l_c": 2.988892585039139, "z_loss": 35.971354484558105, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13808484375476837, "tokens_per_second": 1400173.3774218794, "eta_seconds": 10341.798026943208, "data_wait_seconds": 3.1322240829467773e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 44870, "total_steps": 72479, "tokens": 23524802560, "loss": 4.720823109149933, "l_mix": 2.9282084703445435, "l_a": 3.0082023441791534, "l_c": 2.967179596424103, "z_loss": 35.82485866546631, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1549047976732254, "tokens_per_second": 1468447.448468991, "eta_seconds": 9857.395582723617, "data_wait_seconds": 2.613663673400879e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 44880, "total_steps": 72479, "tokens": 23530045440, "loss": 4.3561301827430725, "l_mix": 2.699880540370941, "l_a": 2.788560152053833, "l_c": 2.732271671295166, "z_loss": 36.08661127090454, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.177824467420578, "tokens_per_second": 1414677.2437721086, "eta_seconds": 10228.357440328598, "data_wait_seconds": 2.9265880584716797e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 44890, "total_steps": 72479, "tokens": 23535288320, "loss": 4.488170385360718, "l_mix": 2.781734734773636, "l_a": 2.866710215806961, "l_c": 2.8214083313941956, "z_loss": 36.28811311721802, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1642964631319046, "tokens_per_second": 1365253.8532327274, "eta_seconds": 10594.792754292488, "data_wait_seconds": 2.0742416381835938e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 44900, "total_steps": 72479, "tokens": 23540531200, "loss": 4.519514560699463, "l_mix": 2.80072021484375, "l_a": 2.889292925596237, "l_c": 2.840021163225174, "z_loss": 37.36677122116089, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18471992015838623, "tokens_per_second": 1464029.4988851123, "eta_seconds": 9876.398503589631, "data_wait_seconds": 1.4990568161010742e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.19515143, "gate_a2b/L1_absmax": 0.72466314, "gate_a2b/L1_rms": 0.20723526, "gate_a2b/L3_absmean": 0.19206397, "gate_a2b/L3_absmax": 0.57007527, "gate_a2b/L3_rms": 0.19819202, "gate_a2b/L5_absmean": 0.29406282, "gate_a2b/L5_absmax": 0.73274046, "gate_a2b/L5_rms": 0.29998973, "gate_c2b/L1_absmean": 0.01872062, "gate_c2b/L1_absmax": 0.62964922, "gate_c2b/L1_rms": 0.06642812, "gate_c2b/L3_absmean": 0.19247547, "gate_c2b/L3_absmax": 0.43254468, "gate_c2b/L3_rms": 0.1964331, "gate_c2b/L5_absmean": 0.54684329, "gate_c2b/L5_absmax": 0.76736522, "gate_c2b/L5_rms": 0.55849648, "gate_b2a/L7_absmean": 0.05139031, "gate_b2a/L7_absmax": 0.32058239, "gate_b2a/L7_rms": 0.056853, "gate_b2a/L11_absmean": 0.08303055, "gate_b2a/L11_absmax": 0.60015827, "gate_b2a/L11_rms": 0.09190527, "gate_b2a/L14_absmean": 0.21349615, "gate_b2a/L14_absmax": 0.78752661, "gate_b2a/L14_rms": 0.22195578, "gate_b2c/L3_absmean": 0.03733277, "gate_b2c/L3_absmax": 0.57850683, "gate_b2c/L3_rms": 0.04874718, "gate_b2c/L5_absmean": 0.04036484, "gate_b2c/L5_absmax": 0.59400773, "gate_b2c/L5_rms": 0.05100248, "gate_b2c/L6_absmean": 0.05763509, "gate_b2c/L6_absmax": 0.38424781, "gate_b2c/L6_rms": 0.06455998}, "streams": {"cos_a2b/round0": 0.364584, "cos_c2b/round0": 0.305802, "cos_b2a/round0": 0.150045, "cos_b2c/round0": 0.130885, "rms_a/round0": 5.449249, "rms_b/round0": 10.101903, "rms_c/round0": 3.907908, "cos_a2b/round1": 0.420528, "cos_c2b/round1": 0.301942, "cos_b2a/round1": 0.249731, "cos_b2c/round1": 0.115372, "rms_a/round1": 10.559396, "rms_b/round1": 23.026102, "rms_c/round1": 8.034822, "cos_a2b/round2": 0.546129, "cos_c2b/round2": 0.527036, "cos_b2a/round2": 0.098999, "cos_b2c/round2": 0.177669, "rms_a/round2": 28.435837, "rms_b/round2": 60.190781, "rms_c/round2": 15.219731}, "alpha_mean": 0.4222045954169005, "alpha_hist": [0.01263, 0.07581, 0.18272, 0.20753, 0.19098, 0.15321, 0.11554, 0.0558, 0.00572, 5e-05], "alpha_by_type": {"special": {"share": 0.00117, "mean": 0.54666, "std": 0.12313}, "whitespace": {"share": 0.024, "mean": 0.518, "std": 0.11973}, "digit": {"share": 0.02165, "mean": 0.46203, "std": 0.11785}, "word_start": {"share": 0.56805, "mean": 0.44151, "std": 0.16324}, "word_cont": {"share": 0.27029, "mean": 0.31336, "std": 0.12631}, "punct": {"share": 0.11482, "mean": 0.55412, "std": 0.15927}}, "alpha_p05": 0.15355, "alpha_p50": 0.4071, "alpha_p95": 0.69781}
{"step": 44910, "total_steps": 72479, "tokens": 23545774080, "loss": 4.467299401760101, "l_mix": 2.7686818540096283, "l_a": 2.8560020327568054, "l_c": 2.8060560524463654, "z_loss": 36.588162422180176, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14161227643489838, "tokens_per_second": 1392799.382862757, "eta_seconds": 10377.729951524736, "data_wait_seconds": 1.5020370483398438e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 44920, "total_steps": 72479, "tokens": 23551016960, "loss": 4.4647961258888245, "l_mix": 2.768057644367218, "l_a": 2.8509733080863953, "l_c": 2.804821401834488, "z_loss": 36.20477914810181, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1477545201778412, "tokens_per_second": 1437377.9519875087, "eta_seconds": 10052.229458522796, "data_wait_seconds": 1.5944242477416992e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 44930, "total_steps": 72479, "tokens": 23556259840, "loss": 3.049107164144516, "l_mix": 1.8865272998809814, "l_a": 1.957175388932228, "l_c": 1.9180907905101776, "z_loss": 42.12594795227051, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19894103705883026, "tokens_per_second": 1477278.9679498347, "eta_seconds": 9777.171695637702, "data_wait_seconds": 3.081560134887695e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 44940, "total_steps": 72479, "tokens": 23561502720, "loss": 3.0808822214603424, "l_mix": 1.9049444496631622, "l_a": 1.9839476943016052, "l_c": 1.935844600200653, "z_loss": 47.19283151626587, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.23145531117916107, "tokens_per_second": 1437251.4080058697, "eta_seconds": 10045.818811917305, "data_wait_seconds": 2.0742416381835938e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 44950, "total_steps": 72479, "tokens": 23566745600, "loss": 3.1078003644943237, "l_mix": 1.9116406291723251, "l_a": 2.0272172391414642, "l_c": 1.9599818289279938, "z_loss": 38.98035526275635, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17141857743263245, "tokens_per_second": 1303488.4012999418, "eta_seconds": 11072.691047811508, "data_wait_seconds": 3.641843795776367e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 44960, "total_steps": 72479, "tokens": 23571988480, "loss": 4.870977878570557, "l_mix": 3.0227798223495483, "l_a": 3.0981008112430573, "l_c": 3.0625590682029724, "z_loss": 34.75545263290405, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15434688329696655, "tokens_per_second": 1464297.7847799342, "eta_seconds": 9853.106124973297, "data_wait_seconds": 3.129243850708008e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 44970, "total_steps": 72479, "tokens": 23577231360, "loss": 4.725570321083069, "l_mix": 2.9324361979961395, "l_a": 3.008243888616562, "l_c": 2.9688697457313538, "z_loss": 36.19053268432617, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17466284334659576, "tokens_per_second": 1332472.2883548606, "eta_seconds": 10823.968887043, "data_wait_seconds": 2.6285648345947266e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 44980, "total_steps": 72479, "tokens": 23582474240, "loss": 4.687559962272644, "l_mix": 2.908114016056061, "l_a": 2.982606828212738, "l_c": 2.9488790929317474, "z_loss": 37.232165813446045, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17041084170341492, "tokens_per_second": 1310466.5333997677, "eta_seconds": 11001.727510428429, "data_wait_seconds": 1.8209218978881836e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 44990, "total_steps": 72479, "tokens": 23587717120, "loss": 4.401304841041565, "l_mix": 2.7283738553524017, "l_a": 2.8145350217819214, "l_c": 2.7619013488292694, "z_loss": 36.37809658050537, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13930009305477142, "tokens_per_second": 1429455.000239215, "eta_seconds": 10082.271096038818, "data_wait_seconds": 1.996755599975586e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 45000, "total_steps": 72479, "tokens": 23592960000, "loss": 4.458061754703522, "l_mix": 2.762553960084915, "l_a": 2.850688338279724, "l_c": 2.8010043799877167, "z_loss": 36.47020769119263, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1307995766401291, "tokens_per_second": 1453065.493410956, "eta_seconds": 9914.838675427438, "data_wait_seconds": 2.345442771911621e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.19396673, "gate_a2b/L1_absmax": 0.72776669, "gate_a2b/L1_rms": 0.20621872, "gate_a2b/L3_absmean": 0.19121544, "gate_a2b/L3_absmax": 0.57301444, "gate_a2b/L3_rms": 0.19748425, "gate_a2b/L5_absmean": 0.29366526, "gate_a2b/L5_absmax": 0.73510027, "gate_a2b/L5_rms": 0.29968607, "gate_c2b/L1_absmean": 0.01899335, "gate_c2b/L1_absmax": 0.63205671, "gate_c2b/L1_rms": 0.06691555, "gate_c2b/L3_absmean": 0.19260548, "gate_c2b/L3_absmax": 0.43314096, "gate_c2b/L3_rms": 0.19664226, "gate_c2b/L5_absmean": 0.54625982, "gate_c2b/L5_absmax": 0.77196318, "gate_c2b/L5_rms": 0.55787855, "gate_b2a/L7_absmean": 0.05144957, "gate_b2a/L7_absmax": 0.31948248, "gate_b2a/L7_rms": 0.05687864, "gate_b2a/L11_absmean": 0.08313917, "gate_b2a/L11_absmax": 0.59660649, "gate_b2a/L11_rms": 0.09195856, "gate_b2a/L14_absmean": 0.21379173, "gate_b2a/L14_absmax": 0.79291791, "gate_b2a/L14_rms": 0.2222724, "gate_b2c/L3_absmean": 0.03741969, "gate_b2c/L3_absmax": 0.58061272, "gate_b2c/L3_rms": 0.04880642, "gate_b2c/L5_absmean": 0.04017764, "gate_b2c/L5_absmax": 0.58214623, "gate_b2c/L5_rms": 0.05047756, "gate_b2c/L6_absmean": 0.05800637, "gate_b2c/L6_absmax": 0.38485721, "gate_b2c/L6_rms": 0.06487618}, "streams": {"cos_a2b/round0": 0.36629, "cos_c2b/round0": 0.313275, "cos_b2a/round0": 0.150591, "cos_b2c/round0": 0.135902, "rms_a/round0": 5.488191, "rms_b/round0": 10.123851, "rms_c/round0": 3.864261, "cos_a2b/round1": 0.422261, "cos_c2b/round1": 0.304534, "cos_b2a/round1": 0.248505, "cos_b2c/round1": 0.113364, "rms_a/round1": 10.54578, "rms_b/round1": 22.718143, "rms_c/round1": 7.999241, "cos_a2b/round2": 0.557826, "cos_c2b/round2": 0.519602, "cos_b2a/round2": 0.106868, "cos_b2c/round2": 0.182331, "rms_a/round2": 28.914801, "rms_b/round2": 60.249104, "rms_c/round2": 14.974486}, "alpha_mean": 0.4517163028693787, "alpha_hist": [0.00826, 0.0609, 0.15944, 0.18256, 0.17856, 0.1833, 0.14186, 0.07475, 0.01027, 0.00011], "alpha_by_type": {"special": {"share": 0.00073, "mean": 0.56792, "std": 0.10563}, "whitespace": {"share": 0.02673, "mean": 0.54852, "std": 0.11007}, "digit": {"share": 0.02731, "mean": 0.4875, "std": 0.12129}, "word_start": {"share": 0.57477, "mean": 0.46823, "std": 0.16261}, "word_cont": {"share": 0.25658, "mean": 0.33617, "std": 0.13726}, "punct": {"share": 0.11388, "mean": 0.59663, "std": 0.16294}}, "alpha_p05": 0.17667, "alpha_p50": 0.43818, "alpha_p95": 0.73791}
{"step": 45010, "total_steps": 72479, "tokens": 23598202880, "loss": 4.433941900730133, "l_mix": 2.7473549842834473, "l_a": 2.837101548910141, "l_c": 2.7848548591136932, "z_loss": 37.36407423019409, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15043675899505615, "tokens_per_second": 1411070.3073598184, "eta_seconds": 10206.20092201233, "data_wait_seconds": 1.6361474990844727e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 45020, "total_steps": 72479, "tokens": 23603445760, "loss": 4.424863815307617, "l_mix": 2.7424203753471375, "l_a": 2.8286556601524353, "l_c": 2.7794889509677887, "z_loss": 36.61742973327637, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11989867687225342, "tokens_per_second": 1474261.1445669727, "eta_seconds": 9765.17915096283, "data_wait_seconds": 1.6123056411743164e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 45030, "total_steps": 72479, "tokens": 23608688640, "loss": 4.423258125782013, "l_mix": 2.7412282824516296, "l_a": 2.828986346721649, "l_c": 2.777779847383499, "z_loss": 36.53184652328491, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12421014159917831, "tokens_per_second": 1387683.1052170086, "eta_seconds": 10370.653975605965, "data_wait_seconds": 1.704692840576172e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 45040, "total_steps": 72479, "tokens": 23613931520, "loss": 2.7502590119838715, "l_mix": 1.7027573436498642, "l_a": 1.764357253909111, "l_c": 1.727314755320549, "z_loss": 44.28165912628174, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17810845375061035, "tokens_per_second": 1409271.7407680058, "eta_seconds": 10208.065638327598, "data_wait_seconds": 2.944469451904297e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 45050, "total_steps": 72479, "tokens": 23619174400, "loss": 2.9379806518554688, "l_mix": 1.817741334438324, "l_a": 1.8857296705245972, "l_c": 1.8484009355306625, "z_loss": 46.40870189666748, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20692069828510284, "tokens_per_second": 1426701.7848843227, "eta_seconds": 10079.678671717644, "data_wait_seconds": 2.3365020751953125e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 45060, "total_steps": 72479, "tokens": 23624417280, "loss": 3.11287721991539, "l_mix": 1.9144695103168488, "l_a": 2.031342461705208, "l_c": 1.9633496701717377, "z_loss": 40.06725454330444, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17809456586837769, "tokens_per_second": 1327938.5753392759, "eta_seconds": 10825.389772510529, "data_wait_seconds": 2.8014183044433594e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 45070, "total_steps": 72479, "tokens": 23629660160, "loss": 3.157818228006363, "l_mix": 1.9430855959653854, "l_a": 2.0581936687231064, "l_c": 1.9909148663282394, "z_loss": 39.25749349594116, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1793849915266037, "tokens_per_second": 1472857.5160061885, "eta_seconds": 9756.68700864315, "data_wait_seconds": 1.6033649444580078e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 45080, "total_steps": 72479, "tokens": 23634903040, "loss": 4.326487839221954, "l_mix": 2.681406706571579, "l_a": 2.7665669918060303, "l_c": 2.7170369625091553, "z_loss": 36.54465961456299, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15132325887680054, "tokens_per_second": 1444523.2132241516, "eta_seconds": 9944.434800696372, "data_wait_seconds": 2.703070640563965e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 45090, "total_steps": 72479, "tokens": 23640145920, "loss": 4.486355245113373, "l_mix": 2.78104767203331, "l_a": 2.8670012056827545, "l_c": 2.817357212305069, "z_loss": 36.0662055015564, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12960276007652283, "tokens_per_second": 1450074.8310832353, "eta_seconds": 9902.74689567089, "data_wait_seconds": 1.537799835205078e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 45100, "total_steps": 72479, "tokens": 23645388800, "loss": 4.456746995449066, "l_mix": 2.762183755636215, "l_a": 2.8502013981342316, "l_c": 2.7983424067497253, "z_loss": 36.56900215148926, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15859712660312653, "tokens_per_second": 1431083.8907941605, "eta_seconds": 10030.495936918258, "data_wait_seconds": 1.8090009689331055e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.19355799, "gate_a2b/L1_absmax": 0.72681653, "gate_a2b/L1_rms": 0.20577674, "gate_a2b/L3_absmean": 0.19060355, "gate_a2b/L3_absmax": 0.573852, "gate_a2b/L3_rms": 0.19685601, "gate_a2b/L5_absmean": 0.29276347, "gate_a2b/L5_absmax": 0.73432261, "gate_a2b/L5_rms": 0.29876307, "gate_c2b/L1_absmean": 0.01895928, "gate_c2b/L1_absmax": 0.63251436, "gate_c2b/L1_rms": 0.06672273, "gate_c2b/L3_absmean": 0.19267054, "gate_c2b/L3_absmax": 0.43350619, "gate_c2b/L3_rms": 0.19660263, "gate_c2b/L5_absmean": 0.54591805, "gate_c2b/L5_absmax": 0.77229548, "gate_c2b/L5_rms": 0.55749065, "gate_b2a/L7_absmean": 0.05133519, "gate_b2a/L7_absmax": 0.32026201, "gate_b2a/L7_rms": 0.05678388, "gate_b2a/L11_absmean": 0.08261951, "gate_b2a/L11_absmax": 0.59696501, "gate_b2a/L11_rms": 0.09146262, "gate_b2a/L14_absmean": 0.21365684, "gate_b2a/L14_absmax": 0.79148442, "gate_b2a/L14_rms": 0.22211166, "gate_b2c/L3_absmean": 0.0371079, "gate_b2c/L3_absmax": 0.57988328, "gate_b2c/L3_rms": 0.04856796, "gate_b2c/L5_absmean": 0.04031242, "gate_b2c/L5_absmax": 0.58591163, "gate_b2c/L5_rms": 0.05063505, "gate_b2c/L6_absmean": 0.05762117, "gate_b2c/L6_absmax": 0.38417208, "gate_b2c/L6_rms": 0.06450494}, "streams": {"cos_a2b/round0": 0.367307, "cos_c2b/round0": 0.305492, "cos_b2a/round0": 0.148533, "cos_b2c/round0": 0.128627, "rms_a/round0": 5.503048, "rms_b/round0": 10.091384, "rms_c/round0": 3.9429, "cos_a2b/round1": 0.415313, "cos_c2b/round1": 0.295222, "cos_b2a/round1": 0.249653, "cos_b2c/round1": 0.112626, "rms_a/round1": 10.524736, "rms_b/round1": 23.03252, "rms_c/round1": 8.060739, "cos_a2b/round2": 0.541161, "cos_c2b/round2": 0.52677, "cos_b2a/round2": 0.099542, "cos_b2c/round2": 0.175894, "rms_a/round2": 28.475487, "rms_b/round2": 60.608162, "rms_c/round2": 15.274039}, "alpha_mean": 0.42770805492330055, "alpha_hist": [0.00795, 0.06693, 0.17622, 0.22217, 0.19339, 0.15572, 0.11111, 0.05902, 0.0074, 9e-05], "alpha_by_type": {"special": {"share": 0.00104, "mean": 0.57266, "std": 0.12516}, "whitespace": {"share": 0.02817, "mean": 0.53575, "std": 0.11857}, "digit": {"share": 0.03537, "mean": 0.45133, "std": 0.11776}, "word_start": {"share": 0.52554, "mean": 0.44771, "std": 0.16244}, "word_cont": {"share": 0.28825, "mean": 0.32464, "std": 0.12288}, "punct": {"share": 0.12163, "mean": 0.55242, "std": 0.15743}}, "alpha_p05": 0.17189, "alpha_p50": 0.40851, "alpha_p95": 0.72604}
{"step": 45110, "total_steps": 72479, "tokens": 23650631680, "loss": 4.473300039768219, "l_mix": 2.772815316915512, "l_a": 2.858935058116913, "l_c": 2.8093471825122833, "z_loss": 36.54906368255615, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14133486151695251, "tokens_per_second": 1324340.4734054587, "eta_seconds": 10835.006979060174, "data_wait_seconds": 1.7374753952026367e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 45120, "total_steps": 72479, "tokens": 23655874560, "loss": 4.512152910232544, "l_mix": 2.7965561747550964, "l_a": 2.8818923234939575, "l_c": 2.8367629051208496, "z_loss": 36.90725898742676, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12500138580799103, "tokens_per_second": 1416574.959955608, "eta_seconds": 10125.828704786301, "data_wait_seconds": 2.142786979675293e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 45130, "total_steps": 72479, "tokens": 23661117440, "loss": 4.436724066734314, "l_mix": 2.7499155700206757, "l_a": 2.8364165127277374, "l_c": 2.786278188228607, "z_loss": 36.74147415161133, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14401350915431976, "tokens_per_second": 1445560.6280726877, "eta_seconds": 9919.163702678681, "data_wait_seconds": 1.704692840576172e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 45140, "total_steps": 72479, "tokens": 23666360320, "loss": 4.430309414863586, "l_mix": 2.7454925179481506, "l_a": 2.834562808275223, "l_c": 2.7814934253692627, "z_loss": 36.292190074920654, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.10864496976137161, "tokens_per_second": 1468906.7038028718, "eta_seconds": 9757.944187259674, "data_wait_seconds": 1.576542854309082e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 45150, "total_steps": 72479, "tokens": 23671603200, "loss": 4.352592289447784, "l_mix": 2.6967659294605255, "l_a": 2.786151945590973, "l_c": 2.73326912522316, "z_loss": 36.886335372924805, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12983976304531097, "tokens_per_second": 1393338.6795103382, "eta_seconds": 10283.405580210685, "data_wait_seconds": 2.6524066925048828e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 45160, "total_steps": 72479, "tokens": 23676846080, "loss": 2.9512627124786377, "l_mix": 1.825674131512642, "l_a": 1.898709923028946, "l_c": 1.8532517552375793, "z_loss": 43.239216804504395, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17949466407299042, "tokens_per_second": 1399268.8853181785, "eta_seconds": 10236.076870059967, "data_wait_seconds": 1.7851591110229492e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 45170, "total_steps": 72479, "tokens": 23682088960, "loss": 3.190556436777115, "l_mix": 1.9649730324745178, "l_a": 2.0736665427684784, "l_c": 2.0116113424301147, "z_loss": 38.122984409332275, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2748897969722748, "tokens_per_second": 1411173.4463680508, "eta_seconds": 10146.010774827004, "data_wait_seconds": 1.7255544662475586e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 45180, "total_steps": 72479, "tokens": 23687331840, "loss": 3.1798338294029236, "l_mix": 1.9561625272035599, "l_a": 2.0729557871818542, "l_c": 2.005948320031166, "z_loss": 39.77258920669556, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18146535754203796, "tokens_per_second": 1412854.1353411283, "eta_seconds": 10130.23054113388, "data_wait_seconds": 3.0249357223510742e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 45190, "total_steps": 72479, "tokens": 23692574720, "loss": 4.553430378437042, "l_mix": 2.8226475715637207, "l_a": 2.910480946302414, "l_c": 2.8587945699691772, "z_loss": 36.02277851104736, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12289287149906158, "tokens_per_second": 1414451.8500995347, "eta_seconds": 10115.081139731408, "data_wait_seconds": 1.5437602996826172e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 45200, "total_steps": 72479, "tokens": 23697817600, "loss": 4.485737204551697, "l_mix": 2.780644476413727, "l_a": 2.864829897880554, "l_c": 2.818812131881714, "z_loss": 36.326698303222656, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1562567502260208, "tokens_per_second": 1364548.1575549343, "eta_seconds": 10481.163506627083, "data_wait_seconds": 2.1129846572875977e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.1929618, "gate_a2b/L1_absmax": 0.7276656, "gate_a2b/L1_rms": 0.20505552, "gate_a2b/L3_absmean": 0.18977895, "gate_a2b/L3_absmax": 0.57372779, "gate_a2b/L3_rms": 0.19607264, "gate_a2b/L5_absmean": 0.29209882, "gate_a2b/L5_absmax": 0.73796749, "gate_a2b/L5_rms": 0.2981123, "gate_c2b/L1_absmean": 0.01873166, "gate_c2b/L1_absmax": 0.63416719, "gate_c2b/L1_rms": 0.06682824, "gate_c2b/L3_absmean": 0.19191325, "gate_c2b/L3_absmax": 0.43042004, "gate_c2b/L3_rms": 0.19595908, "gate_c2b/L5_absmean": 0.54587823, "gate_c2b/L5_absmax": 0.76073211, "gate_c2b/L5_rms": 0.55748957, "gate_b2a/L7_absmean": 0.05108014, "gate_b2a/L7_absmax": 0.32253367, "gate_b2a/L7_rms": 0.05668087, "gate_b2a/L11_absmean": 0.0828331, "gate_b2a/L11_absmax": 0.59658021, "gate_b2a/L11_rms": 0.09168849, "gate_b2a/L14_absmean": 0.21305561, "gate_b2a/L14_absmax": 0.79370457, "gate_b2a/L14_rms": 0.22163279, "gate_b2c/L3_absmean": 0.03747758, "gate_b2c/L3_absmax": 0.58109307, "gate_b2c/L3_rms": 0.04885988, "gate_b2c/L5_absmean": 0.04003394, "gate_b2c/L5_absmax": 0.58607769, "gate_b2c/L5_rms": 0.05045808, "gate_b2c/L6_absmean": 0.05739198, "gate_b2c/L6_absmax": 0.38183725, "gate_b2c/L6_rms": 0.06428041}, "streams": {"cos_a2b/round0": 0.367938, "cos_c2b/round0": 0.310181, "cos_b2a/round0": 0.14897, "cos_b2c/round0": 0.130181, "rms_a/round0": 5.514785, "rms_b/round0": 10.099638, "rms_c/round0": 3.933088, "cos_a2b/round1": 0.414297, "cos_c2b/round1": 0.299634, "cos_b2a/round1": 0.25333, "cos_b2c/round1": 0.113629, "rms_a/round1": 10.486408, "rms_b/round1": 22.958904, "rms_c/round1": 8.011075, "cos_a2b/round2": 0.545834, "cos_c2b/round2": 0.528466, "cos_b2a/round2": 0.098879, "cos_b2c/round2": 0.181717, "rms_a/round2": 28.758078, "rms_b/round2": 61.359245, "rms_c/round2": 15.046111}, "alpha_mean": 0.42297813749821955, "alpha_hist": [0.00812, 0.07069, 0.18748, 0.22278, 0.1868, 0.14835, 0.11116, 0.05739, 0.00719, 5e-05], "alpha_by_type": {"special": {"share": 0.0009, "mean": 0.53764, "std": 0.09187}, "whitespace": {"share": 0.0295, "mean": 0.5323, "std": 0.12898}, "digit": {"share": 0.04526, "mean": 0.42756, "std": 0.12483}, "word_start": {"share": 0.5368, "mean": 0.43888, "std": 0.16541}, "word_cont": {"share": 0.27267, "mean": 0.31969, "std": 0.11801}, "punct": {"share": 0.11487, "mean": 0.56308, "std": 0.15346}}, "alpha_p05": 0.17273, "alpha_p50": 0.41253, "alpha_p95": 0.73048}
{"step": 45210, "total_steps": 72479, "tokens": 23703060480, "loss": 4.482225775718689, "l_mix": 2.7789026498794556, "l_a": 2.863404333591461, "l_c": 2.8143390715122223, "z_loss": 36.59736490249634, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11386708170175552, "tokens_per_second": 1326925.732982644, "eta_seconds": 10774.385571575165, "data_wait_seconds": 2.6404857635498047e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 45220, "total_steps": 72479, "tokens": 23708303360, "loss": 4.362607359886169, "l_mix": 2.704064816236496, "l_a": 2.7887068390846252, "l_c": 2.739767849445343, "z_loss": 37.44051790237427, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12184648960828781, "tokens_per_second": 1377790.6971678776, "eta_seconds": 10372.81397049427, "data_wait_seconds": 1.6063451766967773e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 45230, "total_steps": 72479, "tokens": 23713546240, "loss": 4.848810195922852, "l_mix": 3.0087666511535645, "l_a": 3.0844016075134277, "l_c": 3.0490765273571014, "z_loss": 35.48862075805664, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17237922549247742, "tokens_per_second": 1454680.3435284905, "eta_seconds": 9820.936795878411, "data_wait_seconds": 1.9341707229614258e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 45240, "total_steps": 72479, "tokens": 23718789120, "loss": 4.8586485385894775, "l_mix": 3.015346050262451, "l_a": 3.0922577381134033, "l_c": 3.05208358168602, "z_loss": 35.81963109970093, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1296181082725525, "tokens_per_second": 1434670.0669218556, "eta_seconds": 9954.261374282838, "data_wait_seconds": 1.868605613708496e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 45250, "total_steps": 72479, "tokens": 23724032000, "loss": 4.712965786457062, "l_mix": 2.92392098903656, "l_a": 3.0004935264587402, "l_c": 2.9629887342453003, "z_loss": 36.001272201538086, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1261586844921112, "tokens_per_second": 1382006.4633427134, "eta_seconds": 10329.791018104554, "data_wait_seconds": 2.3871660232543945e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 45260, "total_steps": 72479, "tokens": 23729274880, "loss": 4.805607318878174, "l_mix": 2.9814696609973907, "l_a": 3.0597844421863556, "l_c": 3.0206741988658905, "z_loss": 35.80758571624756, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12480399012565613, "tokens_per_second": 1394756.3775509137, "eta_seconds": 10231.60410068035, "data_wait_seconds": 3.123283386230469e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 45270, "total_steps": 72479, "tokens": 23734517760, "loss": 4.668915152549744, "l_mix": 2.8967577815055847, "l_a": 2.972663313150406, "l_c": 2.9345276951789856, "z_loss": 36.052894115448, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11931589990854263, "tokens_per_second": 1424382.8811793532, "eta_seconds": 10015.110670375823, "data_wait_seconds": 2.3186206817626953e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 45280, "total_steps": 72479, "tokens": 23739760640, "loss": 3.160472184419632, "l_mix": 1.944844588637352, "l_a": 2.0577478259801865, "l_c": 1.9943439811468124, "z_loss": 38.75356197357178, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15712139010429382, "tokens_per_second": 1458320.4328838163, "eta_seconds": 9778.447171449661, "data_wait_seconds": 2.0325183868408203e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 45290, "total_steps": 72479, "tokens": 23745003520, "loss": 3.166080802679062, "l_mix": 1.9481999278068542, "l_a": 2.062414228916168, "l_c": 1.9971886724233627, "z_loss": 39.78106498718262, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1426851898431778, "tokens_per_second": 1439169.0523103294, "eta_seconds": 9904.928409290314, "data_wait_seconds": 3.653764724731445e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 45300, "total_steps": 72479, "tokens": 23750246400, "loss": 4.360675513744354, "l_mix": 2.7017712891101837, "l_a": 2.7889645099639893, "l_c": 2.74071604013443, "z_loss": 37.20149612426758, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13024276494979858, "tokens_per_second": 1403168.7896907043, "eta_seconds": 10155.316777777673, "data_wait_seconds": 1.6570091247558594e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.19332151, "gate_a2b/L1_absmax": 0.7279799, "gate_a2b/L1_rms": 0.20533216, "gate_a2b/L3_absmean": 0.18995917, "gate_a2b/L3_absmax": 0.56903243, "gate_a2b/L3_rms": 0.19616225, "gate_a2b/L5_absmean": 0.29144457, "gate_a2b/L5_absmax": 0.73614794, "gate_a2b/L5_rms": 0.29742724, "gate_c2b/L1_absmean": 0.01926206, "gate_c2b/L1_absmax": 0.63469642, "gate_c2b/L1_rms": 0.06659517, "gate_c2b/L3_absmean": 0.19202502, "gate_c2b/L3_absmax": 0.42865035, "gate_c2b/L3_rms": 0.19604477, "gate_c2b/L5_absmean": 0.54534531, "gate_c2b/L5_absmax": 0.76737881, "gate_c2b/L5_rms": 0.55703336, "gate_b2a/L7_absmean": 0.0512776, "gate_b2a/L7_absmax": 0.32296816, "gate_b2a/L7_rms": 0.0567278, "gate_b2a/L11_absmean": 0.08360961, "gate_b2a/L11_absmax": 0.59807575, "gate_b2a/L11_rms": 0.09228648, "gate_b2a/L14_absmean": 0.21334471, "gate_b2a/L14_absmax": 0.79112619, "gate_b2a/L14_rms": 0.2218529, "gate_b2c/L3_absmean": 0.0372881, "gate_b2c/L3_absmax": 0.58153021, "gate_b2c/L3_rms": 0.04867449, "gate_b2c/L5_absmean": 0.0399182, "gate_b2c/L5_absmax": 0.59018409, "gate_b2c/L5_rms": 0.05051069, "gate_b2c/L6_absmean": 0.05728149, "gate_b2c/L6_absmax": 0.37715682, "gate_b2c/L6_rms": 0.06407771}, "streams": {"cos_a2b/round0": 0.364597, "cos_c2b/round0": 0.301319, "cos_b2a/round0": 0.1463, "cos_b2c/round0": 0.125661, "rms_a/round0": 5.542951, "rms_b/round0": 10.146956, "rms_c/round0": 4.010344, "cos_a2b/round1": 0.409463, "cos_c2b/round1": 0.299469, "cos_b2a/round1": 0.249445, "cos_b2c/round1": 0.109754, "rms_a/round1": 10.520054, "rms_b/round1": 23.127918, "rms_c/round1": 8.07025, "cos_a2b/round2": 0.535298, "cos_c2b/round2": 0.532457, "cos_b2a/round2": 0.100142, "cos_b2c/round2": 0.179485, "rms_a/round2": 28.453941, "rms_b/round2": 60.904724, "rms_c/round2": 15.152879}, "alpha_mean": 0.41830348577809673, "alpha_hist": [0.00772, 0.07721, 0.18375, 0.22658, 0.20151, 0.14206, 0.09642, 0.05602, 0.00873, 2e-05], "alpha_by_type": {"special": {"share": 0.00107, "mean": 0.52254, "std": 0.10941}, "whitespace": {"share": 0.02847, "mean": 0.5074, "std": 0.10079}, "digit": {"share": 0.03503, "mean": 0.42001, "std": 0.132}, "word_start": {"share": 0.52754, "mean": 0.43793, "std": 0.16821}, "word_cont": {"share": 0.25227, "mean": 0.31965, "std": 0.12211}, "punct": {"share": 0.15561, "mean": 0.49429, "std": 0.16394}}, "alpha_p05": 0.1747, "alpha_p50": 0.39467, "alpha_p95": 0.73791}
{"step": 45310, "total_steps": 72479, "tokens": 23755489280, "loss": 4.5036832094192505, "l_mix": 2.791500210762024, "l_a": 2.879182457923889, "l_c": 2.828094005584717, "z_loss": 36.358646392822266, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.10550297796726227, "tokens_per_second": 1155141.9956293125, "eta_seconds": 12331.281111669541, "data_wait_seconds": 2.0712614059448242e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 45320, "total_steps": 72479, "tokens": 23760732160, "loss": 4.470371425151825, "l_mix": 2.770170211791992, "l_a": 2.8599407076835632, "l_c": 2.8073964416980743, "z_loss": 36.34620904922485, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1309901773929596, "tokens_per_second": 1313981.219141785, "eta_seconds": 10836.637224769593, "data_wait_seconds": 2.250075340270996e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 45330, "total_steps": 72479, "tokens": 23765975040, "loss": 4.577250182628632, "l_mix": 2.8382839262485504, "l_a": 2.919561743736267, "l_c": 2.876992106437683, "z_loss": 37.061838150024414, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14960747957229614, "tokens_per_second": 1360034.9607248446, "eta_seconds": 10465.83016102314, "data_wait_seconds": 2.703070640563965e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 45340, "total_steps": 72479, "tokens": 23771217920, "loss": 3.133548140525818, "l_mix": 1.9273441433906555, "l_a": 2.0430734157562256, "l_c": 1.9776063561439514, "z_loss": 40.10246992111206, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16556914150714874, "tokens_per_second": 1367844.3141863486, "eta_seconds": 10402.245258784294, "data_wait_seconds": 2.9921531677246094e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 45350, "total_steps": 72479, "tokens": 23776460800, "loss": 3.1852684915065765, "l_mix": 1.9609966576099396, "l_a": 2.0724839568138123, "l_c": 2.0084221065044403, "z_loss": 38.18856334686279, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20684607326984406, "tokens_per_second": 1443192.7572257153, "eta_seconds": 9855.515890574456, "data_wait_seconds": 1.621246337890625e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 45360, "total_steps": 72479, "tokens": 23781703680, "loss": 3.163808584213257, "l_mix": 1.9470183849334717, "l_a": 2.061539053916931, "l_c": 1.9944282323122025, "z_loss": 40.03998851776123, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.22234132885932922, "tokens_per_second": 1464017.3152683007, "eta_seconds": 9711.747343230249, "data_wait_seconds": 1.621246337890625e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 45370, "total_steps": 72479, "tokens": 23786946560, "loss": 4.839659929275513, "l_mix": 3.004314363002777, "l_a": 3.0721619725227356, "l_c": 3.045656442642212, "z_loss": 34.981669425964355, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.21289734542369843, "tokens_per_second": 1411047.3091451968, "eta_seconds": 10072.605857992172, "data_wait_seconds": 2.5987625122070312e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 45380, "total_steps": 72479, "tokens": 23792189440, "loss": 4.694537162780762, "l_mix": 2.912746489048004, "l_a": 2.989142060279846, "l_c": 2.950160026550293, "z_loss": 36.20781993865967, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15721404552459717, "tokens_per_second": 1444118.9063715835, "eta_seconds": 9838.303791546823, "data_wait_seconds": 2.4080276489257812e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 45390, "total_steps": 72479, "tokens": 23797432320, "loss": 3.143060564994812, "l_mix": 1.9335104823112488, "l_a": 2.0466638952493668, "l_c": 1.9851697534322739, "z_loss": 38.60868167877197, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2081453949213028, "tokens_per_second": 1397949.616465549, "eta_seconds": 10159.477469515801, "data_wait_seconds": 1.71661376953125e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 45400, "total_steps": 72479, "tokens": 23802675200, "loss": 3.1720851957798004, "l_mix": 1.9518983662128448, "l_a": 2.0657489001750946, "l_c": 2.0015404522418976, "z_loss": 37.90909814834595, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1595539152622223, "tokens_per_second": 1357386.068460175, "eta_seconds": 10459.21649107933, "data_wait_seconds": 1.7583370208740234e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.19297329, "gate_a2b/L1_absmax": 0.72487134, "gate_a2b/L1_rms": 0.20498499, "gate_a2b/L3_absmean": 0.18988383, "gate_a2b/L3_absmax": 0.56964529, "gate_a2b/L3_rms": 0.19605064, "gate_a2b/L5_absmean": 0.29106545, "gate_a2b/L5_absmax": 0.73645258, "gate_a2b/L5_rms": 0.29711914, "gate_c2b/L1_absmean": 0.01938485, "gate_c2b/L1_absmax": 0.63310766, "gate_c2b/L1_rms": 0.06654959, "gate_c2b/L3_absmean": 0.19140808, "gate_c2b/L3_absmax": 0.43113452, "gate_c2b/L3_rms": 0.19547661, "gate_c2b/L5_absmean": 0.54454046, "gate_c2b/L5_absmax": 0.7646479, "gate_c2b/L5_rms": 0.55615765, "gate_b2a/L7_absmean": 0.05149144, "gate_b2a/L7_absmax": 0.32246447, "gate_b2a/L7_rms": 0.05696883, "gate_b2a/L11_absmean": 0.08295658, "gate_b2a/L11_absmax": 0.59921867, "gate_b2a/L11_rms": 0.09179448, "gate_b2a/L14_absmean": 0.2133586, "gate_b2a/L14_absmax": 0.78998834, "gate_b2a/L14_rms": 0.22184449, "gate_b2c/L3_absmean": 0.03721961, "gate_b2c/L3_absmax": 0.58111817, "gate_b2c/L3_rms": 0.04859617, "gate_b2c/L5_absmean": 0.04019542, "gate_b2c/L5_absmax": 0.58533174, "gate_b2c/L5_rms": 0.05057832, "gate_b2c/L6_absmean": 0.05767905, "gate_b2c/L6_absmax": 0.37923917, "gate_b2c/L6_rms": 0.06448122}, "streams": {"cos_a2b/round0": 0.364899, "cos_c2b/round0": 0.268211, "cos_b2a/round0": 0.136485, "cos_b2c/round0": 0.108166, "rms_a/round0": 5.789708, "rms_b/round0": 10.350334, "rms_c/round0": 4.059966, "cos_a2b/round1": 0.414771, "cos_c2b/round1": 0.299584, "cos_b2a/round1": 0.243212, "cos_b2c/round1": 0.123854, "rms_a/round1": 11.243586, "rms_b/round1": 24.533352, "rms_c/round1": 8.58168, "cos_a2b/round2": 0.516911, "cos_c2b/round2": 0.51185, "cos_b2a/round2": 0.085927, "cos_b2c/round2": 0.169273, "rms_a/round2": 29.454641, "rms_b/round2": 62.56419, "rms_c/round2": 16.309921}, "alpha_mean": 0.40466413043699845, "alpha_hist": [0.0369, 0.16438, 0.19635, 0.15208, 0.11815, 0.10722, 0.09778, 0.0836, 0.0417, 0.00183], "alpha_by_type": {"special": {"share": 0.00105, "mean": 0.64609, "std": 0.10096}, "whitespace": {"share": 0.03456, "mean": 0.62623, "std": 0.1659}, "digit": {"share": 0.01196, "mean": 0.39632, "std": 0.17708}, "word_start": {"share": 0.55682, "mean": 0.41426, "std": 0.2081}, "word_cont": {"share": 0.27925, "mean": 0.2727, "std": 0.14275}, "punct": {"share": 0.11635, "mean": 0.60833, "std": 0.18909}}, "alpha_p05": 0.12003, "alpha_p50": 0.34953, "alpha_p95": 0.78826}
{"step": 45410, "total_steps": 72479, "tokens": 23807918080, "loss": 4.484848380088806, "l_mix": 2.7795490026474, "l_a": 2.867693156003952, "l_c": 2.8166377544403076, "z_loss": 36.95386457443237, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12777693569660187, "tokens_per_second": 1407634.5189250268, "eta_seconds": 10082.128337430953, "data_wait_seconds": 6.29723072052002e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 45420, "total_steps": 72479, "tokens": 23813160960, "loss": 4.340657114982605, "l_mix": 2.6902197003364563, "l_a": 2.774834245443344, "l_c": 2.726623445749283, "z_loss": 37.79794931411743, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11102042347192764, "tokens_per_second": 1204514.2545288627, "eta_seconds": 11777.950272202492, "data_wait_seconds": 1.773238182067871e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 45430, "total_steps": 72479, "tokens": 23818403840, "loss": 4.420206785202026, "l_mix": 2.7392346560955048, "l_a": 2.8277063965797424, "l_c": 2.7755335867404938, "z_loss": 36.20947742462158, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12533573806285858, "tokens_per_second": 1417117.7608873963, "eta_seconds": 10007.260161018372, "data_wait_seconds": 1.7374753952026367e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 45440, "total_steps": 72479, "tokens": 23823646720, "loss": 4.562473952770233, "l_mix": 2.8295852541923523, "l_a": 2.9087382555007935, "l_c": 2.86755707859993, "z_loss": 35.84336757659912, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20061910152435303, "tokens_per_second": 1409169.692260203, "eta_seconds": 10059.983059430122, "data_wait_seconds": 1.9758939743041992e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 45450, "total_steps": 72479, "tokens": 23828889600, "loss": 3.1498953700065613, "l_mix": 1.937746301293373, "l_a": 2.052531510591507, "l_c": 1.9879652261734009, "z_loss": 39.34906721115112, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18202705681324005, "tokens_per_second": 1400421.1762279905, "eta_seconds": 10119.084595799446, "data_wait_seconds": 3.975629806518555e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 45460, "total_steps": 72479, "tokens": 23834132480, "loss": 3.1717550456523895, "l_mix": 1.9511372596025467, "l_a": 2.068583697080612, "l_c": 2.0001420974731445, "z_loss": 38.94881010055542, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17856919765472412, "tokens_per_second": 1403965.8298837894, "eta_seconds": 10089.802166461945, "data_wait_seconds": 3.2842159271240234e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 45470, "total_steps": 72479, "tokens": 23839375360, "loss": 3.1397860944271088, "l_mix": 1.9321777522563934, "l_a": 2.0437809973955154, "l_c": 1.9815799742937088, "z_loss": 37.83495759963989, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2510753870010376, "tokens_per_second": 1409617.7319650806, "eta_seconds": 10045.6274569273, "data_wait_seconds": 5.245208740234375e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 45480, "total_steps": 72479, "tokens": 23844618240, "loss": 3.176989436149597, "l_mix": 1.9547046422958374, "l_a": 2.0699739158153534, "l_c": 2.004308521747589, "z_loss": 38.793296813964844, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17649832367897034, "tokens_per_second": 1471300.6116718736, "eta_seconds": 9620.91064171791, "data_wait_seconds": 4.139542579650879e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 45490, "total_steps": 72479, "tokens": 23849861120, "loss": 3.1514503061771393, "l_mix": 1.9390307366847992, "l_a": 2.054108202457428, "l_c": 1.9872902482748032, "z_loss": 39.8924503326416, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19181133806705475, "tokens_per_second": 1472904.1783498637, "eta_seconds": 9606.876699781418, "data_wait_seconds": 2.6553869247436523e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 45500, "total_steps": 72479, "tokens": 23855104000, "loss": 3.167314052581787, "l_mix": 1.9487834870815277, "l_a": 2.06235933303833, "l_c": 1.9994092136621475, "z_loss": 39.39749574661255, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1713724583387375, "tokens_per_second": 1414434.655065552, "eta_seconds": 10000.296515178681, "data_wait_seconds": 3.1441450119018555e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.19212468, "gate_a2b/L1_absmax": 0.7251879, "gate_a2b/L1_rms": 0.20420884, "gate_a2b/L3_absmean": 0.18927041, "gate_a2b/L3_absmax": 0.56551731, "gate_a2b/L3_rms": 0.19545875, "gate_a2b/L5_absmean": 0.29062298, "gate_a2b/L5_absmax": 0.73460126, "gate_a2b/L5_rms": 0.29659486, "gate_c2b/L1_absmean": 0.01976618, "gate_c2b/L1_absmax": 0.63199246, "gate_c2b/L1_rms": 0.06678692, "gate_c2b/L3_absmean": 0.19169937, "gate_c2b/L3_absmax": 0.43279189, "gate_c2b/L3_rms": 0.1958051, "gate_c2b/L5_absmean": 0.54398292, "gate_c2b/L5_absmax": 0.76576275, "gate_c2b/L5_rms": 0.55560088, "gate_b2a/L7_absmean": 0.05145576, "gate_b2a/L7_absmax": 0.32077813, "gate_b2a/L7_rms": 0.05697361, "gate_b2a/L11_absmean": 0.08294207, "gate_b2a/L11_absmax": 0.5976454, "gate_b2a/L11_rms": 0.09176318, "gate_b2a/L14_absmean": 0.2130027, "gate_b2a/L14_absmax": 0.78966498, "gate_b2a/L14_rms": 0.22150603, "gate_b2c/L3_absmean": 0.03702034, "gate_b2c/L3_absmax": 0.57891554, "gate_b2c/L3_rms": 0.0484263, "gate_b2c/L5_absmean": 0.0397842, "gate_b2c/L5_absmax": 0.58520859, "gate_b2c/L5_rms": 0.05021479, "gate_b2c/L6_absmean": 0.05748378, "gate_b2c/L6_absmax": 0.37966678, "gate_b2c/L6_rms": 0.06428955}, "streams": {"cos_a2b/round0": 0.361532, "cos_c2b/round0": 0.271653, "cos_b2a/round0": 0.136952, "cos_b2c/round0": 0.107532, "rms_a/round0": 5.763312, "rms_b/round0": 10.372541, "rms_c/round0": 4.074874, "cos_a2b/round1": 0.414728, "cos_c2b/round1": 0.300159, "cos_b2a/round1": 0.240259, "cos_b2c/round1": 0.118704, "rms_a/round1": 11.203849, "rms_b/round1": 24.418568, "rms_c/round1": 8.581767, "cos_a2b/round2": 0.51866, "cos_c2b/round2": 0.511431, "cos_b2a/round2": 0.085593, "cos_b2c/round2": 0.168523, "rms_a/round2": 29.432217, "rms_b/round2": 62.470097, "rms_c/round2": 16.26692}, "alpha_mean": 0.4029321896864957, "alpha_hist": [0.03909, 0.17711, 0.19504, 0.14569, 0.11099, 0.10092, 0.09517, 0.08746, 0.04495, 0.00357], "alpha_by_type": {"special": {"share": 0.00111, "mean": 0.6475, "std": 0.11338}, "whitespace": {"share": 0.03154, "mean": 0.62548, "std": 0.17164}, "digit": {"share": 0.00874, "mean": 0.38421, "std": 0.19112}, "word_start": {"share": 0.56509, "mean": 0.41149, "std": 0.21333}, "word_cont": {"share": 0.2771, "mean": 0.265, "std": 0.13776}, "punct": {"share": 0.11641, "mean": 0.62851, "std": 0.1869}}, "alpha_p05": 0.11086, "alpha_p50": 0.34643, "alpha_p95": 0.79343}
{"step": 45510, "total_steps": 72479, "tokens": 23860346880, "loss": 3.1215491592884064, "l_mix": 1.9203154742717743, "l_a": 2.0345503389835358, "l_c": 1.9695619344711304, "z_loss": 38.99892234802246, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17301881313323975, "tokens_per_second": 1409958.1072804555, "eta_seconds": 10028.328500676154, "data_wait_seconds": 2.2351741790771484e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 45520, "total_steps": 72479, "tokens": 23865589760, "loss": 4.335983991622925, "l_mix": 2.6871416866779327, "l_a": 2.773190498352051, "l_c": 2.722950130701065, "z_loss": 37.75433540344238, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11793521046638489, "tokens_per_second": 1434076.2358029422, "eta_seconds": 9856.017301678658, "data_wait_seconds": 1.6868114471435547e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 45530, "total_steps": 72479, "tokens": 23870832640, "loss": 4.468591690063477, "l_mix": 2.7698415517807007, "l_a": 2.855362743139267, "l_c": 2.807137668132782, "z_loss": 37.37010860443115, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13568340241909027, "tokens_per_second": 1377629.0293825932, "eta_seconds": 10256.053705787659, "data_wait_seconds": 1.749396324157715e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 45540, "total_steps": 72479, "tokens": 23876075520, "loss": 4.474973738193512, "l_mix": 2.7730799317359924, "l_a": 2.860541433095932, "l_c": 2.8124379217624664, "z_loss": 37.74718904495239, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1270398497581482, "tokens_per_second": 1451878.1787077417, "eta_seconds": 9727.94731619358, "data_wait_seconds": 1.7821788787841797e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 45550, "total_steps": 72479, "tokens": 23881318400, "loss": 4.550539493560791, "l_mix": 2.821610599756241, "l_a": 2.905612140893936, "l_c": 2.857484072446823, "z_loss": 36.06183910369873, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16798682510852814, "tokens_per_second": 1407544.869932132, "eta_seconds": 10030.62272017002, "data_wait_seconds": 3.230571746826172e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 45560, "total_steps": 72479, "tokens": 23886561280, "loss": 3.1546859741210938, "l_mix": 1.939962312579155, "l_a": 2.0552026629447937, "l_c": 1.9938760846853256, "z_loss": 38.57423543930054, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2458869218826294, "tokens_per_second": 1296304.1430727364, "eta_seconds": 10887.343643403054, "data_wait_seconds": 2.1159648895263672e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 45570, "total_steps": 72479, "tokens": 23891804160, "loss": 3.1671327650547028, "l_mix": 1.9493293464183807, "l_a": 2.063741907477379, "l_c": 1.9956025630235672, "z_loss": 39.49476623535156, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.21761584281921387, "tokens_per_second": 1472905.6581765728, "eta_seconds": 9578.3906550169, "data_wait_seconds": 1.8090009689331055e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 45580, "total_steps": 72479, "tokens": 23897047040, "loss": 3.117296040058136, "l_mix": 1.9176747649908066, "l_a": 2.0319330245256424, "l_c": 1.9668042957782745, "z_loss": 40.198843479156494, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18122290074825287, "tokens_per_second": 1432454.0600438786, "eta_seconds": 9845.21829032898, "data_wait_seconds": 2.059340476989746e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 45590, "total_steps": 72479, "tokens": 23902289920, "loss": 3.1198021471500397, "l_mix": 1.9193839132785797, "l_a": 2.031908243894577, "l_c": 1.9694856107234955, "z_loss": 39.315269470214844, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14059405028820038, "tokens_per_second": 1431444.9626962487, "eta_seconds": 9848.496029806138, "data_wait_seconds": 3.4868717193603516e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 45600, "total_steps": 72479, "tokens": 23907532800, "loss": 3.1620360910892487, "l_mix": 1.9472519308328629, "l_a": 2.0558965504169464, "l_c": 1.9933837205171585, "z_loss": 38.77952575683594, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.22141548991203308, "tokens_per_second": 1285019.6318723573, "eta_seconds": 10966.631794929504, "data_wait_seconds": 1.704692840576172e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.19160703, "gate_a2b/L1_absmax": 0.72151804, "gate_a2b/L1_rms": 0.20371436, "gate_a2b/L3_absmean": 0.18811081, "gate_a2b/L3_absmax": 0.57193297, "gate_a2b/L3_rms": 0.19436127, "gate_a2b/L5_absmean": 0.28972957, "gate_a2b/L5_absmax": 0.73319405, "gate_a2b/L5_rms": 0.29572546, "gate_c2b/L1_absmean": 0.01943254, "gate_c2b/L1_absmax": 0.63060296, "gate_c2b/L1_rms": 0.06644351, "gate_c2b/L3_absmean": 0.19162479, "gate_c2b/L3_absmax": 0.43397674, "gate_c2b/L3_rms": 0.19561386, "gate_c2b/L5_absmean": 0.54372853, "gate_c2b/L5_absmax": 0.76612931, "gate_c2b/L5_rms": 0.55528939, "gate_b2a/L7_absmean": 0.05145402, "gate_b2a/L7_absmax": 0.32187229, "gate_b2a/L7_rms": 0.05689772, "gate_b2a/L11_absmean": 0.0828882, "gate_b2a/L11_absmax": 0.59819263, "gate_b2a/L11_rms": 0.09170869, "gate_b2a/L14_absmean": 0.21322337, "gate_b2a/L14_absmax": 0.78814179, "gate_b2a/L14_rms": 0.22168851, "gate_b2c/L3_absmean": 0.03698071, "gate_b2c/L3_absmax": 0.57819742, "gate_b2c/L3_rms": 0.04836186, "gate_b2c/L5_absmean": 0.04009338, "gate_b2c/L5_absmax": 0.58725935, "gate_b2c/L5_rms": 0.05049623, "gate_b2c/L6_absmean": 0.05748973, "gate_b2c/L6_absmax": 0.37871689, "gate_b2c/L6_rms": 0.06426045}, "streams": {"cos_a2b/round0": 0.363606, "cos_c2b/round0": 0.279071, "cos_b2a/round0": 0.138862, "cos_b2c/round0": 0.113266, "rms_a/round0": 5.818585, "rms_b/round0": 10.372199, "rms_c/round0": 4.035574, "cos_a2b/round1": 0.412566, "cos_c2b/round1": 0.296028, "cos_b2a/round1": 0.240895, "cos_b2c/round1": 0.117864, "rms_a/round1": 11.257721, "rms_b/round1": 24.314821, "rms_c/round1": 8.542355, "cos_a2b/round2": 0.525121, "cos_c2b/round2": 0.510807, "cos_b2a/round2": 0.087605, "cos_b2c/round2": 0.171203, "rms_a/round2": 29.749397, "rms_b/round2": 62.643444, "rms_c/round2": 16.314932}, "alpha_mean": 0.4158890152086059, "alpha_hist": [0.03252, 0.14751, 0.18513, 0.15392, 0.12733, 0.1189, 0.10765, 0.08749, 0.03807, 0.00148], "alpha_by_type": {"special": {"share": 0.00108, "mean": 0.59348, "std": 0.11669}, "whitespace": {"share": 0.03717, "mean": 0.60827, "std": 0.16376}, "digit": {"share": 0.00858, "mean": 0.38155, "std": 0.18876}, "word_start": {"share": 0.5547, "mean": 0.42265, "std": 0.20177}, "word_cont": {"share": 0.2616, "mean": 0.27445, "std": 0.13921}, "punct": {"share": 0.13687, "mean": 0.6073, "std": 0.18266}}, "alpha_p05": 0.11437, "alpha_p50": 0.38237, "alpha_p95": 0.78629}
{"step": 45610, "total_steps": 72479, "tokens": 23912775680, "loss": 3.16359081864357, "l_mix": 1.9465859979391098, "l_a": 2.0603688657283783, "l_c": 1.9963139742612839, "z_loss": 39.90545701980591, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16334302723407745, "tokens_per_second": 1422330.058290984, "eta_seconds": 9904.237198591232, "data_wait_seconds": 1.84476375579834e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 45620, "total_steps": 72479, "tokens": 23918018560, "loss": 4.402685344219208, "l_mix": 2.7285419404506683, "l_a": 2.811148077249527, "l_c": 2.7693297266960144, "z_loss": 37.47889232635498, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.21596123278141022, "tokens_per_second": 1391389.0037975216, "eta_seconds": 10120.71487812996, "data_wait_seconds": 1.9162893295288086e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 45630, "total_steps": 72479, "tokens": 23923261440, "loss": 4.431761264801025, "l_mix": 2.746725708246231, "l_a": 2.8333419859409332, "l_c": 2.783443033695221, "z_loss": 37.46698236465454, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17731431126594543, "tokens_per_second": 1458682.7057395806, "eta_seconds": 9650.219651341438, "data_wait_seconds": 1.6003847122192383e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 45640, "total_steps": 72479, "tokens": 23928504320, "loss": 4.449902772903442, "l_mix": 2.7566616237163544, "l_a": 2.8482264280319214, "l_c": 2.795910567045212, "z_loss": 38.50322484970093, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1571381390094757, "tokens_per_second": 1450919.2690743227, "eta_seconds": 9698.241612696647, "data_wait_seconds": 2.467632293701172e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 45650, "total_steps": 72479, "tokens": 23933747200, "loss": 4.566450595855713, "l_mix": 2.8314049243927, "l_a": 2.9121542870998383, "l_c": 2.8713311851024628, "z_loss": 37.91908359527588, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14130736887454987, "tokens_per_second": 1443297.7091005489, "eta_seconds": 9745.822128939628, "data_wait_seconds": 2.0235776901245117e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 45660, "total_steps": 72479, "tokens": 23938990080, "loss": 4.419070661067963, "l_mix": 2.7391643822193146, "l_a": 2.8227425515651703, "l_c": 2.7769449651241302, "z_loss": 37.43634223937988, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1182590201497078, "tokens_per_second": 1438500.442634792, "eta_seconds": 9774.678863668441, "data_wait_seconds": 1.58846378326416e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 45670, "total_steps": 72479, "tokens": 23944232960, "loss": 3.169555753469467, "l_mix": 1.9495374262332916, "l_a": 2.0620680451393127, "l_c": 2.0046596229076385, "z_loss": 38.15307283401489, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2874982953071594, "tokens_per_second": 1387775.8470688453, "eta_seconds": 10128.17525372505, "data_wait_seconds": 1.8715858459472656e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 45680, "total_steps": 72479, "tokens": 23949475840, "loss": 3.1719457507133484, "l_mix": 1.9521692097187042, "l_a": 2.0661046653985977, "l_c": 1.9998170286417007, "z_loss": 39.461684226989746, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17068590223789215, "tokens_per_second": 1261141.256851899, "eta_seconds": 11141.015358638764, "data_wait_seconds": 3.87728214263916e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 45690, "total_steps": 72479, "tokens": 23954718720, "loss": 3.14924892783165, "l_mix": 1.9368528127670288, "l_a": 2.053385093808174, "l_c": 1.9879351407289505, "z_loss": 40.87746620178223, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15870562195777893, "tokens_per_second": 1131394.552545647, "eta_seconds": 12414.01702032089, "data_wait_seconds": 2.5838613510131836e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 45700, "total_steps": 72479, "tokens": 23959961600, "loss": 3.1913956701755524, "l_mix": 1.9637330174446106, "l_a": 2.0781372785568237, "l_c": 2.0140712559223175, "z_loss": 38.88377380371094, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14631594717502594, "tokens_per_second": 1443985.578388906, "eta_seconds": 9723.025328040123, "data_wait_seconds": 2.130866050720215e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.19148086, "gate_a2b/L1_absmax": 0.72332448, "gate_a2b/L1_rms": 0.20342816, "gate_a2b/L3_absmean": 0.18804747, "gate_a2b/L3_absmax": 0.56926888, "gate_a2b/L3_rms": 0.1942281, "gate_a2b/L5_absmean": 0.28882661, "gate_a2b/L5_absmax": 0.73413086, "gate_a2b/L5_rms": 0.29482272, "gate_c2b/L1_absmean": 0.01944244, "gate_c2b/L1_absmax": 0.63225228, "gate_c2b/L1_rms": 0.06663637, "gate_c2b/L3_absmean": 0.19118981, "gate_c2b/L3_absmax": 0.43395829, "gate_c2b/L3_rms": 0.19521679, "gate_c2b/L5_absmean": 0.54227918, "gate_c2b/L5_absmax": 0.76116353, "gate_c2b/L5_rms": 0.55380583, "gate_b2a/L7_absmean": 0.05089137, "gate_b2a/L7_absmax": 0.32216838, "gate_b2a/L7_rms": 0.05646151, "gate_b2a/L11_absmean": 0.08280759, "gate_b2a/L11_absmax": 0.59779745, "gate_b2a/L11_rms": 0.09168546, "gate_b2a/L14_absmean": 0.213195, "gate_b2a/L14_absmax": 0.78953993, "gate_b2a/L14_rms": 0.22169538, "gate_b2c/L3_absmean": 0.03695207, "gate_b2c/L3_absmax": 0.57749343, "gate_b2c/L3_rms": 0.04837227, "gate_b2c/L5_absmean": 0.03970016, "gate_b2c/L5_absmax": 0.5839141, "gate_b2c/L5_rms": 0.05013736, "gate_b2c/L6_absmean": 0.05737031, "gate_b2c/L6_absmax": 0.37786728, "gate_b2c/L6_rms": 0.06413201}, "streams": {"cos_a2b/round0": 0.364754, "cos_c2b/round0": 0.275259, "cos_b2a/round0": 0.140562, "cos_b2c/round0": 0.110248, "rms_a/round0": 5.78528, "rms_b/round0": 10.371366, "rms_c/round0": 4.066019, "cos_a2b/round1": 0.414201, "cos_c2b/round1": 0.303529, "cos_b2a/round1": 0.238268, "cos_b2c/round1": 0.121377, "rms_a/round1": 11.380056, "rms_b/round1": 24.608286, "rms_c/round1": 8.631485, "cos_a2b/round2": 0.524802, "cos_c2b/round2": 0.507621, "cos_b2a/round2": 0.088566, "cos_b2c/round2": 0.17065, "rms_a/round2": 29.976166, "rms_b/round2": 63.09449, "rms_c/round2": 16.356659}, "alpha_mean": 0.41660990606837345, "alpha_hist": [0.02902, 0.15157, 0.19304, 0.1557, 0.12354, 0.10759, 0.10104, 0.08797, 0.04695, 0.00359], "alpha_by_type": {"special": {"share": 0.00111, "mean": 0.63823, "std": 0.12167}, "whitespace": {"share": 0.03883, "mean": 0.62678, "std": 0.16353}, "digit": {"share": 0.00966, "mean": 0.40434, "std": 0.19748}, "word_start": {"share": 0.5773, "mean": 0.41468, "std": 0.20858}, "word_cont": {"share": 0.25389, "mean": 0.28854, "std": 0.14243}, "punct": {"share": 0.1192, "mean": 0.62921, "std": 0.18443}}, "alpha_p05": 0.13432, "alpha_p50": 0.38978, "alpha_p95": 0.80592}
{"step": 45710, "total_steps": 72479, "tokens": 23965204480, "loss": 3.1665942668914795, "l_mix": 1.9485368579626083, "l_a": 2.065297245979309, "l_c": 1.9948938637971878, "z_loss": 38.94028949737549, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16562946140766144, "tokens_per_second": 1392463.27285691, "eta_seconds": 10079.020212292671, "data_wait_seconds": 1.609325408935547e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 45720, "total_steps": 72479, "tokens": 23970447360, "loss": 3.14267361164093, "l_mix": 1.9332921504974365, "l_a": 2.046388268470764, "l_c": 1.984883114695549, "z_loss": 39.78726816177368, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1699022799730301, "tokens_per_second": 1438782.8925432905, "eta_seconds": 9750.896166968345, "data_wait_seconds": 2.250075340270996e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 45730, "total_steps": 72479, "tokens": 23975690240, "loss": 4.429417371749878, "l_mix": 2.744616150856018, "l_a": 2.8332475125789642, "l_c": 2.782756268978119, "z_loss": 38.48289203643799, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17185360193252563, "tokens_per_second": 1428695.5008145233, "eta_seconds": 9816.073266839981, "data_wait_seconds": 1.6003847122192383e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 45740, "total_steps": 72479, "tokens": 23980933120, "loss": 4.448260843753815, "l_mix": 2.756779432296753, "l_a": 2.8416593968868256, "l_c": 2.796611577272415, "z_loss": 37.0894775390625, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12845374643802643, "tokens_per_second": 1461646.8361941937, "eta_seconds": 9591.192950892448, "data_wait_seconds": 1.7404556274414062e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 45750, "total_steps": 72479, "tokens": 23986176000, "loss": 4.482680976390839, "l_mix": 2.778285324573517, "l_a": 2.863813877105713, "l_c": 2.8175048232078552, "z_loss": 36.13695812225342, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1942857801914215, "tokens_per_second": 1428048.919121269, "eta_seconds": 9813.175000071526, "data_wait_seconds": 1.9162893295288086e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 45760, "total_steps": 72479, "tokens": 23991418880, "loss": 4.451894164085388, "l_mix": 2.759414166212082, "l_a": 2.8451733589172363, "l_c": 2.7964263260364532, "z_loss": 36.67516231536865, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12670519948005676, "tokens_per_second": 1466719.2355905697, "eta_seconds": 9550.874313282966, "data_wait_seconds": 1.6450881958007812e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 45770, "total_steps": 72479, "tokens": 23996661760, "loss": 3.168782740831375, "l_mix": 1.9491791427135468, "l_a": 2.0663927495479584, "l_c": 1.998952478170395, "z_loss": 40.06226444244385, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14050590991973877, "tokens_per_second": 1335878.4954826036, "eta_seconds": 10482.39659471512, "data_wait_seconds": 3.4809112548828125e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 45780, "total_steps": 72479, "tokens": 24001904640, "loss": 3.1403503119945526, "l_mix": 1.9314885139465332, "l_a": 2.0476064234972, "l_c": 1.9819326549768448, "z_loss": 40.07597541809082, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14874152839183807, "tokens_per_second": 1396025.2208442201, "eta_seconds": 10027.014629101752, "data_wait_seconds": 2.658367156982422e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 45790, "total_steps": 72479, "tokens": 24007147520, "loss": 3.1827731132507324, "l_mix": 1.9588870108127594, "l_a": 2.073107659816742, "l_c": 2.0065127313137054, "z_loss": 39.28335905075073, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14765360951423645, "tokens_per_second": 1466460.6233825507, "eta_seconds": 9541.833042693139, "data_wait_seconds": 1.749396324157715e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 45800, "total_steps": 72479, "tokens": 24012390400, "loss": 3.1675506234169006, "l_mix": 1.949363797903061, "l_a": 2.0642981231212616, "l_c": 1.9963244050741196, "z_loss": 39.09064483642578, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17266425490379333, "tokens_per_second": 1458154.6889833368, "eta_seconds": 9592.589632415771, "data_wait_seconds": 2.6166439056396484e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.19122653, "gate_a2b/L1_absmax": 0.72219783, "gate_a2b/L1_rms": 0.20325424, "gate_a2b/L3_absmean": 0.18780248, "gate_a2b/L3_absmax": 0.5726701, "gate_a2b/L3_rms": 0.19403867, "gate_a2b/L5_absmean": 0.28812227, "gate_a2b/L5_absmax": 0.73509884, "gate_a2b/L5_rms": 0.29416293, "gate_c2b/L1_absmean": 0.02014371, "gate_c2b/L1_absmax": 0.63042229, "gate_c2b/L1_rms": 0.06665862, "gate_c2b/L3_absmean": 0.19075586, "gate_c2b/L3_absmax": 0.43403515, "gate_c2b/L3_rms": 0.19481702, "gate_c2b/L5_absmean": 0.54217178, "gate_c2b/L5_absmax": 0.76319921, "gate_c2b/L5_rms": 0.55371767, "gate_b2a/L7_absmean": 0.05135253, "gate_b2a/L7_absmax": 0.3212547, "gate_b2a/L7_rms": 0.05679892, "gate_b2a/L11_absmean": 0.08296291, "gate_b2a/L11_absmax": 0.59770787, "gate_b2a/L11_rms": 0.09178851, "gate_b2a/L14_absmean": 0.21303156, "gate_b2a/L14_absmax": 0.79023391, "gate_b2a/L14_rms": 0.22155194, "gate_b2c/L3_absmean": 0.03695351, "gate_b2c/L3_absmax": 0.57734811, "gate_b2c/L3_rms": 0.04836197, "gate_b2c/L5_absmean": 0.03979612, "gate_b2c/L5_absmax": 0.58338922, "gate_b2c/L5_rms": 0.05014615, "gate_b2c/L6_absmean": 0.05730546, "gate_b2c/L6_absmax": 0.3780055, "gate_b2c/L6_rms": 0.06405253}, "streams": {"cos_a2b/round0": 0.365811, "cos_c2b/round0": 0.277675, "cos_b2a/round0": 0.137841, "cos_b2c/round0": 0.111964, "rms_a/round0": 5.854787, "rms_b/round0": 10.398313, "rms_c/round0": 4.060246, "cos_a2b/round1": 0.409838, "cos_c2b/round1": 0.296863, "cos_b2a/round1": 0.237267, "cos_b2c/round1": 0.117757, "rms_a/round1": 11.350307, "rms_b/round1": 24.387199, "rms_c/round1": 8.598603, "cos_a2b/round2": 0.521191, "cos_c2b/round2": 0.513642, "cos_b2a/round2": 0.085571, "cos_b2c/round2": 0.169057, "rms_a/round2": 29.82338, "rms_b/round2": 62.718533, "rms_c/round2": 16.36013}, "alpha_mean": 0.4346913881712453, "alpha_hist": [0.02284, 0.12654, 0.1806, 0.16101, 0.13034, 0.11699, 0.11665, 0.09482, 0.04543, 0.00478], "alpha_by_type": {"special": {"share": 0.00104, "mean": 0.67986, "std": 0.10297}, "whitespace": {"share": 0.04756, "mean": 0.62381, "std": 0.15439}, "digit": {"share": 0.01416, "mean": 0.44356, "std": 0.14394}, "word_start": {"share": 0.54457, "mean": 0.43732, "std": 0.20629}, "word_cont": {"share": 0.2612, "mean": 0.29642, "std": 0.14306}, "punct": {"share": 0.13147, "mean": 0.62719, "std": 0.1747}}, "alpha_p05": 0.13095, "alpha_p50": 0.41111, "alpha_p95": 0.79629}
{"step": 45810, "total_steps": 72479, "tokens": 24017633280, "loss": 3.1770169734954834, "l_mix": 1.9553197175264359, "l_a": 2.068779021501541, "l_c": 2.0035448372364044, "z_loss": 39.18490552902222, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17382940649986267, "tokens_per_second": 1206557.6651386325, "eta_seconds": 11588.535779094696, "data_wait_seconds": 2.3752450942993164e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 45820, "total_steps": 72479, "tokens": 24022876160, "loss": 3.138120710849762, "l_mix": 1.9297594279050827, "l_a": 2.047832638025284, "l_c": 1.9800379127264023, "z_loss": 40.244484424591064, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16188524663448334, "tokens_per_second": 1434236.3636655353, "eta_seconds": 9745.251303124429, "data_wait_seconds": 2.765655517578125e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 45830, "total_steps": 72479, "tokens": 24028119040, "loss": 3.171015977859497, "l_mix": 1.9512830823659897, "l_a": 2.0695397406816483, "l_c": 1.9962364584207535, "z_loss": 39.25265645980835, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18983420729637146, "tokens_per_second": 1449221.824583274, "eta_seconds": 9640.864272809029, "data_wait_seconds": 2.0056962966918945e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 45840, "total_steps": 72479, "tokens": 24033361920, "loss": 4.448487758636475, "l_mix": 2.756951242685318, "l_a": 2.8422788977622986, "l_c": 2.796175926923752, "z_loss": 37.71505069732666, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16977664828300476, "tokens_per_second": 1433403.9395520592, "eta_seconds": 9743.595400166512, "data_wait_seconds": 1.84476375579834e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 45850, "total_steps": 72479, "tokens": 24038604800, "loss": 4.4086925983428955, "l_mix": 2.7314705550670624, "l_a": 2.8223754167556763, "l_c": 2.7683646082878113, "z_loss": 38.35486650466919, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2111380398273468, "tokens_per_second": 1474898.0279111606, "eta_seconds": 9465.918923068048, "data_wait_seconds": 1.615285873413086e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 45860, "total_steps": 72479, "tokens": 24043847680, "loss": 4.389566540718079, "l_mix": 2.7200623750686646, "l_a": 2.8072506487369537, "l_c": 2.757763087749481, "z_loss": 37.64946269989014, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11396907269954681, "tokens_per_second": 1455880.151380804, "eta_seconds": 9585.969187617302, "data_wait_seconds": 1.704692840576172e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 45870, "total_steps": 72479, "tokens": 24049090560, "loss": 4.494720160961151, "l_mix": 2.7861968278884888, "l_a": 2.8716528117656708, "l_c": 2.823424845933914, "z_loss": 36.73609018325806, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14652279019355774, "tokens_per_second": 1465773.4556446911, "eta_seconds": 9517.69138557911, "data_wait_seconds": 1.570582389831543e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 45880, "total_steps": 72479, "tokens": 24054333440, "loss": 3.150543659925461, "l_mix": 1.9383045732975006, "l_a": 2.0526816695928574, "l_c": 1.988115057349205, "z_loss": 38.86807107925415, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18133427202701569, "tokens_per_second": 1454662.2527287197, "eta_seconds": 9586.786545014382, "data_wait_seconds": 2.8342008590698242e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 45890, "total_steps": 72479, "tokens": 24059576320, "loss": 3.148816466331482, "l_mix": 1.936947450041771, "l_a": 2.053353801369667, "l_c": 1.9862096011638641, "z_loss": 40.25275802612305, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15961375832557678, "tokens_per_second": 1337717.758135279, "eta_seconds": 10420.952811026573, "data_wait_seconds": 2.7239322662353516e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 45900, "total_steps": 72479, "tokens": 24064819200, "loss": 4.82958710193634, "l_mix": 2.9975871443748474, "l_a": 3.0701769590377808, "l_c": 3.036489486694336, "z_loss": 35.6651406288147, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13465870916843414, "tokens_per_second": 1444440.5689945277, "eta_seconds": 9647.368712234496, "data_wait_seconds": 2.4259090423583984e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.19115095, "gate_a2b/L1_absmax": 0.72220886, "gate_a2b/L1_rms": 0.20310083, "gate_a2b/L3_absmean": 0.187796, "gate_a2b/L3_absmax": 0.57027537, "gate_a2b/L3_rms": 0.19395232, "gate_a2b/L5_absmean": 0.28752208, "gate_a2b/L5_absmax": 0.73347801, "gate_a2b/L5_rms": 0.29355532, "gate_c2b/L1_absmean": 0.01921667, "gate_c2b/L1_absmax": 0.63063377, "gate_c2b/L1_rms": 0.06667668, "gate_c2b/L3_absmean": 0.19021066, "gate_c2b/L3_absmax": 0.43228078, "gate_c2b/L3_rms": 0.19412337, "gate_c2b/L5_absmean": 0.54152501, "gate_c2b/L5_absmax": 0.76188064, "gate_c2b/L5_rms": 0.55301356, "gate_b2a/L7_absmean": 0.05100948, "gate_b2a/L7_absmax": 0.32194403, "gate_b2a/L7_rms": 0.05652216, "gate_b2a/L11_absmean": 0.08242212, "gate_b2a/L11_absmax": 0.59764558, "gate_b2a/L11_rms": 0.09130115, "gate_b2a/L14_absmean": 0.21303827, "gate_b2a/L14_absmax": 0.78949159, "gate_b2a/L14_rms": 0.22153212, "gate_b2c/L3_absmean": 0.0369237, "gate_b2c/L3_absmax": 0.57688057, "gate_b2c/L3_rms": 0.04825523, "gate_b2c/L5_absmean": 0.03988972, "gate_b2c/L5_absmax": 0.58323848, "gate_b2c/L5_rms": 0.05023342, "gate_b2c/L6_absmean": 0.05714498, "gate_b2c/L6_absmax": 0.37650719, "gate_b2c/L6_rms": 0.06386354}, "streams": {"cos_a2b/round0": 0.366695, "cos_c2b/round0": 0.31511, "cos_b2a/round0": 0.15795, "cos_b2c/round0": 0.137322, "rms_a/round0": 5.637872, "rms_b/round0": 10.319056, "rms_c/round0": 3.922472, "cos_a2b/round1": 0.423231, "cos_c2b/round1": 0.311631, "cos_b2a/round1": 0.252845, "cos_b2c/round1": 0.121798, "rms_a/round1": 10.793648, "rms_b/round1": 22.99004, "rms_c/round1": 8.15247, "cos_a2b/round2": 0.567703, "cos_c2b/round2": 0.516489, "cos_b2a/round2": 0.110675, "cos_b2c/round2": 0.187551, "rms_a/round2": 29.437038, "rms_b/round2": 60.385338, "rms_c/round2": 15.28834}, "alpha_mean": 0.4667725489300878, "alpha_hist": [0.00334, 0.04057, 0.13409, 0.18417, 0.20557, 0.2005, 0.15076, 0.07343, 0.00734, 0.00023], "alpha_by_type": {"special": {"share": 0.00053, "mean": 0.56782, "std": 0.1262}, "whitespace": {"share": 0.0493, "mean": 0.54916, "std": 0.12358}, "digit": {"share": 0.02211, "mean": 0.40998, "std": 0.13183}, "word_start": {"share": 0.59155, "mean": 0.48246, "std": 0.15254}, "word_cont": {"share": 0.23181, "mean": 0.35638, "std": 0.12912}, "punct": {"share": 0.10469, "mean": 0.59525, "std": 0.14571}}, "alpha_p05": 0.21568, "alpha_p50": 0.4598, "alpha_p95": 0.73029}
{"step": 45910, "total_steps": 72479, "tokens": 24070062080, "loss": 4.750219464302063, "l_mix": 2.947801172733307, "l_a": 3.0226293802261353, "l_c": 2.9854313135147095, "z_loss": 35.98993110656738, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1169963851571083, "tokens_per_second": 1466130.3505787954, "eta_seconds": 9501.070533394814, "data_wait_seconds": 2.765655517578125e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 45920, "total_steps": 72479, "tokens": 24075304960, "loss": 4.6735169887542725, "l_mix": 2.899309992790222, "l_a": 2.9774038195610046, "l_c": 2.936619311571121, "z_loss": 36.87317228317261, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12863671779632568, "tokens_per_second": 1466068.1844180934, "eta_seconds": 9497.897260165215, "data_wait_seconds": 1.5020370483398438e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 45930, "total_steps": 72479, "tokens": 24080547840, "loss": 3.153824269771576, "l_mix": 1.9406006783246994, "l_a": 2.0538953244686127, "l_c": 1.9901833087205887, "z_loss": 40.40659236907959, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1545209288597107, "tokens_per_second": 1408610.674601436, "eta_seconds": 9881.596358013154, "data_wait_seconds": 3.409385681152344e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 45940, "total_steps": 72479, "tokens": 24085790720, "loss": 3.1714763939380646, "l_mix": 1.9515177756547928, "l_a": 2.0672437250614166, "l_c": 1.9992847740650177, "z_loss": 39.20945501327515, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1886945515871048, "tokens_per_second": 1433171.979178493, "eta_seconds": 9708.590060472488, "data_wait_seconds": 2.3484230041503906e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 45950, "total_steps": 72479, "tokens": 24091033600, "loss": 4.463810563087463, "l_mix": 2.767027646303177, "l_a": 2.84993839263916, "l_c": 2.806004524230957, "z_loss": 37.79106569290161, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1422652304172516, "tokens_per_second": 1366325.481726848, "eta_seconds": 10179.738677215577, "data_wait_seconds": 1.8358230590820312e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 45960, "total_steps": 72479, "tokens": 24096276480, "loss": 4.4054524302482605, "l_mix": 2.7292631566524506, "l_a": 2.8186645209789276, "l_c": 2.768632799386978, "z_loss": 37.38744068145752, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17013466358184814, "tokens_per_second": 1396536.2428771213, "eta_seconds": 9955.769886326789, "data_wait_seconds": 1.6987323760986328e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 45970, "total_steps": 72479, "tokens": 24101519360, "loss": 4.392898440361023, "l_mix": 2.722968816757202, "l_a": 2.807348757982254, "l_c": 2.7590833008289337, "z_loss": 36.75183391571045, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14541997015476227, "tokens_per_second": 1345100.8053710489, "eta_seconds": 10332.57175707817, "data_wait_seconds": 1.7583370208740234e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 45980, "total_steps": 72479, "tokens": 24106762240, "loss": 4.4634469747543335, "l_mix": 2.7657666504383087, "l_a": 2.8542882204055786, "l_c": 2.804645836353302, "z_loss": 36.666396617889404, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2529154121875763, "tokens_per_second": 1488506.1367920642, "eta_seconds": 9333.591154646874, "data_wait_seconds": 1.5407800674438477e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 45990, "total_steps": 72479, "tokens": 24112005120, "loss": 3.2059895992279053, "l_mix": 1.9728662371635437, "l_a": 2.0870660543441772, "l_c": 2.023344963788986, "z_loss": 38.78697395324707, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.214283749461174, "tokens_per_second": 1417732.268413401, "eta_seconds": 9795.830384492874, "data_wait_seconds": 3.36766242980957e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 46000, "total_steps": 72479, "tokens": 24117248000, "loss": 3.190048635005951, "l_mix": 1.9639482200145721, "l_a": 2.0742982774972916, "l_c": 2.0127029567956924, "z_loss": 38.0770959854126, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20415844023227692, "tokens_per_second": 1423773.1048621435, "eta_seconds": 9750.585893630981, "data_wait_seconds": 2.8818845748901367e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.19109188, "gate_a2b/L1_absmax": 0.72009546, "gate_a2b/L1_rms": 0.20306288, "gate_a2b/L3_absmean": 0.18712404, "gate_a2b/L3_absmax": 0.56839281, "gate_a2b/L3_rms": 0.19336793, "gate_a2b/L5_absmean": 0.28719673, "gate_a2b/L5_absmax": 0.73505282, "gate_a2b/L5_rms": 0.29319218, "gate_c2b/L1_absmean": 0.01911981, "gate_c2b/L1_absmax": 0.62893325, "gate_c2b/L1_rms": 0.06642182, "gate_c2b/L3_absmean": 0.18998358, "gate_c2b/L3_absmax": 0.4310247, "gate_c2b/L3_rms": 0.1939605, "gate_c2b/L5_absmean": 0.54075056, "gate_c2b/L5_absmax": 0.76521498, "gate_c2b/L5_rms": 0.55218011, "gate_b2a/L7_absmean": 0.05101613, "gate_b2a/L7_absmax": 0.3211453, "gate_b2a/L7_rms": 0.05649858, "gate_b2a/L11_absmean": 0.08254851, "gate_b2a/L11_absmax": 0.59709364, "gate_b2a/L11_rms": 0.09144959, "gate_b2a/L14_absmean": 0.21269713, "gate_b2a/L14_absmax": 0.79059505, "gate_b2a/L14_rms": 0.22125578, "gate_b2c/L3_absmean": 0.0367592, "gate_b2c/L3_absmax": 0.5762558, "gate_b2c/L3_rms": 0.04810577, "gate_b2c/L5_absmean": 0.03993536, "gate_b2c/L5_absmax": 0.58625078, "gate_b2c/L5_rms": 0.05030565, "gate_b2c/L6_absmean": 0.05712043, "gate_b2c/L6_absmax": 0.37671626, "gate_b2c/L6_rms": 0.06390697}, "streams": {"cos_a2b/round0": 0.365602, "cos_c2b/round0": 0.271891, "cos_b2a/round0": 0.138803, "cos_b2c/round0": 0.110955, "rms_a/round0": 5.881283, "rms_b/round0": 10.499967, "rms_c/round0": 4.107013, "cos_a2b/round1": 0.414477, "cos_c2b/round1": 0.303779, "cos_b2a/round1": 0.240662, "cos_b2c/round1": 0.123042, "rms_a/round1": 11.469871, "rms_b/round1": 24.649704, "rms_c/round1": 8.708159, "cos_a2b/round2": 0.526845, "cos_c2b/round2": 0.510358, "cos_b2a/round2": 0.086859, "cos_b2c/round2": 0.172633, "rms_a/round2": 30.336668, "rms_b/round2": 63.49575, "rms_c/round2": 16.628773}, "alpha_mean": 0.4240744671169523, "alpha_hist": [0.02208, 0.12917, 0.19356, 0.16667, 0.13554, 0.11525, 0.10425, 0.09004, 0.0419, 0.00154], "alpha_by_type": {"special": {"share": 0.00108, "mean": 0.61871, "std": 0.12287}, "whitespace": {"share": 0.03314, "mean": 0.60991, "std": 0.17541}, "digit": {"share": 0.00952, "mean": 0.4118, "std": 0.17963}, "word_start": {"share": 0.5751, "mean": 0.434, "std": 0.19779}, "word_cont": {"share": 0.2596, "mean": 0.28675, "std": 0.13988}, "punct": {"share": 0.12155, "mean": 0.61897, "std": 0.17805}}, "alpha_p05": 0.12809, "alpha_p50": 0.38908, "alpha_p95": 0.7876}
{"step": 46010, "total_steps": 72479, "tokens": 24122490880, "loss": 2.9190271198749542, "l_mix": 1.8066521137952805, "l_a": 1.8705370277166367, "l_c": 1.8373795002698898, "z_loss": 46.43179988861084, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2779732942581177, "tokens_per_second": 1462011.4442253187, "eta_seconds": 9491.977047657965, "data_wait_seconds": 3.6090612411499023e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 46020, "total_steps": 72479, "tokens": 24127733760, "loss": 4.748550236225128, "l_mix": 2.946341335773468, "l_a": 3.021598905324936, "l_c": 2.9857641458511353, "z_loss": 36.51635551452637, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13805603981018066, "tokens_per_second": 1355824.9022459968, "eta_seconds": 10231.51010799408, "data_wait_seconds": 2.0176172256469727e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 46030, "total_steps": 72479, "tokens": 24132976640, "loss": 4.896297812461853, "l_mix": 3.0391155779361725, "l_a": 3.1121183931827545, "l_c": 3.0784887969493866, "z_loss": 35.94878387451172, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13670845329761505, "tokens_per_second": 1437305.5176274604, "eta_seconds": 9647.839754271508, "data_wait_seconds": 1.6868114471435547e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 46040, "total_steps": 72479, "tokens": 24138219520, "loss": 4.632437884807587, "l_mix": 2.8729373812675476, "l_a": 2.951020509004593, "l_c": 2.913980782032013, "z_loss": 37.81387138366699, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15540051460266113, "tokens_per_second": 1441240.2947859047, "eta_seconds": 9617.862116503715, "data_wait_seconds": 2.9653310775756836e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 46050, "total_steps": 72479, "tokens": 24143462400, "loss": 4.7692607045173645, "l_mix": 2.9605957865715027, "l_a": 3.032541036605835, "l_c": 2.9963418841362, "z_loss": 35.56175184249878, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.25651276111602783, "tokens_per_second": 1388624.8487791533, "eta_seconds": 9978.5104408741, "data_wait_seconds": 1.8805265426635742e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 46060, "total_steps": 72479, "tokens": 24148705280, "loss": 3.192496597766876, "l_mix": 1.9656620919704437, "l_a": 2.0772411227226257, "l_c": 2.0122068375349045, "z_loss": 38.93499994277954, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.21374335885047913, "tokens_per_second": 1476690.7968096295, "eta_seconds": 9379.867946577071, "data_wait_seconds": 2.0444393157958984e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 46070, "total_steps": 72479, "tokens": 24153948160, "loss": 4.513856887817383, "l_mix": 2.7984445691108704, "l_a": 2.8796815276145935, "l_c": 2.838359385728836, "z_loss": 37.33773374557495, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12484465539455414, "tokens_per_second": 1460606.1973363713, "eta_seconds": 9479.572123718262, "data_wait_seconds": 2.1576881408691406e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 46080, "total_steps": 72479, "tokens": 24159191040, "loss": 4.468215763568878, "l_mix": 2.769560068845749, "l_a": 2.85798242688179, "l_c": 2.804202765226364, "z_loss": 37.616682052612305, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2091551125049591, "tokens_per_second": 1447956.2077565384, "eta_seconds": 9558.768999958038, "data_wait_seconds": 1.940131187438965e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 46090, "total_steps": 72479, "tokens": 24164433920, "loss": 4.432575523853302, "l_mix": 2.745751768350601, "l_a": 2.8321807086467743, "l_c": 2.790565013885498, "z_loss": 38.48470687866211, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2028658539056778, "tokens_per_second": 1414177.3253835842, "eta_seconds": 9783.381322598458, "data_wait_seconds": 3.11732292175293e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 46100, "total_steps": 72479, "tokens": 24169676800, "loss": 3.1539515554904938, "l_mix": 1.9398770034313202, "l_a": 2.0557959973812103, "l_c": 1.9911189526319504, "z_loss": 40.44364881515503, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17818297445774078, "tokens_per_second": 1449577.2015967462, "eta_seconds": 9540.846211409567, "data_wait_seconds": 1.6510486602783203e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.19098715, "gate_a2b/L1_absmax": 0.71975613, "gate_a2b/L1_rms": 0.20285097, "gate_a2b/L3_absmean": 0.18686584, "gate_a2b/L3_absmax": 0.56693697, "gate_a2b/L3_rms": 0.19306415, "gate_a2b/L5_absmean": 0.28631994, "gate_a2b/L5_absmax": 0.73596764, "gate_a2b/L5_rms": 0.29233292, "gate_c2b/L1_absmean": 0.01920545, "gate_c2b/L1_absmax": 0.62839651, "gate_c2b/L1_rms": 0.06628345, "gate_c2b/L3_absmean": 0.18963695, "gate_c2b/L3_absmax": 0.42813092, "gate_c2b/L3_rms": 0.19359824, "gate_c2b/L5_absmean": 0.53974426, "gate_c2b/L5_absmax": 0.75995046, "gate_c2b/L5_rms": 0.55109733, "gate_b2a/L7_absmean": 0.05085045, "gate_b2a/L7_absmax": 0.32117563, "gate_b2a/L7_rms": 0.0563416, "gate_b2a/L11_absmean": 0.08258221, "gate_b2a/L11_absmax": 0.59823292, "gate_b2a/L11_rms": 0.09150587, "gate_b2a/L14_absmean": 0.212161, "gate_b2a/L14_absmax": 0.78961879, "gate_b2a/L14_rms": 0.22074206, "gate_b2c/L3_absmean": 0.03670223, "gate_b2c/L3_absmax": 0.57586199, "gate_b2c/L3_rms": 0.04808189, "gate_b2c/L5_absmean": 0.03947082, "gate_b2c/L5_absmax": 0.58430368, "gate_b2c/L5_rms": 0.04995154, "gate_b2c/L6_absmean": 0.05700576, "gate_b2c/L6_absmax": 0.37647733, "gate_b2c/L6_rms": 0.06380252}, "streams": {"cos_a2b/round0": 0.364632, "cos_c2b/round0": 0.26711, "cos_b2a/round0": 0.136183, "cos_b2c/round0": 0.108091, "rms_a/round0": 5.900311, "rms_b/round0": 10.488688, "rms_c/round0": 4.114734, "cos_a2b/round1": 0.412605, "cos_c2b/round1": 0.302169, "cos_b2a/round1": 0.235742, "cos_b2c/round1": 0.118308, "rms_a/round1": 11.529884, "rms_b/round1": 24.870806, "rms_c/round1": 8.731413, "cos_a2b/round2": 0.518556, "cos_c2b/round2": 0.51476, "cos_b2a/round2": 0.084956, "cos_b2c/round2": 0.167129, "rms_a/round2": 30.327475, "rms_b/round2": 64.019608, "rms_c/round2": 16.525139}, "alpha_mean": 0.39921881569424755, "alpha_hist": [0.03911, 0.1765, 0.19635, 0.15018, 0.11577, 0.10208, 0.09158, 0.08075, 0.0448, 0.00288], "alpha_by_type": {"special": {"share": 0.00108, "mean": 0.58549, "std": 0.12811}, "whitespace": {"share": 0.04175, "mean": 0.58816, "std": 0.18234}, "digit": {"share": 0.00888, "mean": 0.36192, "std": 0.191}, "word_start": {"share": 0.56085, "mean": 0.40243, "std": 0.21108}, "word_cont": {"share": 0.263, "mean": 0.26681, "std": 0.13933}, "punct": {"share": 0.12444, "mean": 0.60227, "std": 0.19707}}, "alpha_p05": 0.1128, "alpha_p50": 0.37708, "alpha_p95": 0.79693}
{"step": 46110, "total_steps": 72479, "tokens": 24174919680, "loss": 3.164782464504242, "l_mix": 1.9485889375209808, "l_a": 2.061469793319702, "l_c": 1.9925085306167603, "z_loss": 39.601139068603516, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.22341077029705048, "tokens_per_second": 1468566.5996938816, "eta_seconds": 9413.907598662376, "data_wait_seconds": 2.4557113647460938e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 46120, "total_steps": 72479, "tokens": 24180162560, "loss": 2.933214157819748, "l_mix": 1.8161206394433975, "l_a": 1.8813822120428085, "l_c": 1.8422628939151764, "z_loss": 42.876595973968506, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1771230697631836, "tokens_per_second": 1449444.0103312165, "eta_seconds": 9534.488599419594, "data_wait_seconds": 2.5153160095214844e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 46130, "total_steps": 72479, "tokens": 24185405440, "loss": 4.539718270301819, "l_mix": 2.813345730304718, "l_a": 2.8997834026813507, "l_c": 2.8547917008399963, "z_loss": 39.39447593688965, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.22119949758052826, "tokens_per_second": 1425342.8697973753, "eta_seconds": 9692.029058218002, "data_wait_seconds": 2.1725893020629883e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 46140, "total_steps": 72479, "tokens": 24190648320, "loss": 4.555150032043457, "l_mix": 2.8238441944122314, "l_a": 2.906750053167343, "l_c": 2.8642691373825073, "z_loss": 36.66481590270996, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1572542041540146, "tokens_per_second": 1426250.9613160596, "eta_seconds": 9682.182173085213, "data_wait_seconds": 1.6510486602783203e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 46150, "total_steps": 72479, "tokens": 24195891200, "loss": 4.488982081413269, "l_mix": 2.782658487558365, "l_a": 2.8661801517009735, "l_c": 2.8215648531913757, "z_loss": 36.166436195373535, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18514232337474823, "tokens_per_second": 1375873.328112286, "eta_seconds": 10032.884910225868, "data_wait_seconds": 1.665949821472168e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 46160, "total_steps": 72479, "tokens": 24201134080, "loss": 4.433038771152496, "l_mix": 2.747941344976425, "l_a": 2.832283765077591, "l_c": 2.7847076654434204, "z_loss": 36.93196773529053, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12073498219251633, "tokens_per_second": 1448507.2994062565, "eta_seconds": 9526.176276540757, "data_wait_seconds": 1.9282102584838867e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 46170, "total_steps": 72479, "tokens": 24206376960, "loss": 4.414007484912872, "l_mix": 2.734743893146515, "l_a": 2.8220556676387787, "l_c": 2.775489240884781, "z_loss": 38.36647701263428, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14441551268100739, "tokens_per_second": 1457756.922003007, "eta_seconds": 9462.135136389732, "data_wait_seconds": 1.6510486602783203e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 46180, "total_steps": 72479, "tokens": 24211619840, "loss": 4.528482615947723, "l_mix": 2.8071173429489136, "l_a": 2.894763708114624, "l_c": 2.843120664358139, "z_loss": 37.543524742126465, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13574591279029846, "tokens_per_second": 1434886.2210892895, "eta_seconds": 9609.298569703104, "data_wait_seconds": 2.2798776626586914e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 46190, "total_steps": 72479, "tokens": 24216862720, "loss": 4.389216065406799, "l_mix": 2.7202629446983337, "l_a": 2.806891679763794, "l_c": 2.7562853693962097, "z_loss": 38.78953742980957, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16917556524276733, "tokens_per_second": 1378985.4346137987, "eta_seconds": 9995.034672617912, "data_wait_seconds": 1.9341707229614258e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 46200, "total_steps": 72479, "tokens": 24222105600, "loss": 4.3919588923454285, "l_mix": 2.7215637862682343, "l_a": 2.806502014398575, "l_c": 2.761481672525406, "z_loss": 38.04126977920532, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1277685910463333, "tokens_per_second": 1457358.3095564647, "eta_seconds": 9453.930623412132, "data_wait_seconds": 2.372264862060547e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.19016424, "gate_a2b/L1_absmax": 0.71978384, "gate_a2b/L1_rms": 0.20215724, "gate_a2b/L3_absmean": 0.18618457, "gate_a2b/L3_absmax": 0.56554186, "gate_a2b/L3_rms": 0.19238697, "gate_a2b/L5_absmean": 0.2859906, "gate_a2b/L5_absmax": 0.7341888, "gate_a2b/L5_rms": 0.29197031, "gate_c2b/L1_absmean": 0.01941426, "gate_c2b/L1_absmax": 0.63156015, "gate_c2b/L1_rms": 0.06635942, "gate_c2b/L3_absmean": 0.18957527, "gate_c2b/L3_absmax": 0.42924389, "gate_c2b/L3_rms": 0.19356051, "gate_c2b/L5_absmean": 0.53911507, "gate_c2b/L5_absmax": 0.75950068, "gate_c2b/L5_rms": 0.55057925, "gate_b2a/L7_absmean": 0.05111166, "gate_b2a/L7_absmax": 0.32366809, "gate_b2a/L7_rms": 0.05663013, "gate_b2a/L11_absmean": 0.08262356, "gate_b2a/L11_absmax": 0.59798092, "gate_b2a/L11_rms": 0.09151215, "gate_b2a/L14_absmean": 0.2124548, "gate_b2a/L14_absmax": 0.78977805, "gate_b2a/L14_rms": 0.22102225, "gate_b2c/L3_absmean": 0.03696091, "gate_b2c/L3_absmax": 0.57552719, "gate_b2c/L3_rms": 0.04825125, "gate_b2c/L5_absmean": 0.03985114, "gate_b2c/L5_absmax": 0.58829123, "gate_b2c/L5_rms": 0.05031215, "gate_b2c/L6_absmean": 0.05696818, "gate_b2c/L6_absmax": 0.37432009, "gate_b2c/L6_rms": 0.06370621}, "streams": {"cos_a2b/round0": 0.365013, "cos_c2b/round0": 0.30238, "cos_b2a/round0": 0.149094, "cos_b2c/round0": 0.131169, "rms_a/round0": 5.669628, "rms_b/round0": 10.459896, "rms_c/round0": 4.021973, "cos_a2b/round1": 0.415103, "cos_c2b/round1": 0.297158, "cos_b2a/round1": 0.247733, "cos_b2c/round1": 0.113813, "rms_a/round1": 10.976607, "rms_b/round1": 23.873598, "rms_c/round1": 8.255696, "cos_a2b/round2": 0.54527, "cos_c2b/round2": 0.518648, "cos_b2a/round2": 0.09963, "cos_b2c/round2": 0.178793, "rms_a/round2": 29.908024, "rms_b/round2": 62.840435, "rms_c/round2": 15.799861}, "alpha_mean": 0.427570286279618, "alpha_hist": [0.01126, 0.07263, 0.18378, 0.20853, 0.18442, 0.15114, 0.11269, 0.06621, 0.0092, 0.00015], "alpha_by_type": {"special": {"share": 0.00134, "mean": 0.51636, "std": 0.13299}, "whitespace": {"share": 0.02824, "mean": 0.50675, "std": 0.12838}, "digit": {"share": 0.03856, "mean": 0.42197, "std": 0.12852}, "word_start": {"share": 0.5491, "mean": 0.44722, "std": 0.17005}, "word_cont": {"share": 0.26805, "mean": 0.32303, "std": 0.12643}, "punct": {"share": 0.1147, "mean": 0.55918, "std": 0.17077}}, "alpha_p05": 0.17526, "alpha_p50": 0.38792, "alpha_p95": 0.71602}
{"step": 46210, "total_steps": 72479, "tokens": 24227348480, "loss": 3.18254017829895, "l_mix": 1.9576524943113327, "l_a": 2.0720661878585815, "l_c": 2.010892704129219, "z_loss": 40.47543525695801, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16357341408729553, "tokens_per_second": 1398466.4064143333, "eta_seconds": 9848.303404951095, "data_wait_seconds": 1.6868114471435547e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 46220, "total_steps": 72479, "tokens": 24232591360, "loss": 3.149435728788376, "l_mix": 1.9380059093236923, "l_a": 2.052493542432785, "l_c": 1.9856058359146118, "z_loss": 40.63842487335205, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19249258935451508, "tokens_per_second": 1472062.934511364, "eta_seconds": 9352.370927381515, "data_wait_seconds": 2.2411346435546875e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 46230, "total_steps": 72479, "tokens": 24237834240, "loss": 2.811576336622238, "l_mix": 1.7405168414115906, "l_a": 1.802792564034462, "l_c": 1.7674054652452469, "z_loss": 44.526957988739014, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1624552607536316, "tokens_per_second": 1415137.3534598616, "eta_seconds": 9724.87630147934, "data_wait_seconds": 2.4646520614624023e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 46240, "total_steps": 72479, "tokens": 24243077120, "loss": 4.458803355693817, "l_mix": 2.7631039023399353, "l_a": 2.8503697514533997, "l_c": 2.801961362361908, "z_loss": 39.470523834228516, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.22309231758117676, "tokens_per_second": 1402119.6940720677, "eta_seconds": 9811.425436902045, "data_wait_seconds": 1.6361474990844727e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 46250, "total_steps": 72479, "tokens": 24248320000, "loss": 4.481116473674774, "l_mix": 2.7776834666728973, "l_a": 2.861617684364319, "l_c": 2.8164921700954437, "z_loss": 37.79400444030762, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.23267270624637604, "tokens_per_second": 1476193.0675994623, "eta_seconds": 9315.549743342399, "data_wait_seconds": 1.627206802368164e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 46260, "total_steps": 72479, "tokens": 24253562880, "loss": 4.447115540504456, "l_mix": 2.755473166704178, "l_a": 2.844986081123352, "l_c": 2.7938216626644135, "z_loss": 37.64048480987549, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18204684555530548, "tokens_per_second": 1416672.7904113075, "eta_seconds": 9703.23363661766, "data_wait_seconds": 2.8133392333984375e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 46270, "total_steps": 72479, "tokens": 24258805760, "loss": 4.441211998462677, "l_mix": 2.751911073923111, "l_a": 2.840488761663437, "l_c": 2.7905140817165375, "z_loss": 36.989691734313965, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18112993240356445, "tokens_per_second": 1364254.4034905683, "eta_seconds": 10072.215385079384, "data_wait_seconds": 2.2977590560913086e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 46280, "total_steps": 72479, "tokens": 24264048640, "loss": 4.417240560054779, "l_mix": 2.737361192703247, "l_a": 2.8243995904922485, "l_c": 2.775198310613632, "z_loss": 37.66995429992676, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14033035933971405, "tokens_per_second": 1431983.6485684905, "eta_seconds": 9592.16351788044, "data_wait_seconds": 1.940131187438965e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 46290, "total_steps": 72479, "tokens": 24269291520, "loss": 4.8167582750320435, "l_mix": 2.9890478253364563, "l_a": 3.0632958710193634, "l_c": 3.029072105884552, "z_loss": 36.06492567062378, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13986578583717346, "tokens_per_second": 1427052.7768712689, "eta_seconds": 9621.633239173889, "data_wait_seconds": 2.7477741241455078e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 46300, "total_steps": 72479, "tokens": 24274534400, "loss": 4.616523742675781, "l_mix": 2.8640689849853516, "l_a": 2.9399129450321198, "l_c": 2.9016027748584747, "z_loss": 37.49129819869995, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17844711244106293, "tokens_per_second": 1432377.082829146, "eta_seconds": 9582.208286166191, "data_wait_seconds": 2.9027462005615234e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.18961473, "gate_a2b/L1_absmax": 0.72140867, "gate_a2b/L1_rms": 0.20158044, "gate_a2b/L3_absmean": 0.18616268, "gate_a2b/L3_absmax": 0.56805414, "gate_a2b/L3_rms": 0.19236715, "gate_a2b/L5_absmean": 0.28544962, "gate_a2b/L5_absmax": 0.73748773, "gate_a2b/L5_rms": 0.29144049, "gate_c2b/L1_absmean": 0.02035163, "gate_c2b/L1_absmax": 0.63339889, "gate_c2b/L1_rms": 0.06669828, "gate_c2b/L3_absmean": 0.18937375, "gate_c2b/L3_absmax": 0.42530137, "gate_c2b/L3_rms": 0.19337326, "gate_c2b/L5_absmean": 0.53908145, "gate_c2b/L5_absmax": 0.76614487, "gate_c2b/L5_rms": 0.55050433, "gate_b2a/L7_absmean": 0.05091702, "gate_b2a/L7_absmax": 0.32389504, "gate_b2a/L7_rms": 0.05648059, "gate_b2a/L11_absmean": 0.08285767, "gate_b2a/L11_absmax": 0.59861636, "gate_b2a/L11_rms": 0.09173825, "gate_b2a/L14_absmean": 0.21269692, "gate_b2a/L14_absmax": 0.7903164, "gate_b2a/L14_rms": 0.22127503, "gate_b2c/L3_absmean": 0.0368594, "gate_b2c/L3_absmax": 0.5754351, "gate_b2c/L3_rms": 0.04814675, "gate_b2c/L5_absmean": 0.03955194, "gate_b2c/L5_absmax": 0.58702832, "gate_b2c/L5_rms": 0.05004115, "gate_b2c/L6_absmean": 0.05694749, "gate_b2c/L6_absmax": 0.3762801, "gate_b2c/L6_rms": 0.06368048}, "streams": {"cos_a2b/round0": 0.364903, "cos_c2b/round0": 0.308138, "cos_b2a/round0": 0.151023, "cos_b2c/round0": 0.132657, "rms_a/round0": 5.679657, "rms_b/round0": 10.361382, "rms_c/round0": 4.003861, "cos_a2b/round1": 0.41441, "cos_c2b/round1": 0.301891, "cos_b2a/round1": 0.249965, "cos_b2c/round1": 0.114669, "rms_a/round1": 10.836829, "rms_b/round1": 23.50304, "rms_c/round1": 8.284285, "cos_a2b/round2": 0.551718, "cos_c2b/round2": 0.521057, "cos_b2a/round2": 0.10599, "cos_b2c/round2": 0.184059, "rms_a/round2": 29.651545, "rms_b/round2": 62.164696, "rms_c/round2": 15.79084}, "alpha_mean": 0.4471234138110134, "alpha_hist": [0.00815, 0.05377, 0.15524, 0.19882, 0.19937, 0.18109, 0.12733, 0.06689, 0.00896, 0.00037], "alpha_by_type": {"special": {"share": 0.00081, "mean": 0.5672, "std": 0.15457}, "whitespace": {"share": 0.05626, "mean": 0.56064, "std": 0.12312}, "digit": {"share": 0.0305, "mean": 0.47854, "std": 0.11545}, "word_start": {"share": 0.55171, "mean": 0.45806, "std": 0.16183}, "word_cont": {"share": 0.26218, "mean": 0.34397, "std": 0.12814}, "punct": {"share": 0.09854, "mean": 0.58481, "std": 0.15625}}, "alpha_p05": 0.18832, "alpha_p50": 0.43915, "alpha_p95": 0.71443}
{"step": 46310, "total_steps": 72479, "tokens": 24279777280, "loss": 4.702962398529053, "l_mix": 2.917928636074066, "l_a": 2.9930956959724426, "l_c": 2.957016557455063, "z_loss": 37.69477653503418, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1825655847787857, "tokens_per_second": 1396031.5132542236, "eta_seconds": 9827.924757957458, "data_wait_seconds": 2.1457672119140625e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 46320, "total_steps": 72479, "tokens": 24285020160, "loss": 4.824038207530975, "l_mix": 2.993169844150543, "l_a": 3.070083200931549, "l_c": 3.032811015844345, "z_loss": 36.073872566223145, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15261024236679077, "tokens_per_second": 1359521.552115771, "eta_seconds": 10087.997333073616, "data_wait_seconds": 1.6897916793823242e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 46330, "total_steps": 72479, "tokens": 24290263040, "loss": 4.630849301815033, "l_mix": 2.8734264075756073, "l_a": 2.946546822786331, "l_c": 2.911529302597046, "z_loss": 36.17090702056885, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14291100203990936, "tokens_per_second": 1451160.3620511126, "eta_seconds": 9447.34108684063, "data_wait_seconds": 2.5004148483276367e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 46340, "total_steps": 72479, "tokens": 24295505920, "loss": 3.1496985852718353, "l_mix": 1.9387120604515076, "l_a": 2.050097644329071, "l_c": 1.986523985862732, "z_loss": 39.87745666503906, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1999812126159668, "tokens_per_second": 1464468.244185743, "eta_seconds": 9357.911369132997, "data_wait_seconds": 2.8520822525024414e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 46350, "total_steps": 72479, "tokens": 24300748800, "loss": 4.470390200614929, "l_mix": 2.7701527774333954, "l_a": 2.859699636697769, "l_c": 2.8077584207057953, "z_loss": 37.41893196105957, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12491293996572495, "tokens_per_second": 1445881.8875929785, "eta_seconds": 9474.578296852113, "data_wait_seconds": 1.5079975128173828e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 46360, "total_steps": 72479, "tokens": 24305991680, "loss": 4.454922378063202, "l_mix": 2.7612414360046387, "l_a": 2.8440655171871185, "l_c": 2.801537424325943, "z_loss": 38.07506799697876, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17646382749080658, "tokens_per_second": 1442945.1199455953, "eta_seconds": 9490.228063917159, "data_wait_seconds": 1.6570091247558594e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 46370, "total_steps": 72479, "tokens": 24311234560, "loss": 4.500507831573486, "l_mix": 2.7897001206874847, "l_a": 2.874809145927429, "l_c": 2.8278830349445343, "z_loss": 38.19202423095703, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1406482309103012, "tokens_per_second": 1456100.6233369077, "eta_seconds": 9400.88560681343, "data_wait_seconds": 1.811981201171875e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 46380, "total_steps": 72479, "tokens": 24316477440, "loss": 4.496192157268524, "l_mix": 2.787201553583145, "l_a": 2.870755821466446, "l_c": 2.8258793652057648, "z_loss": 38.10286235809326, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12254338711500168, "tokens_per_second": 1353601.8481375738, "eta_seconds": 10108.875464987754, "data_wait_seconds": 1.767277717590332e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 46390, "total_steps": 72479, "tokens": 24321720320, "loss": 4.361177742481232, "l_mix": 2.702652186155319, "l_a": 2.78732368350029, "l_c": 2.7410943806171417, "z_loss": 37.136845111846924, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17662313580513, "tokens_per_second": 1397994.6748040668, "eta_seconds": 9784.121412277222, "data_wait_seconds": 1.6868114471435547e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 46400, "total_steps": 72479, "tokens": 24326963200, "loss": 3.130091965198517, "l_mix": 1.9376750141382217, "l_a": 2.006568416953087, "l_c": 1.9681546241044998, "z_loss": 45.41761350631714, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16383664309978485, "tokens_per_second": 1437178.0471263933, "eta_seconds": 9513.718066692352, "data_wait_seconds": 1.6629695892333984e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.18917629, "gate_a2b/L1_absmax": 0.72219461, "gate_a2b/L1_rms": 0.20105883, "gate_a2b/L3_absmean": 0.18596867, "gate_a2b/L3_absmax": 0.56603146, "gate_a2b/L3_rms": 0.1921629, "gate_a2b/L5_absmean": 0.28510532, "gate_a2b/L5_absmax": 0.7369799, "gate_a2b/L5_rms": 0.29106617, "gate_c2b/L1_absmean": 0.01995423, "gate_c2b/L1_absmax": 0.6326527, "gate_c2b/L1_rms": 0.06649675, "gate_c2b/L3_absmean": 0.18970007, "gate_c2b/L3_absmax": 0.42985079, "gate_c2b/L3_rms": 0.1936837, "gate_c2b/L5_absmean": 0.53775233, "gate_c2b/L5_absmax": 0.7602849, "gate_c2b/L5_rms": 0.54914021, "gate_b2a/L7_absmean": 0.05120128, "gate_b2a/L7_absmax": 0.32229203, "gate_b2a/L7_rms": 0.0567033, "gate_b2a/L11_absmean": 0.08275698, "gate_b2a/L11_absmax": 0.59929532, "gate_b2a/L11_rms": 0.09165411, "gate_b2a/L14_absmean": 0.21242896, "gate_b2a/L14_absmax": 0.79132289, "gate_b2a/L14_rms": 0.22103429, "gate_b2c/L3_absmean": 0.03690889, "gate_b2c/L3_absmax": 0.57465595, "gate_b2c/L3_rms": 0.04813702, "gate_b2c/L5_absmean": 0.03967518, "gate_b2c/L5_absmax": 0.58405119, "gate_b2c/L5_rms": 0.05004172, "gate_b2c/L6_absmean": 0.05711253, "gate_b2c/L6_absmax": 0.37417078, "gate_b2c/L6_rms": 0.0638319}, "streams": {"cos_a2b/round0": 0.375866, "cos_c2b/round0": 0.315772, "cos_b2a/round0": 0.141223, "cos_b2c/round0": 0.133177, "rms_a/round0": 5.813471, "rms_b/round0": 10.15139, "rms_c/round0": 3.931858, "cos_a2b/round1": 0.392341, "cos_c2b/round1": 0.274414, "cos_b2a/round1": 0.225164, "cos_b2c/round1": 0.10692, "rms_a/round1": 11.103522, "rms_b/round1": 23.513733, "rms_c/round1": 8.274837, "cos_a2b/round2": 0.512612, "cos_c2b/round2": 0.534642, "cos_b2a/round2": 0.083974, "cos_b2c/round2": 0.170039, "rms_a/round2": 29.333654, "rms_b/round2": 63.625835, "rms_c/round2": 15.563315}, "alpha_mean": 0.43761014238831564, "alpha_hist": [0.01883, 0.05519, 0.149, 0.20595, 0.21129, 0.17671, 0.11584, 0.05661, 0.00983, 0.00075], "alpha_by_type": {"special": {"share": 0.0007, "mean": 0.56184, "std": 0.15087}, "whitespace": {"share": 0.08084, "mean": 0.52182, "std": 0.14032}, "digit": {"share": 0.10201, "mean": 0.41334, "std": 0.13326}, "word_start": {"share": 0.38768, "mean": 0.43371, "std": 0.16721}, "word_cont": {"share": 0.21584, "mean": 0.32715, "std": 0.1297}, "punct": {"share": 0.21294, "mean": 0.53593, "std": 0.15373}}, "alpha_p05": 0.15923, "alpha_p50": 0.43927, "alpha_p95": 0.73259}
{"step": 46410, "total_steps": 72479, "tokens": 24332206080, "loss": 4.764214277267456, "l_mix": 2.9555429220199585, "l_a": 3.0360513031482697, "l_c": 2.9928528666496277, "z_loss": 37.715120792388916, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18335698544979095, "tokens_per_second": 1343560.4485843673, "eta_seconds": 10172.719721245765, "data_wait_seconds": 2.8878450393676758e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 46420, "total_steps": 72479, "tokens": 24337448960, "loss": 4.602205514907837, "l_mix": 2.854911506175995, "l_a": 2.9292323291301727, "l_c": 2.8950809836387634, "z_loss": 36.63531255722046, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14970017969608307, "tokens_per_second": 1467339.3372514015, "eta_seconds": 9311.016644310952, "data_wait_seconds": 3.916025161743164e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 46430, "total_steps": 72479, "tokens": 24342691840, "loss": 4.704773366451263, "l_mix": 2.918726295232773, "l_a": 2.9983048737049103, "l_c": 2.9551849365234375, "z_loss": 37.06769609451294, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15661346912384033, "tokens_per_second": 1480681.2567006177, "eta_seconds": 9223.577356839181, "data_wait_seconds": 2.345442771911621e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 46440, "total_steps": 72479, "tokens": 24347934720, "loss": 4.647174596786499, "l_mix": 2.8820504546165466, "l_a": 2.9605115056037903, "l_c": 2.9232351183891296, "z_loss": 36.93514156341553, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18539026379585266, "tokens_per_second": 1464161.1923565867, "eta_seconds": 9324.065754008294, "data_wait_seconds": 3.17990779876709e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 46450, "total_steps": 72479, "tokens": 24353177600, "loss": 4.688721179962158, "l_mix": 2.9088254272937775, "l_a": 2.987743556499481, "l_c": 2.9452421963214874, "z_loss": 36.34114933013916, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18947625160217285, "tokens_per_second": 1452404.2488695772, "eta_seconds": 9395.932546067237, "data_wait_seconds": 2.6106834411621094e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 46460, "total_steps": 72479, "tokens": 24358420480, "loss": 4.4312498569488525, "l_mix": 2.7466824054718018, "l_a": 2.8293050825595856, "l_c": 2.7859192490577698, "z_loss": 37.26994180679321, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13483722507953644, "tokens_per_second": 1419662.115953341, "eta_seconds": 9608.94097173214, "data_wait_seconds": 1.570582389831543e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 46470, "total_steps": 72479, "tokens": 24363663360, "loss": 4.466886937618256, "l_mix": 2.768441140651703, "l_a": 2.8529309034347534, "l_c": 2.8085551857948303, "z_loss": 38.37011909484863, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14950980246067047, "tokens_per_second": 1418990.8104390933, "eta_seconds": 9609.792037892343, "data_wait_seconds": 1.5497207641601562e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 46480, "total_steps": 72479, "tokens": 24368906240, "loss": 4.469318926334381, "l_mix": 2.77028551697731, "l_a": 2.855120360851288, "l_c": 2.808324158191681, "z_loss": 37.3234076499939, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1623353660106659, "tokens_per_second": 1100050.5425367889, "eta_seconds": 12391.215844106673, "data_wait_seconds": 1.9162893295288086e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 46490, "total_steps": 72479, "tokens": 24374149120, "loss": 4.436359643936157, "l_mix": 2.749611347913742, "l_a": 2.835015684366226, "l_c": 2.7874782383441925, "z_loss": 38.49885654449463, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1419343501329422, "tokens_per_second": 1476991.718423742, "eta_seconds": 9225.319723892211, "data_wait_seconds": 1.5556812286376953e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 46500, "total_steps": 72479, "tokens": 24379392000, "loss": 4.4355416893959045, "l_mix": 2.74883571267128, "l_a": 2.8333758115768433, "l_c": 2.788977324962616, "z_loss": 37.24494504928589, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18752822279930115, "tokens_per_second": 1290636.5779276832, "eta_seconds": 10553.302288913728, "data_wait_seconds": 2.9653310775756836e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.18956904, "gate_a2b/L1_absmax": 0.72044748, "gate_a2b/L1_rms": 0.20133257, "gate_a2b/L3_absmean": 0.18550842, "gate_a2b/L3_absmax": 0.56674296, "gate_a2b/L3_rms": 0.19171199, "gate_a2b/L5_absmean": 0.28383908, "gate_a2b/L5_absmax": 0.73701686, "gate_a2b/L5_rms": 0.28989273, "gate_c2b/L1_absmean": 0.02030613, "gate_c2b/L1_absmax": 0.63162208, "gate_c2b/L1_rms": 0.06638153, "gate_c2b/L3_absmean": 0.18924364, "gate_c2b/L3_absmax": 0.42253187, "gate_c2b/L3_rms": 0.19314639, "gate_c2b/L5_absmean": 0.53776318, "gate_c2b/L5_absmax": 0.75731796, "gate_c2b/L5_rms": 0.54920048, "gate_b2a/L7_absmean": 0.05090267, "gate_b2a/L7_absmax": 0.32227218, "gate_b2a/L7_rms": 0.0564061, "gate_b2a/L11_absmean": 0.08264249, "gate_b2a/L11_absmax": 0.59978473, "gate_b2a/L11_rms": 0.09157179, "gate_b2a/L14_absmean": 0.21206331, "gate_b2a/L14_absmax": 0.79045868, "gate_b2a/L14_rms": 0.22065887, "gate_b2c/L3_absmean": 0.03683012, "gate_b2c/L3_absmax": 0.57454318, "gate_b2c/L3_rms": 0.04810391, "gate_b2c/L5_absmean": 0.03966153, "gate_b2c/L5_absmax": 0.58659589, "gate_b2c/L5_rms": 0.05009488, "gate_b2c/L6_absmean": 0.05699456, "gate_b2c/L6_absmax": 0.37399679, "gate_b2c/L6_rms": 0.0636538}, "streams": {"cos_a2b/round0": 0.367282, "cos_c2b/round0": 0.301313, "cos_b2a/round0": 0.150625, "cos_b2c/round0": 0.133125, "rms_a/round0": 5.711937, "rms_b/round0": 10.531032, "rms_c/round0": 4.037513, "cos_a2b/round1": 0.414008, "cos_c2b/round1": 0.30226, "cos_b2a/round1": 0.24675, "cos_b2c/round1": 0.113883, "rms_a/round1": 11.056792, "rms_b/round1": 23.963932, "rms_c/round1": 8.348295, "cos_a2b/round2": 0.54276, "cos_c2b/round2": 0.520049, "cos_b2a/round2": 0.099402, "cos_b2c/round2": 0.178679, "rms_a/round2": 30.057636, "rms_b/round2": 63.081455, "rms_c/round2": 15.944161}, "alpha_mean": 0.42919604877307904, "alpha_hist": [0.0139, 0.07364, 0.1781, 0.20267, 0.18224, 0.15771, 0.11772, 0.06485, 0.00883, 0.00034], "alpha_by_type": {"special": {"share": 0.00156, "mean": 0.54747, "std": 0.13027}, "whitespace": {"share": 0.02858, "mean": 0.51561, "std": 0.11874}, "digit": {"share": 0.03482, "mean": 0.45128, "std": 0.10919}, "word_start": {"share": 0.56543, "mean": 0.44561, "std": 0.17027}, "word_cont": {"share": 0.26454, "mean": 0.32387, "std": 0.13393}, "punct": {"share": 0.10507, "mean": 0.57349, "std": 0.16434}}, "alpha_p05": 0.17696, "alpha_p50": 0.41253, "alpha_p95": 0.72604}
{"step": 46510, "total_steps": 72479, "tokens": 24384634880, "loss": 2.9143136739730835, "l_mix": 1.804060697555542, "l_a": 1.8709045946598053, "l_c": 1.8299383223056793, "z_loss": 44.80660009384155, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.144267275929451, "tokens_per_second": 1409605.081775781, "eta_seconds": 9658.900388503074, "data_wait_seconds": 1.9043684005737305e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 46520, "total_steps": 72479, "tokens": 24389877760, "loss": 2.987388461828232, "l_mix": 1.8470364660024643, "l_a": 1.9192095696926117, "l_c": 1.8819635659456253, "z_loss": 47.17002058029175, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16783104836940765, "tokens_per_second": 1388197.8519913927, "eta_seconds": 9804.072360777855, "data_wait_seconds": 2.3543834686279297e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 46530, "total_steps": 72479, "tokens": 24395120640, "loss": 4.687269151210785, "l_mix": 2.9077709913253784, "l_a": 2.9840883314609528, "l_c": 2.947571873664856, "z_loss": 36.48323202133179, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13904812932014465, "tokens_per_second": 1423284.8857753144, "eta_seconds": 9558.697241830825, "data_wait_seconds": 1.627206802368164e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 46540, "total_steps": 72479, "tokens": 24400363520, "loss": 4.634237110614777, "l_mix": 2.8739881217479706, "l_a": 2.9533864557743073, "l_c": 2.914110004901886, "z_loss": 38.43847179412842, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17532411217689514, "tokens_per_second": 1449443.5326447615, "eta_seconds": 9382.570707798004, "data_wait_seconds": 4.1157007217407227e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 46550, "total_steps": 72479, "tokens": 24405606400, "loss": 4.690834224224091, "l_mix": 2.9105037450790405, "l_a": 2.983316421508789, "l_c": 2.9511179327964783, "z_loss": 37.74731254577637, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13690322637557983, "tokens_per_second": 1315095.8099373544, "eta_seconds": 10337.089852523804, "data_wait_seconds": 3.1381845474243164e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 46560, "total_steps": 72479, "tokens": 24410849280, "loss": 4.639648675918579, "l_mix": 2.8788561820983887, "l_a": 2.953062802553177, "l_c": 2.916245400905609, "z_loss": 36.5481390953064, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12179839611053467, "tokens_per_second": 1455015.1045062314, "eta_seconds": 9339.43615424633, "data_wait_seconds": 1.710653305053711e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 46570, "total_steps": 72479, "tokens": 24416092160, "loss": 4.486093580722809, "l_mix": 2.780427187681198, "l_a": 2.8665890991687775, "l_c": 2.818965047597885, "z_loss": 38.25438690185547, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12204256653785706, "tokens_per_second": 1398624.5512885908, "eta_seconds": 9712.24034318924, "data_wait_seconds": 1.665949821472168e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 46580, "total_steps": 72479, "tokens": 24421335040, "loss": 4.491129815578461, "l_mix": 2.7835237979888916, "l_a": 2.8707831501960754, "l_c": 2.821236401796341, "z_loss": 37.526729106903076, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11779450625181198, "tokens_per_second": 1470907.6444663187, "eta_seconds": 9231.398696637154, "data_wait_seconds": 1.5437602996826172e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 46590, "total_steps": 72479, "tokens": 24426577920, "loss": 4.533306360244751, "l_mix": 2.809970259666443, "l_a": 2.8959298133850098, "l_c": 2.848523825407028, "z_loss": 36.55043649673462, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1576182246208191, "tokens_per_second": 1473174.8387828502, "eta_seconds": 9213.63281171322, "data_wait_seconds": 1.5437602996826172e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 46600, "total_steps": 72479, "tokens": 24431820800, "loss": 4.5050950050354, "l_mix": 2.7924802005290985, "l_a": 2.880052328109741, "l_c": 2.828663647174835, "z_loss": 38.22762107849121, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14511217176914215, "tokens_per_second": 1411909.167721414, "eta_seconds": 9609.718147730828, "data_wait_seconds": 1.9252300262451172e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.18911441, "gate_a2b/L1_absmax": 0.7182678, "gate_a2b/L1_rms": 0.20096804, "gate_a2b/L3_absmean": 0.18474893, "gate_a2b/L3_absmax": 0.56653845, "gate_a2b/L3_rms": 0.19102755, "gate_a2b/L5_absmean": 0.28413272, "gate_a2b/L5_absmax": 0.73647785, "gate_a2b/L5_rms": 0.29013997, "gate_c2b/L1_absmean": 0.01986833, "gate_c2b/L1_absmax": 0.63017619, "gate_c2b/L1_rms": 0.06623361, "gate_c2b/L3_absmean": 0.18880017, "gate_c2b/L3_absmax": 0.42708012, "gate_c2b/L3_rms": 0.19275618, "gate_c2b/L5_absmean": 0.53677183, "gate_c2b/L5_absmax": 0.75114417, "gate_c2b/L5_rms": 0.54820871, "gate_b2a/L7_absmean": 0.0511381, "gate_b2a/L7_absmax": 0.32474366, "gate_b2a/L7_rms": 0.05672283, "gate_b2a/L11_absmean": 0.08247437, "gate_b2a/L11_absmax": 0.59261072, "gate_b2a/L11_rms": 0.09120985, "gate_b2a/L14_absmean": 0.21188737, "gate_b2a/L14_absmax": 0.78334874, "gate_b2a/L14_rms": 0.22052577, "gate_b2c/L3_absmean": 0.03683539, "gate_b2c/L3_absmax": 0.57241398, "gate_b2c/L3_rms": 0.04797467, "gate_b2c/L5_absmean": 0.03935082, "gate_b2c/L5_absmax": 0.57731253, "gate_b2c/L5_rms": 0.04960716, "gate_b2c/L6_absmean": 0.05688425, "gate_b2c/L6_absmax": 0.3816002, "gate_b2c/L6_rms": 0.06381363}, "streams": {"cos_a2b/round0": 0.366683, "cos_c2b/round0": 0.30353, "cos_b2a/round0": 0.147484, "cos_b2c/round0": 0.133299, "rms_a/round0": 5.77492, "rms_b/round0": 10.473503, "rms_c/round0": 4.030917, "cos_a2b/round1": 0.414718, "cos_c2b/round1": 0.302704, "cos_b2a/round1": 0.253984, "cos_b2c/round1": 0.114932, "rms_a/round1": 10.973151, "rms_b/round1": 23.600775, "rms_c/round1": 8.263497, "cos_a2b/round2": 0.551291, "cos_c2b/round2": 0.521834, "cos_b2a/round2": 0.103234, "cos_b2c/round2": 0.178003, "rms_a/round2": 29.908886, "rms_b/round2": 62.752434, "rms_c/round2": 15.361512}, "alpha_mean": 0.4481420016973843, "alpha_hist": [0.00641, 0.05449, 0.15482, 0.19739, 0.19359, 0.18872, 0.13467, 0.06111, 0.00844, 0.00037], "alpha_by_type": {"special": {"share": 0.0011, "mean": 0.57015, "std": 0.12673}, "whitespace": {"share": 0.04288, "mean": 0.56097, "std": 0.11993}, "digit": {"share": 0.0457, "mean": 0.44222, "std": 0.12585}, "word_start": {"share": 0.51465, "mean": 0.46116, "std": 0.16031}, "word_cont": {"share": 0.28203, "mean": 0.35292, "std": 0.13404}, "punct": {"share": 0.11365, "mean": 0.58413, "std": 0.1466}}, "alpha_p05": 0.18832, "alpha_p50": 0.43614, "alpha_p95": 0.7066}
{"step": 46610, "total_steps": 72479, "tokens": 24437063680, "loss": 4.380815029144287, "l_mix": 2.715272009372711, "l_a": 2.798713654279709, "l_c": 2.7530957460403442, "z_loss": 38.18499565124512, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1434096395969391, "tokens_per_second": 1235871.2331065664, "eta_seconds": 10974.287537956237, "data_wait_seconds": 0.06344503164291382, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 46620, "total_steps": 72479, "tokens": 24442306560, "loss": 4.524131178855896, "l_mix": 2.8048123717308044, "l_a": 2.887011468410492, "l_c": 2.844050943851471, "z_loss": 37.36204481124878, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16010116040706635, "tokens_per_second": 1309609.769330669, "eta_seconds": 10352.368857884408, "data_wait_seconds": 3.644824028015137e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 46630, "total_steps": 72479, "tokens": 24447549440, "loss": 2.7932332158088684, "l_mix": 1.7294306308031082, "l_a": 1.789454996585846, "l_c": 1.7565533369779587, "z_loss": 46.843082904815674, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1762821227312088, "tokens_per_second": 1408369.6206498235, "eta_seconds": 9622.701536083221, "data_wait_seconds": 3.325939178466797e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 46640, "total_steps": 72479, "tokens": 24452792320, "loss": 4.393374294042587, "l_mix": 2.7213023602962494, "l_a": 2.8149023056030273, "l_c": 2.7586707770824432, "z_loss": 39.27551746368408, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15274569392204285, "tokens_per_second": 1441806.9876557975, "eta_seconds": 9395.902328109742, "data_wait_seconds": 2.2202730178833008e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 46650, "total_steps": 72479, "tokens": 24458035200, "loss": 4.500777244567871, "l_mix": 2.789754807949066, "l_a": 2.874874234199524, "l_c": 2.8285335898399353, "z_loss": 36.92094659805298, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18308435380458832, "tokens_per_second": 1423392.7664527155, "eta_seconds": 9513.772355151177, "data_wait_seconds": 2.0682811737060547e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 46660, "total_steps": 72479, "tokens": 24463278080, "loss": 4.520435810089111, "l_mix": 2.803258329629898, "l_a": 2.8844663500785828, "l_c": 2.839458078145981, "z_loss": 36.818392276763916, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1820879876613617, "tokens_per_second": 1356969.0208618667, "eta_seconds": 9975.608627676964, "data_wait_seconds": 1.5228986740112305e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 46670, "total_steps": 72479, "tokens": 24468520960, "loss": 4.511252343654633, "l_mix": 2.7962948977947235, "l_a": 2.8846136331558228, "l_c": 2.8319112360477448, "z_loss": 36.84618377685547, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1634611338376999, "tokens_per_second": 1417519.6978052722, "eta_seconds": 9545.792564964295, "data_wait_seconds": 1.9371509552001953e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 46680, "total_steps": 72479, "tokens": 24473763840, "loss": 4.374766170978546, "l_mix": 2.7109602093696594, "l_a": 2.797206163406372, "l_c": 2.7488133907318115, "z_loss": 37.898630142211914, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1428002268075943, "tokens_per_second": 1442265.903372617, "eta_seconds": 9378.37196342945, "data_wait_seconds": 1.6808509826660156e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 46690, "total_steps": 72479, "tokens": 24479006720, "loss": 4.472701132297516, "l_mix": 2.7721463441848755, "l_a": 2.859064906835556, "l_c": 2.8094506561756134, "z_loss": 37.93301725387573, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13265961408615112, "tokens_per_second": 1361806.4431849709, "eta_seconds": 9928.623336791992, "data_wait_seconds": 2.0056962966918945e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 46700, "total_steps": 72479, "tokens": 24484249600, "loss": 4.510859787464142, "l_mix": 2.7955407202243805, "l_a": 2.88321653008461, "l_c": 2.8345136642456055, "z_loss": 38.13361406326294, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11675956100225449, "tokens_per_second": 1392511.2408486414, "eta_seconds": 9705.932674384117, "data_wait_seconds": 1.710653305053711e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.18896718, "gate_a2b/L1_absmax": 0.71674651, "gate_a2b/L1_rms": 0.20079316, "gate_a2b/L3_absmean": 0.18467832, "gate_a2b/L3_absmax": 0.56331062, "gate_a2b/L3_rms": 0.19086517, "gate_a2b/L5_absmean": 0.28302315, "gate_a2b/L5_absmax": 0.73624533, "gate_a2b/L5_rms": 0.28903326, "gate_c2b/L1_absmean": 0.01984606, "gate_c2b/L1_absmax": 0.62913936, "gate_c2b/L1_rms": 0.06626558, "gate_c2b/L3_absmean": 0.18873861, "gate_c2b/L3_absmax": 0.423769, "gate_c2b/L3_rms": 0.19269526, "gate_c2b/L5_absmean": 0.53667217, "gate_c2b/L5_absmax": 0.7562747, "gate_c2b/L5_rms": 0.54815006, "gate_b2a/L7_absmean": 0.0510745, "gate_b2a/L7_absmax": 0.32444692, "gate_b2a/L7_rms": 0.05660955, "gate_b2a/L11_absmean": 0.08249953, "gate_b2a/L11_absmax": 0.59598577, "gate_b2a/L11_rms": 0.09128915, "gate_b2a/L14_absmean": 0.21222109, "gate_b2a/L14_absmax": 0.78303534, "gate_b2a/L14_rms": 0.22077973, "gate_b2c/L3_absmean": 0.03658771, "gate_b2c/L3_absmax": 0.5714671, "gate_b2c/L3_rms": 0.04785174, "gate_b2c/L5_absmean": 0.03958114, "gate_b2c/L5_absmax": 0.57834673, "gate_b2c/L5_rms": 0.04978707, "gate_b2c/L6_absmean": 0.05684828, "gate_b2c/L6_absmax": 0.37830958, "gate_b2c/L6_rms": 0.06368423}, "streams": {"cos_a2b/round0": 0.364875, "cos_c2b/round0": 0.308536, "cos_b2a/round0": 0.152974, "cos_b2c/round0": 0.136693, "rms_a/round0": 5.776579, "rms_b/round0": 10.503926, "rms_c/round0": 4.04429, "cos_a2b/round1": 0.419025, "cos_c2b/round1": 0.304323, "cos_b2a/round1": 0.250592, "cos_b2c/round1": 0.113006, "rms_a/round1": 11.020171, "rms_b/round1": 23.838306, "rms_c/round1": 8.302214, "cos_a2b/round2": 0.55012, "cos_c2b/round2": 0.521335, "cos_b2a/round2": 0.107184, "cos_b2c/round2": 0.183004, "rms_a/round2": 30.235817, "rms_b/round2": 63.336109, "rms_c/round2": 15.621382}, "alpha_mean": 0.4292920542850993, "alpha_hist": [0.01076, 0.076, 0.17793, 0.19661, 0.18765, 0.16039, 0.12076, 0.06332, 0.00648, 9e-05], "alpha_by_type": {"special": {"share": 0.00096, "mean": 0.54141, "std": 0.13137}, "whitespace": {"share": 0.02969, "mean": 0.48752, "std": 0.11618}, "digit": {"share": 0.0273, "mean": 0.41414, "std": 0.12091}, "word_start": {"share": 0.57059, "mean": 0.45191, "std": 0.16461}, "word_cont": {"share": 0.26276, "mean": 0.31225, "std": 0.1274}, "punct": {"share": 0.1087, "mean": 0.58041, "std": 0.15195}}, "alpha_p05": 0.16559, "alpha_p50": 0.40804, "alpha_p95": 0.72643}
{"step": 46710, "total_steps": 72479, "tokens": 24489492480, "loss": 4.528481423854828, "l_mix": 2.8073648810386658, "l_a": 2.893289476633072, "l_c": 2.843765139579773, "z_loss": 36.701284408569336, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1388096660375595, "tokens_per_second": 1453428.8102049583, "eta_seconds": 9295.520617961884, "data_wait_seconds": 1.755356788635254e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 46720, "total_steps": 72479, "tokens": 24494735360, "loss": 4.3929519057273865, "l_mix": 2.7215078473091125, "l_a": 2.812522292137146, "l_c": 2.7589575350284576, "z_loss": 38.051294803619385, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14028416574001312, "tokens_per_second": 1415159.3923386566, "eta_seconds": 9543.189738988876, "data_wait_seconds": 2.0891427993774414e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 46730, "total_steps": 72479, "tokens": 24499978240, "loss": 4.59828108549118, "l_mix": 2.8504901826381683, "l_a": 2.936471074819565, "l_c": 2.889498293399811, "z_loss": 37.8946590423584, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1535453349351883, "tokens_per_second": 1404422.6747371685, "eta_seconds": 9612.41366636753, "data_wait_seconds": 2.893805503845215e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 46740, "total_steps": 72479, "tokens": 24505221120, "loss": 3.023266524076462, "l_mix": 1.8714035898447037, "l_a": 1.9394976198673248, "l_c": 1.9000453054904938, "z_loss": 45.25603199005127, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12038542330265045, "tokens_per_second": 1466833.2143238785, "eta_seconds": 9199.852239656448, "data_wait_seconds": 2.518296241760254e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 46750, "total_steps": 72479, "tokens": 24510464000, "loss": 4.4695063829422, "l_mix": 2.770260840654373, "l_a": 2.855656683444977, "l_c": 2.8084949254989624, "z_loss": 37.55992031097412, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11240675300359726, "tokens_per_second": 1335520.2203469668, "eta_seconds": 10100.487994480134, "data_wait_seconds": 2.09808349609375e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 46760, "total_steps": 72479, "tokens": 24515706880, "loss": 4.4370197057724, "l_mix": 2.7503491640090942, "l_a": 2.8351496756076813, "l_c": 2.787085145711899, "z_loss": 36.94576454162598, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14517058432102203, "tokens_per_second": 1346668.6962743662, "eta_seconds": 10012.977289295197, "data_wait_seconds": 2.2113323211669922e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 46770, "total_steps": 72479, "tokens": 24520949760, "loss": 4.5190388560295105, "l_mix": 2.80106320977211, "l_a": 2.8853387534618378, "l_c": 2.8412464261054993, "z_loss": 37.02083969116211, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16746361553668976, "tokens_per_second": 1436185.359314233, "eta_seconds": 9385.223226642609, "data_wait_seconds": 1.5676021575927734e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 46780, "total_steps": 72479, "tokens": 24526192640, "loss": 4.481458067893982, "l_mix": 2.778181880712509, "l_a": 2.864976942539215, "l_c": 2.8126097917556763, "z_loss": 36.277846336364746, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12882864475250244, "tokens_per_second": 1434094.4728614024, "eta_seconds": 9395.250847816467, "data_wait_seconds": 3.516674041748047e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 46790, "total_steps": 72479, "tokens": 24531435520, "loss": 4.437609851360321, "l_mix": 2.7495803236961365, "l_a": 2.8386426866054535, "l_c": 2.788122057914734, "z_loss": 38.44985628128052, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15778692066669464, "tokens_per_second": 1338273.9537410021, "eta_seconds": 10064.03389556408, "data_wait_seconds": 2.6345252990722656e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 46800, "total_steps": 72479, "tokens": 24536678400, "loss": 4.283646613359451, "l_mix": 2.654835522174835, "l_a": 2.7395308315753937, "l_c": 2.6898393630981445, "z_loss": 38.456724643707275, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14938974380493164, "tokens_per_second": 1462157.5522611074, "eta_seconds": 9207.757078695297, "data_wait_seconds": 2.1517276763916016e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.18891488, "gate_a2b/L1_absmax": 0.71701241, "gate_a2b/L1_rms": 0.20070878, "gate_a2b/L3_absmean": 0.18445753, "gate_a2b/L3_absmax": 0.56132334, "gate_a2b/L3_rms": 0.1906087, "gate_a2b/L5_absmean": 0.28282252, "gate_a2b/L5_absmax": 0.7352047, "gate_a2b/L5_rms": 0.28879136, "gate_c2b/L1_absmean": 0.02012924, "gate_c2b/L1_absmax": 0.62833405, "gate_c2b/L1_rms": 0.06608825, "gate_c2b/L3_absmean": 0.18833397, "gate_c2b/L3_absmax": 0.42606944, "gate_c2b/L3_rms": 0.19234596, "gate_c2b/L5_absmean": 0.53646803, "gate_c2b/L5_absmax": 0.75199997, "gate_c2b/L5_rms": 0.54782182, "gate_b2a/L7_absmean": 0.05113983, "gate_b2a/L7_absmax": 0.32514998, "gate_b2a/L7_rms": 0.05669094, "gate_b2a/L11_absmean": 0.08231522, "gate_b2a/L11_absmax": 0.59593135, "gate_b2a/L11_rms": 0.09114085, "gate_b2a/L14_absmean": 0.21203315, "gate_b2a/L14_absmax": 0.7841711, "gate_b2a/L14_rms": 0.22066909, "gate_b2c/L3_absmean": 0.03653434, "gate_b2c/L3_absmax": 0.57245332, "gate_b2c/L3_rms": 0.04784132, "gate_b2c/L5_absmean": 0.03941743, "gate_b2c/L5_absmax": 0.57864404, "gate_b2c/L5_rms": 0.04964694, "gate_b2c/L6_absmean": 0.05666642, "gate_b2c/L6_absmax": 0.37705138, "gate_b2c/L6_rms": 0.06349776}, "streams": {"cos_a2b/round0": 0.364813, "cos_c2b/round0": 0.306726, "cos_b2a/round0": 0.153732, "cos_b2c/round0": 0.135456, "rms_a/round0": 5.755548, "rms_b/round0": 10.51164, "rms_c/round0": 4.03596, "cos_a2b/round1": 0.418382, "cos_c2b/round1": 0.301553, "cos_b2a/round1": 0.248948, "cos_b2c/round1": 0.117039, "rms_a/round1": 11.112041, "rms_b/round1": 23.917059, "rms_c/round1": 8.259861, "cos_a2b/round2": 0.557148, "cos_c2b/round2": 0.514355, "cos_b2a/round2": 0.106306, "cos_b2c/round2": 0.184001, "rms_a/round2": 30.481007, "rms_b/round2": 63.444324, "rms_c/round2": 15.638391}, "alpha_mean": 0.4472567551841351, "alpha_hist": [0.00687, 0.06244, 0.16333, 0.19075, 0.18391, 0.17619, 0.13232, 0.07358, 0.0105, 0.00011], "alpha_by_type": {"special": {"share": 0.00107, "mean": 0.53518, "std": 0.09861}, "whitespace": {"share": 0.02788, "mean": 0.52684, "std": 0.11126}, "digit": {"share": 0.0285, "mean": 0.47885, "std": 0.11557}, "word_start": {"share": 0.58441, "mean": 0.4596, "std": 0.1658}, "word_cont": {"share": 0.24657, "mean": 0.33705, "std": 0.13069}, "punct": {"share": 0.11157, "mean": 0.59738, "std": 0.15801}}, "alpha_p05": 0.17106, "alpha_p50": 0.44505, "alpha_p95": 0.74614}
{"step": 46810, "total_steps": 72479, "tokens": 24541921280, "loss": 4.643418788909912, "l_mix": 2.8805906176567078, "l_a": 2.9568223357200623, "l_c": 2.9192711412906647, "z_loss": 37.13028573989868, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1136736050248146, "tokens_per_second": 1453652.9605534056, "eta_seconds": 9258.020337176324, "data_wait_seconds": 3.272294998168945e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 46820, "total_steps": 72479, "tokens": 24547164160, "loss": 4.705567955970764, "l_mix": 2.9189494848251343, "l_a": 2.999451607465744, "l_c": 2.955942988395691, "z_loss": 36.17878293991089, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19281157851219177, "tokens_per_second": 1400784.1604594982, "eta_seconds": 9603.696395015717, "data_wait_seconds": 1.7404556274414062e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 46830, "total_steps": 72479, "tokens": 24552407040, "loss": 4.7187517285346985, "l_mix": 2.928348124027252, "l_a": 3.0007824897766113, "l_c": 2.967229276895523, "z_loss": 35.76124286651611, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17899130284786224, "tokens_per_second": 1426225.985583401, "eta_seconds": 9428.704180073737, "data_wait_seconds": 1.665949821472168e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 46840, "total_steps": 72479, "tokens": 24557649920, "loss": 4.752823531627655, "l_mix": 2.9484024047851562, "l_a": 3.029703974723816, "l_c": 2.9850328266620636, "z_loss": 36.28980588912964, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15645048022270203, "tokens_per_second": 1386071.0819100097, "eta_seconds": 9698.07407963276, "data_wait_seconds": 4.935264587402344e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 46850, "total_steps": 72479, "tokens": 24562892800, "loss": 4.54732346534729, "l_mix": 2.82185959815979, "l_a": 2.8921969830989838, "l_c": 2.8593489825725555, "z_loss": 37.32183790206909, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14150314033031464, "tokens_per_second": 1383822.359835821, "eta_seconds": 9710.044830894469, "data_wait_seconds": 2.637505531311035e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 46860, "total_steps": 72479, "tokens": 24568135680, "loss": 2.981442868709564, "l_mix": 1.8455466628074646, "l_a": 1.9114890545606613, "l_c": 1.8748314827680588, "z_loss": 44.98384428024292, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1720578670501709, "tokens_per_second": 1434562.6225641072, "eta_seconds": 9362.947326755524, "data_wait_seconds": 1.671910285949707e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 46870, "total_steps": 72479, "tokens": 24573378560, "loss": 4.356516510248184, "l_mix": 2.7002044022083282, "l_a": 2.7828140556812286, "l_c": 2.738226056098938, "z_loss": 38.26240062713623, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11035943776369095, "tokens_per_second": 1392355.533276977, "eta_seconds": 9643.005016398429, "data_wait_seconds": 3.790855407714844e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 46880, "total_steps": 72479, "tokens": 24578621440, "loss": 4.412695944309235, "l_mix": 2.735147476196289, "l_a": 2.8197392523288727, "l_c": 2.7720886766910553, "z_loss": 37.337642192840576, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12663520872592926, "tokens_per_second": 1438165.7123154865, "eta_seconds": 9332.198923301696, "data_wait_seconds": 1.8149614334106445e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 46890, "total_steps": 72479, "tokens": 24583864320, "loss": 4.511885106563568, "l_mix": 2.7971992790699005, "l_a": 2.8810143768787384, "l_c": 2.8346052765846252, "z_loss": 37.54812717437744, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1257537454366684, "tokens_per_second": 1409463.7765243503, "eta_seconds": 9518.51750676632, "data_wait_seconds": 1.996755599975586e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 46900, "total_steps": 72479, "tokens": 24589107200, "loss": 4.475602090358734, "l_mix": 2.77334064245224, "l_a": 2.8632228672504425, "l_c": 2.810982048511505, "z_loss": 37.332247257232666, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1267814189195633, "tokens_per_second": 1382499.9707986906, "eta_seconds": 9700.371092414856, "data_wait_seconds": 1.5497207641601562e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.18893223, "gate_a2b/L1_absmax": 0.71713638, "gate_a2b/L1_rms": 0.20058431, "gate_a2b/L3_absmean": 0.18419579, "gate_a2b/L3_absmax": 0.56324536, "gate_a2b/L3_rms": 0.19030097, "gate_a2b/L5_absmean": 0.28196478, "gate_a2b/L5_absmax": 0.73646098, "gate_a2b/L5_rms": 0.28798303, "gate_c2b/L1_absmean": 0.02072083, "gate_c2b/L1_absmax": 0.629816, "gate_c2b/L1_rms": 0.06626775, "gate_c2b/L3_absmean": 0.18805425, "gate_c2b/L3_absmax": 0.42587051, "gate_c2b/L3_rms": 0.19201206, "gate_c2b/L5_absmean": 0.53562039, "gate_c2b/L5_absmax": 0.75513691, "gate_c2b/L5_rms": 0.5469954, "gate_b2a/L7_absmean": 0.05092325, "gate_b2a/L7_absmax": 0.32402095, "gate_b2a/L7_rms": 0.05646554, "gate_b2a/L11_absmean": 0.0823996, "gate_b2a/L11_absmax": 0.59589744, "gate_b2a/L11_rms": 0.09126072, "gate_b2a/L14_absmean": 0.21171969, "gate_b2a/L14_absmax": 0.78534949, "gate_b2a/L14_rms": 0.22032039, "gate_b2c/L3_absmean": 0.03660347, "gate_b2c/L3_absmax": 0.57194108, "gate_b2c/L3_rms": 0.04787423, "gate_b2c/L5_absmean": 0.03935741, "gate_b2c/L5_absmax": 0.57998306, "gate_b2c/L5_rms": 0.04966053, "gate_b2c/L6_absmean": 0.05688021, "gate_b2c/L6_absmax": 0.37631601, "gate_b2c/L6_rms": 0.06365241}, "streams": {"cos_a2b/round0": 0.36754, "cos_c2b/round0": 0.304995, "cos_b2a/round0": 0.152854, "cos_b2c/round0": 0.134833, "rms_a/round0": 5.806042, "rms_b/round0": 10.487967, "rms_c/round0": 4.079404, "cos_a2b/round1": 0.416952, "cos_c2b/round1": 0.301863, "cos_b2a/round1": 0.251187, "cos_b2c/round1": 0.114266, "rms_a/round1": 11.093093, "rms_b/round1": 23.995787, "rms_c/round1": 8.345975, "cos_a2b/round2": 0.552494, "cos_c2b/round2": 0.52063, "cos_b2a/round2": 0.104038, "cos_b2c/round2": 0.181698, "rms_a/round2": 30.389702, "rms_b/round2": 63.728035, "rms_c/round2": 15.763453}, "alpha_mean": 0.4395026043127146, "alpha_hist": [0.00916, 0.06961, 0.17329, 0.19548, 0.1797, 0.16074, 0.12788, 0.07199, 0.01212, 3e-05], "alpha_by_type": {"special": {"share": 0.00107, "mean": 0.54392, "std": 0.10861}, "whitespace": {"share": 0.02599, "mean": 0.50549, "std": 0.12719}, "digit": {"share": 0.03743, "mean": 0.46541, "std": 0.11301}, "word_start": {"share": 0.58438, "mean": 0.45471, "std": 0.17166}, "word_cont": {"share": 0.25015, "mean": 0.33134, "std": 0.13477}, "punct": {"share": 0.10098, "mean": 0.59176, "std": 0.15793}}, "alpha_p05": 0.16996, "alpha_p50": 0.44722, "alpha_p95": 0.74945}
{"step": 46910, "total_steps": 72479, "tokens": 24594350080, "loss": 4.547862887382507, "l_mix": 2.8197626769542694, "l_a": 2.901861995458603, "l_c": 2.858471781015396, "z_loss": 37.658583641052246, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12925797700881958, "tokens_per_second": 1417102.236079905, "eta_seconds": 9459.81138882637, "data_wait_seconds": 2.014636993408203e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 46920, "total_steps": 72479, "tokens": 24599592960, "loss": 4.693965673446655, "l_mix": 2.911807030439377, "l_a": 2.9905762672424316, "l_c": 2.9499524533748627, "z_loss": 37.14697456359863, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15281715989112854, "tokens_per_second": 1452193.0462230337, "eta_seconds": 9227.614074349403, "data_wait_seconds": 2.714991569519043e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 46930, "total_steps": 72479, "tokens": 24604835840, "loss": 4.721231281757355, "l_mix": 2.928851693868637, "l_a": 3.0103313326835632, "l_c": 2.964267134666443, "z_loss": 36.75731563568115, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14006754755973816, "tokens_per_second": 1369362.0929659372, "eta_seconds": 9781.951888990401, "data_wait_seconds": 2.2649765014648438e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 46940, "total_steps": 72479, "tokens": 24610078720, "loss": 4.5044368505477905, "l_mix": 2.7944553792476654, "l_a": 2.8697111308574677, "l_c": 2.830226629972458, "z_loss": 36.160207748413086, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.205180823802948, "tokens_per_second": 1446890.6002071684, "eta_seconds": 9254.183578276634, "data_wait_seconds": 1.9788742065429688e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 46950, "total_steps": 72479, "tokens": 24615321600, "loss": 4.615775763988495, "l_mix": 2.862328350543976, "l_a": 2.9437931180000305, "l_c": 2.9010313153266907, "z_loss": 37.92055892944336, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15679112076759338, "tokens_per_second": 1483025.8097390402, "eta_seconds": 9025.162113904953, "data_wait_seconds": 2.6434659957885742e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 46960, "total_steps": 72479, "tokens": 24620564480, "loss": 4.734989166259766, "l_mix": 2.937365800142288, "l_a": 3.0163512229919434, "l_c": 2.9757262468338013, "z_loss": 36.9917778968811, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11817710846662521, "tokens_per_second": 1379481.1998285672, "eta_seconds": 9698.795078659057, "data_wait_seconds": 4.0084123611450195e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 46970, "total_steps": 72479, "tokens": 24625807360, "loss": 2.826504200696945, "l_mix": 1.7503492683172226, "l_a": 1.8107809722423553, "l_c": 1.776401847600937, "z_loss": 44.35527229309082, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.21661128103733063, "tokens_per_second": 1354238.9667346017, "eta_seconds": 9875.703565263748, "data_wait_seconds": 2.765655517578125e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 46980, "total_steps": 72479, "tokens": 24631050240, "loss": 4.3252257108688354, "l_mix": 2.6800964176654816, "l_a": 2.768406420946121, "l_c": 2.7153574526309967, "z_loss": 38.05308961868286, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14259952306747437, "tokens_per_second": 1445955.474216434, "eta_seconds": 9245.664856481551, "data_wait_seconds": 2.327561378479004e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 46990, "total_steps": 72479, "tokens": 24636293120, "loss": 4.412988662719727, "l_mix": 2.734801232814789, "l_a": 2.8220970928668976, "l_c": 2.7718609869480133, "z_loss": 38.20204544067383, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1272767335176468, "tokens_per_second": 1322015.7331747173, "eta_seconds": 10108.485471582411, "data_wait_seconds": 2.1070241928100586e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 47000, "total_steps": 72479, "tokens": 24641536000, "loss": 4.38175106048584, "l_mix": 2.7159522771835327, "l_a": 2.7979067265987396, "l_c": 2.7547556161880493, "z_loss": 37.69718790054321, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13915464282035828, "tokens_per_second": 1420553.712642788, "eta_seconds": 9403.610601353645, "data_wait_seconds": 1.5795230865478516e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.18891692, "gate_a2b/L1_absmax": 0.71766293, "gate_a2b/L1_rms": 0.20059201, "gate_a2b/L3_absmean": 0.18393052, "gate_a2b/L3_absmax": 0.55950803, "gate_a2b/L3_rms": 0.19005366, "gate_a2b/L5_absmean": 0.2811318, "gate_a2b/L5_absmax": 0.7358951, "gate_a2b/L5_rms": 0.28713655, "gate_c2b/L1_absmean": 0.01991603, "gate_c2b/L1_absmax": 0.63046896, "gate_c2b/L1_rms": 0.06612292, "gate_c2b/L3_absmean": 0.18844223, "gate_c2b/L3_absmax": 0.42216668, "gate_c2b/L3_rms": 0.19245896, "gate_c2b/L5_absmean": 0.53567034, "gate_c2b/L5_absmax": 0.75373632, "gate_c2b/L5_rms": 0.54712027, "gate_b2a/L7_absmean": 0.05100146, "gate_b2a/L7_absmax": 0.3232215, "gate_b2a/L7_rms": 0.05656287, "gate_b2a/L11_absmean": 0.08252573, "gate_b2a/L11_absmax": 0.59548992, "gate_b2a/L11_rms": 0.09133246, "gate_b2a/L14_absmean": 0.21173875, "gate_b2a/L14_absmax": 0.78620416, "gate_b2a/L14_rms": 0.22037418, "gate_b2c/L3_absmean": 0.03672715, "gate_b2c/L3_absmax": 0.57113051, "gate_b2c/L3_rms": 0.04791467, "gate_b2c/L5_absmean": 0.03933547, "gate_b2c/L5_absmax": 0.58107591, "gate_b2c/L5_rms": 0.04967854, "gate_b2c/L6_absmean": 0.05672874, "gate_b2c/L6_absmax": 0.3767817, "gate_b2c/L6_rms": 0.06344404}, "streams": {"cos_a2b/round0": 0.36896, "cos_c2b/round0": 0.306941, "cos_b2a/round0": 0.151248, "cos_b2c/round0": 0.134028, "rms_a/round0": 5.824857, "rms_b/round0": 10.531402, "rms_c/round0": 4.102353, "cos_a2b/round1": 0.417986, "cos_c2b/round1": 0.299955, "cos_b2a/round1": 0.252891, "cos_b2c/round1": 0.116157, "rms_a/round1": 11.150623, "rms_b/round1": 24.082365, "rms_c/round1": 8.386672, "cos_a2b/round2": 0.54243, "cos_c2b/round2": 0.528574, "cos_b2a/round2": 0.102411, "cos_b2c/round2": 0.182799, "rms_a/round2": 30.50873, "rms_b/round2": 64.361343, "rms_c/round2": 16.031994}, "alpha_mean": 0.4251679645651194, "alpha_hist": [0.01012, 0.08063, 0.18518, 0.20154, 0.17763, 0.16017, 0.1172, 0.06148, 0.00606, 0.0], "alpha_by_type": {"special": {"share": 0.00125, "mean": 0.56895, "std": 0.10881}, "whitespace": {"share": 0.02986, "mean": 0.50667, "std": 0.12141}, "digit": {"share": 0.0381, "mean": 0.47485, "std": 0.1237}, "word_start": {"share": 0.56599, "mean": 0.43663, "std": 0.16759}, "word_cont": {"share": 0.26051, "mean": 0.32374, "std": 0.13376}, "punct": {"share": 0.10428, "mean": 0.57316, "std": 0.15987}}, "alpha_p05": 0.16318, "alpha_p50": 0.40592, "alpha_p95": 0.72487}
{"step": 47010, "total_steps": 72479, "tokens": 24646778880, "loss": 4.4722596406936646, "l_mix": 2.7715228497982025, "l_a": 2.860859304666519, "l_c": 2.8082627952098846, "z_loss": 38.134657859802246, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13320167362689972, "tokens_per_second": 1394931.381426092, "eta_seconds": 9572.579160380363, "data_wait_seconds": 4.330277442932129e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 47020, "total_steps": 72479, "tokens": 24652021760, "loss": 4.439101755619049, "l_mix": 2.7505355775356293, "l_a": 2.839214324951172, "l_c": 2.789339303970337, "z_loss": 37.96755886077881, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14292091131210327, "tokens_per_second": 1429481.8547444677, "eta_seconds": 9337.542933964729, "data_wait_seconds": 1.704692840576172e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 47030, "total_steps": 72479, "tokens": 24657264640, "loss": 2.906620681285858, "l_mix": 1.7984898388385773, "l_a": 1.8668962717056274, "l_c": 1.8268730193376541, "z_loss": 44.79787349700928, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16354843974113464, "tokens_per_second": 1361256.9781801295, "eta_seconds": 9801.68001036644, "data_wait_seconds": 2.0444393157958984e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 47040, "total_steps": 72479, "tokens": 24662507520, "loss": 4.582875192165375, "l_mix": 2.8434912860393524, "l_a": 2.9191659092903137, "l_c": 2.8787801861763, "z_loss": 36.181260108947754, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14757230877876282, "tokens_per_second": 1461995.1146025173, "eta_seconds": 9122.713406348228, "data_wait_seconds": 2.0742416381835938e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 47050, "total_steps": 72479, "tokens": 24667750400, "loss": 4.659262716770172, "l_mix": 2.889909476041794, "l_a": 2.9657354652881622, "l_c": 2.932108461856842, "z_loss": 37.69909715652466, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.22617539763450623, "tokens_per_second": 1384963.4711415146, "eta_seconds": 9626.33298985958, "data_wait_seconds": 2.333521842956543e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 47060, "total_steps": 72479, "tokens": 24672993280, "loss": 4.5597721338272095, "l_mix": 2.8288192451000214, "l_a": 2.904090076684952, "l_c": 2.8657528460025787, "z_loss": 38.920523166656494, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.26027899980545044, "tokens_per_second": 1442941.2379727461, "eta_seconds": 9235.91087515354, "data_wait_seconds": 2.2113323211669922e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 47070, "total_steps": 72479, "tokens": 24678236160, "loss": 4.6503337025642395, "l_mix": 2.8844490945339203, "l_a": 2.9623986780643463, "l_c": 2.923883080482483, "z_loss": 37.820332527160645, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1241711974143982, "tokens_per_second": 1475568.2340775358, "eta_seconds": 9028.13809916973, "data_wait_seconds": 1.9222497940063477e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 47080, "total_steps": 72479, "tokens": 24683479040, "loss": 4.642429769039154, "l_mix": 2.8801198601722717, "l_a": 2.955678313970566, "l_c": 2.9186878204345703, "z_loss": 37.13322353363037, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11865409463644028, "tokens_per_second": 1445466.8434197346, "eta_seconds": 9212.519105935096, "data_wait_seconds": 1.6301870346069336e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 47090, "total_steps": 72479, "tokens": 24688721920, "loss": 4.485410928726196, "l_mix": 2.7802669405937195, "l_a": 2.8661534786224365, "l_c": 2.817659914493561, "z_loss": 37.66439437866211, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.10570353269577026, "tokens_per_second": 1339606.811315056, "eta_seconds": 9936.608204412461, "data_wait_seconds": 1.8596649169921875e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 47100, "total_steps": 72479, "tokens": 24693964800, "loss": 4.454736948013306, "l_mix": 2.7607629597187042, "l_a": 2.847892314195633, "l_c": 2.798687666654587, "z_loss": 37.99630355834961, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11086033284664154, "tokens_per_second": 1430485.3935923649, "eta_seconds": 9301.671454739571, "data_wait_seconds": 2.250075340270996e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.18760109, "gate_a2b/L1_absmax": 0.71694154, "gate_a2b/L1_rms": 0.19931297, "gate_a2b/L3_absmean": 0.18347834, "gate_a2b/L3_absmax": 0.56002808, "gate_a2b/L3_rms": 0.18959862, "gate_a2b/L5_absmean": 0.28057107, "gate_a2b/L5_absmax": 0.73608744, "gate_a2b/L5_rms": 0.28653076, "gate_c2b/L1_absmean": 0.02070372, "gate_c2b/L1_absmax": 0.63029885, "gate_c2b/L1_rms": 0.06624299, "gate_c2b/L3_absmean": 0.18826051, "gate_c2b/L3_absmax": 0.42542854, "gate_c2b/L3_rms": 0.19222884, "gate_c2b/L5_absmean": 0.53521484, "gate_c2b/L5_absmax": 0.75354159, "gate_c2b/L5_rms": 0.54652762, "gate_b2a/L7_absmean": 0.05049412, "gate_b2a/L7_absmax": 0.32365373, "gate_b2a/L7_rms": 0.0560426, "gate_b2a/L11_absmean": 0.08209307, "gate_b2a/L11_absmax": 0.59709352, "gate_b2a/L11_rms": 0.09098679, "gate_b2a/L14_absmean": 0.21203816, "gate_b2a/L14_absmax": 0.78683078, "gate_b2a/L14_rms": 0.22065492, "gate_b2c/L3_absmean": 0.03653789, "gate_b2c/L3_absmax": 0.56985825, "gate_b2c/L3_rms": 0.04774141, "gate_b2c/L5_absmean": 0.03933221, "gate_b2c/L5_absmax": 0.58176416, "gate_b2c/L5_rms": 0.04968813, "gate_b2c/L6_absmean": 0.05670938, "gate_b2c/L6_absmax": 0.37572297, "gate_b2c/L6_rms": 0.06347454}, "streams": {"cos_a2b/round0": 0.366913, "cos_c2b/round0": 0.30616, "cos_b2a/round0": 0.149328, "cos_b2c/round0": 0.13361, "rms_a/round0": 5.823698, "rms_b/round0": 10.528157, "rms_c/round0": 4.097012, "cos_a2b/round1": 0.412748, "cos_c2b/round1": 0.301834, "cos_b2a/round1": 0.24901, "cos_b2c/round1": 0.113646, "rms_a/round1": 11.170544, "rms_b/round1": 24.343945, "rms_c/round1": 8.50043, "cos_a2b/round2": 0.53704, "cos_c2b/round2": 0.526414, "cos_b2a/round2": 0.103373, "cos_b2c/round2": 0.179894, "rms_a/round2": 30.681505, "rms_b/round2": 64.881607, "rms_c/round2": 16.063095}, "alpha_mean": 0.42683314538974315, "alpha_hist": [0.01027, 0.07011, 0.19191, 0.21596, 0.17966, 0.14073, 0.10965, 0.06909, 0.01244, 0.00018], "alpha_by_type": {"special": {"share": 0.00095, "mean": 0.57195, "std": 0.12629}, "whitespace": {"share": 0.02797, "mean": 0.52597, "std": 0.13247}, "digit": {"share": 0.04524, "mean": 0.44841, "std": 0.11775}, "word_start": {"share": 0.57584, "mean": 0.43923, "std": 0.16912}, "word_cont": {"share": 0.24731, "mean": 0.31571, "std": 0.12453}, "punct": {"share": 0.10269, "mean": 0.58708, "std": 0.16686}}, "alpha_p05": 0.18477, "alpha_p50": 0.42203, "alpha_p95": 0.72331}
{"step": 47110, "total_steps": 72479, "tokens": 24699207680, "loss": 4.480266988277435, "l_mix": 2.777120441198349, "l_a": 2.8634919226169586, "l_c": 2.813662827014923, "z_loss": 37.683316230773926, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12415546178817749, "tokens_per_second": 1331690.542100718, "eta_seconds": 9987.802609920502, "data_wait_seconds": 1.811981201171875e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 47120, "total_steps": 72479, "tokens": 24704450560, "loss": 4.406773328781128, "l_mix": 2.7309942543506622, "l_a": 2.8184844255447388, "l_c": 2.7674457132816315, "z_loss": 37.70227384567261, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13681118190288544, "tokens_per_second": 1407027.0664624504, "eta_seconds": 9449.298957288265, "data_wait_seconds": 2.0623207092285156e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 47130, "total_steps": 72479, "tokens": 24709693440, "loss": 4.614670276641846, "l_mix": 2.860268861055374, "l_a": 2.946532219648361, "l_c": 2.901472181081772, "z_loss": 37.457308769226074, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17226572334766388, "tokens_per_second": 1470835.5297881512, "eta_seconds": 9035.800565624237, "data_wait_seconds": 3.7550926208496094e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 47140, "total_steps": 72479, "tokens": 24714936320, "loss": 2.8201495110988617, "l_mix": 1.7463861852884293, "l_a": 1.8095435500144958, "l_c": 1.7696673572063446, "z_loss": 47.123281478881836, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16757500171661377, "tokens_per_second": 1268281.6076496125, "eta_seconds": 10474.75067987442, "data_wait_seconds": 2.092123031616211e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 47150, "total_steps": 72479, "tokens": 24720179200, "loss": 3.1602485477924347, "l_mix": 1.9557055085897446, "l_a": 2.0284135341644287, "l_c": 1.9867298156023026, "z_loss": 46.09733581542969, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17595452070236206, "tokens_per_second": 1395519.2663720474, "eta_seconds": 9515.949418973923, "data_wait_seconds": 3.4689903259277344e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 47160, "total_steps": 72479, "tokens": 24725422080, "loss": 4.461690664291382, "l_mix": 2.7662722766399384, "l_a": 2.851016581058502, "l_c": 2.800377994775772, "z_loss": 37.41090631484985, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18845561146736145, "tokens_per_second": 1398308.5640451298, "eta_seconds": 9493.217887187004, "data_wait_seconds": 1.5974044799804688e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 47170, "total_steps": 72479, "tokens": 24730664960, "loss": 4.433221399784088, "l_mix": 2.7483296990394592, "l_a": 2.8336411118507385, "l_c": 2.7826642096042633, "z_loss": 37.06187915802002, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13469462096691132, "tokens_per_second": 1265032.0957810734, "eta_seconds": 10489.223977994918, "data_wait_seconds": 0.03037145733833313, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 47180, "total_steps": 72479, "tokens": 24735907840, "loss": 4.480903685092926, "l_mix": 2.7769171595573425, "l_a": 2.862942785024643, "l_c": 2.817012459039688, "z_loss": 38.739519119262695, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15992537140846252, "tokens_per_second": 1484231.8773269341, "eta_seconds": 8936.583504652977, "data_wait_seconds": 2.2202730178833008e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 47190, "total_steps": 72479, "tokens": 24741150720, "loss": 4.507270097732544, "l_mix": 2.794011563062668, "l_a": 2.8787676095962524, "l_c": 2.8320937156677246, "z_loss": 37.66971969604492, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11968594789505005, "tokens_per_second": 1448398.917349396, "eta_seconds": 9154.052155923844, "data_wait_seconds": 1.609325408935547e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 47200, "total_steps": 72479, "tokens": 24746393600, "loss": 4.494040310382843, "l_mix": 2.785902053117752, "l_a": 2.8728696405887604, "l_c": 2.8209242820739746, "z_loss": 37.10916757583618, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12333925068378448, "tokens_per_second": 1189840.4475743934, "eta_seconds": 11138.868559241295, "data_wait_seconds": 3.316998481750488e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.18746555, "gate_a2b/L1_absmax": 0.71518666, "gate_a2b/L1_rms": 0.1991795, "gate_a2b/L3_absmean": 0.18316464, "gate_a2b/L3_absmax": 0.55943722, "gate_a2b/L3_rms": 0.18933621, "gate_a2b/L5_absmean": 0.28007111, "gate_a2b/L5_absmax": 0.73580688, "gate_a2b/L5_rms": 0.28610247, "gate_c2b/L1_absmean": 0.02024649, "gate_c2b/L1_absmax": 0.62898833, "gate_c2b/L1_rms": 0.06613792, "gate_c2b/L3_absmean": 0.18794428, "gate_c2b/L3_absmax": 0.42701331, "gate_c2b/L3_rms": 0.19189456, "gate_c2b/L5_absmean": 0.53433114, "gate_c2b/L5_absmax": 0.74837422, "gate_c2b/L5_rms": 0.54559118, "gate_b2a/L7_absmean": 0.05049757, "gate_b2a/L7_absmax": 0.32357776, "gate_b2a/L7_rms": 0.05606792, "gate_b2a/L11_absmean": 0.08235946, "gate_b2a/L11_absmax": 0.59737223, "gate_b2a/L11_rms": 0.09119391, "gate_b2a/L14_absmean": 0.21178246, "gate_b2a/L14_absmax": 0.78642112, "gate_b2a/L14_rms": 0.22040442, "gate_b2c/L3_absmean": 0.03664347, "gate_b2c/L3_absmax": 0.56889433, "gate_b2c/L3_rms": 0.0477691, "gate_b2c/L5_absmean": 0.03933119, "gate_b2c/L5_absmax": 0.58018237, "gate_b2c/L5_rms": 0.04963916, "gate_b2c/L6_absmean": 0.05654259, "gate_b2c/L6_absmax": 0.37472707, "gate_b2c/L6_rms": 0.06329242}, "streams": {"cos_a2b/round0": 0.366151, "cos_c2b/round0": 0.302427, "cos_b2a/round0": 0.147622, "cos_b2c/round0": 0.133358, "rms_a/round0": 5.863638, "rms_b/round0": 10.5533, "rms_c/round0": 4.098266, "cos_a2b/round1": 0.411372, "cos_c2b/round1": 0.298358, "cos_b2a/round1": 0.253804, "cos_b2c/round1": 0.112113, "rms_a/round1": 11.137452, "rms_b/round1": 24.271271, "rms_c/round1": 8.429202, "cos_a2b/round2": 0.536741, "cos_c2b/round2": 0.52999, "cos_b2a/round2": 0.092283, "cos_b2c/round2": 0.180396, "rms_a/round2": 30.642866, "rms_b/round2": 64.870148, "rms_c/round2": 16.046188}, "alpha_mean": 0.42756734532717644, "alpha_hist": [0.01016, 0.08289, 0.19127, 0.1987, 0.17166, 0.14149, 0.11748, 0.07672, 0.00955, 8e-05], "alpha_by_type": {"special": {"share": 0.00087, "mean": 0.5666, "std": 0.09332}, "whitespace": {"share": 0.02597, "mean": 0.52411, "std": 0.12401}, "digit": {"share": 0.04674, "mean": 0.46346, "std": 0.11384}, "word_start": {"share": 0.54849, "mean": 0.45126, "std": 0.17444}, "word_cont": {"share": 0.27534, "mean": 0.30568, "std": 0.12355}, "punct": {"share": 0.10258, "mean": 0.5861, "std": 0.16144}}, "alpha_p05": 0.17413, "alpha_p50": 0.39794, "alpha_p95": 0.7299}
{"step": 47210, "total_steps": 72479, "tokens": 24751636480, "loss": 4.451972246170044, "l_mix": 2.759558528661728, "l_a": 2.8453680276870728, "l_c": 2.796010911464691, "z_loss": 36.391531467437744, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1350916177034378, "tokens_per_second": 1473848.1221617498, "eta_seconds": 8988.872918987274, "data_wait_seconds": 1.6301870346069336e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 47220, "total_steps": 72479, "tokens": 24756879360, "loss": 4.476221740245819, "l_mix": 2.774414598941803, "l_a": 2.860863298177719, "l_c": 2.8118268251419067, "z_loss": 37.49754333496094, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.10886551439762115, "tokens_per_second": 1362012.3327717078, "eta_seconds": 9723.106225514413, "data_wait_seconds": 2.6732683181762695e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 47230, "total_steps": 72479, "tokens": 24762122240, "loss": 4.430915474891663, "l_mix": 2.74556165933609, "l_a": 2.834503173828125, "l_c": 2.7833423912525177, "z_loss": 38.07079792022705, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1285559982061386, "tokens_per_second": 1430724.0249413094, "eta_seconds": 9252.481597590446, "data_wait_seconds": 1.576542854309082e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 47240, "total_steps": 72479, "tokens": 24767365120, "loss": 4.5692333579063416, "l_mix": 2.8327242136001587, "l_a": 2.91620272397995, "l_c": 2.8721609711647034, "z_loss": 37.26655197143555, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1428220570087433, "tokens_per_second": 1310018.3445618015, "eta_seconds": 10101.007277441024, "data_wait_seconds": 1.749396324157715e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 47250, "total_steps": 72479, "tokens": 24772608000, "loss": 2.9535921812057495, "l_mix": 1.8279239684343338, "l_a": 1.8954927027225494, "l_c": 1.856734275817871, "z_loss": 44.233266830444336, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17482581734657288, "tokens_per_second": 1378095.8373435456, "eta_seconds": 9598.216316723823, "data_wait_seconds": 3.4362077713012695e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 47260, "total_steps": 72479, "tokens": 24777850880, "loss": 3.005293160676956, "l_mix": 1.8608673512935638, "l_a": 1.9266733676195145, "l_c": 1.8880791515111923, "z_loss": 46.12088203430176, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19051691889762878, "tokens_per_second": 1431674.5001363624, "eta_seconds": 9235.352777981758, "data_wait_seconds": 3.272294998168945e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 47270, "total_steps": 72479, "tokens": 24783093760, "loss": 4.486587882041931, "l_mix": 2.7808389961719513, "l_a": 2.867592930793762, "l_c": 2.8182364106178284, "z_loss": 37.197861194610596, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11295965313911438, "tokens_per_second": 1454501.1874676105, "eta_seconds": 9086.810176491737, "data_wait_seconds": 3.591179847717285e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 47280, "total_steps": 72479, "tokens": 24788336640, "loss": 4.549451053142548, "l_mix": 2.819584906101227, "l_a": 2.9097341299057007, "l_c": 2.856486201286316, "z_loss": 36.85190153121948, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1446457803249359, "tokens_per_second": 1391550.483166024, "eta_seconds": 9494.109967136385, "data_wait_seconds": 2.8133392333984375e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 47290, "total_steps": 72479, "tokens": 24793579520, "loss": 4.553145945072174, "l_mix": 2.8224314749240875, "l_a": 2.9108957946300507, "l_c": 2.858152061700821, "z_loss": 37.544092655181885, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17154303193092346, "tokens_per_second": 1410266.9914064407, "eta_seconds": 9364.390227150918, "data_wait_seconds": 1.9371509552001953e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 47300, "total_steps": 72479, "tokens": 24798822400, "loss": 4.3679457902908325, "l_mix": 2.7063011825084686, "l_a": 2.792415678501129, "l_c": 2.7463995218276978, "z_loss": 37.76223182678223, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1803087741136551, "tokens_per_second": 1340399.4345341881, "eta_seconds": 9848.59230158329, "data_wait_seconds": 1.558661460876465e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.18680964, "gate_a2b/L1_absmax": 0.71866179, "gate_a2b/L1_rms": 0.19870059, "gate_a2b/L3_absmean": 0.18284857, "gate_a2b/L3_absmax": 0.55880839, "gate_a2b/L3_rms": 0.1890427, "gate_a2b/L5_absmean": 0.27938625, "gate_a2b/L5_absmax": 0.73573327, "gate_a2b/L5_rms": 0.28537849, "gate_c2b/L1_absmean": 0.02051761, "gate_c2b/L1_absmax": 0.63043416, "gate_c2b/L1_rms": 0.06628183, "gate_c2b/L3_absmean": 0.18776834, "gate_c2b/L3_absmax": 0.41963398, "gate_c2b/L3_rms": 0.19172579, "gate_c2b/L5_absmean": 0.53416824, "gate_c2b/L5_absmax": 0.75150985, "gate_c2b/L5_rms": 0.54544401, "gate_b2a/L7_absmean": 0.05091266, "gate_b2a/L7_absmax": 0.32376662, "gate_b2a/L7_rms": 0.05647791, "gate_b2a/L11_absmean": 0.08179903, "gate_b2a/L11_absmax": 0.59881395, "gate_b2a/L11_rms": 0.09078208, "gate_b2a/L14_absmean": 0.21199125, "gate_b2a/L14_absmax": 0.78803819, "gate_b2a/L14_rms": 0.22059688, "gate_b2c/L3_absmean": 0.0366888, "gate_b2c/L3_absmax": 0.56910491, "gate_b2c/L3_rms": 0.04779292, "gate_b2c/L5_absmean": 0.03906621, "gate_b2c/L5_absmax": 0.57593155, "gate_b2c/L5_rms": 0.04932733, "gate_b2c/L6_absmean": 0.05621724, "gate_b2c/L6_absmax": 0.37473693, "gate_b2c/L6_rms": 0.06296935}, "streams": {"cos_a2b/round0": 0.364232, "cos_c2b/round0": 0.295651, "cos_b2a/round0": 0.140803, "cos_b2c/round0": 0.127265, "rms_a/round0": 5.823374, "rms_b/round0": 10.484163, "rms_c/round0": 4.133101, "cos_a2b/round1": 0.405974, "cos_c2b/round1": 0.293766, "cos_b2a/round1": 0.245919, "cos_b2c/round1": 0.100441, "rms_a/round1": 11.081909, "rms_b/round1": 24.564165, "rms_c/round1": 8.402762, "cos_a2b/round2": 0.531043, "cos_c2b/round2": 0.527166, "cos_b2a/round2": 0.087116, "cos_b2c/round2": 0.176421, "rms_a/round2": 30.77343, "rms_b/round2": 66.074898, "rms_c/round2": 15.442743}, "alpha_mean": 0.4078388928716805, "alpha_hist": [0.00761, 0.07648, 0.22406, 0.23178, 0.17067, 0.12975, 0.09874, 0.05539, 0.00545, 8e-05], "alpha_by_type": {"special": {"share": 0.00026, "mean": 0.52576, "std": 0.08891}, "whitespace": {"share": 0.02037, "mean": 0.48574, "std": 0.13426}, "digit": {"share": 0.04546, "mean": 0.47181, "std": 0.11469}, "word_start": {"share": 0.52422, "mean": 0.42277, "std": 0.16667}, "word_cont": {"share": 0.28387, "mean": 0.30722, "std": 0.10882}, "punct": {"share": 0.12582, "mean": 0.53667, "std": 0.16875}}, "alpha_p05": 0.18126, "alpha_p50": 0.39467, "alpha_p95": 0.72174}
{"step": 47310, "total_steps": 72479, "tokens": 24804065280, "loss": 4.447147727012634, "l_mix": 2.7562031745910645, "l_a": 2.8409920036792755, "l_c": 2.795489579439163, "z_loss": 37.53931188583374, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1789843887090683, "tokens_per_second": 1454475.4049434313, "eta_seconds": 9072.552637982368, "data_wait_seconds": 1.6629695892333984e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 47320, "total_steps": 72479, "tokens": 24809308160, "loss": 4.691948235034943, "l_mix": 2.9109884798526764, "l_a": 2.987074166536331, "l_c": 2.949457973241806, "z_loss": 37.231741428375244, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11404174566268921, "tokens_per_second": 1466239.936160895, "eta_seconds": 8996.182320976257, "data_wait_seconds": 1.6450881958007812e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 47330, "total_steps": 72479, "tokens": 24814551040, "loss": 4.610460877418518, "l_mix": 2.8593743443489075, "l_a": 2.9395677745342255, "l_c": 2.8973871171474457, "z_loss": 37.09371519088745, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12085981667041779, "tokens_per_second": 1449410.7641055868, "eta_seconds": 9097.020139861108, "data_wait_seconds": 2.372264862060547e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 47340, "total_steps": 72479, "tokens": 24819793920, "loss": 4.662419855594635, "l_mix": 2.8926579654216766, "l_a": 2.967671900987625, "l_c": 2.931534469127655, "z_loss": 37.510440826416016, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12691031396389008, "tokens_per_second": 1308103.1257099984, "eta_seconds": 10075.716335320472, "data_wait_seconds": 2.753734588623047e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 47350, "total_steps": 72479, "tokens": 24825036800, "loss": 4.573959529399872, "l_mix": 2.8365062475204468, "l_a": 2.9155429303646088, "l_c": 2.87596794962883, "z_loss": 38.08741807937622, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14644354581832886, "tokens_per_second": 1390068.5523180175, "eta_seconds": 9477.829802012444, "data_wait_seconds": 2.473592758178711e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 47360, "total_steps": 72479, "tokens": 24830279680, "loss": 4.559026598930359, "l_mix": 2.827714681625366, "l_a": 2.904620200395584, "l_c": 2.866419106721878, "z_loss": 38.153446674346924, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16120979189872742, "tokens_per_second": 1123376.5263644333, "eta_seconds": 11723.2200984478, "data_wait_seconds": 2.5987625122070312e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 47370, "total_steps": 72479, "tokens": 24835522560, "loss": 2.8587846159934998, "l_mix": 1.7692888975143433, "l_a": 1.8339934796094894, "l_c": 1.7976587563753128, "z_loss": 46.167436599731445, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2149968147277832, "tokens_per_second": 1450613.8528872782, "eta_seconds": 9075.01839017868, "data_wait_seconds": 2.536177635192871e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 47380, "total_steps": 72479, "tokens": 24840765440, "loss": 4.541110575199127, "l_mix": 2.8149063885211945, "l_a": 2.9030084908008575, "l_c": 2.851005107164383, "z_loss": 37.65010976791382, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15614309906959534, "tokens_per_second": 1355117.22690856, "eta_seconds": 9710.676132440567, "data_wait_seconds": 1.6391277313232422e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 47390, "total_steps": 72479, "tokens": 24846008320, "loss": 4.4413942694664, "l_mix": 2.7520123720169067, "l_a": 2.8402750492095947, "l_c": 2.7909977734088898, "z_loss": 38.035393714904785, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16162718832492828, "tokens_per_second": 1450652.5133242244, "eta_seconds": 9067.548231697083, "data_wait_seconds": 3.0159950256347656e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 47400, "total_steps": 72479, "tokens": 24851251200, "loss": 4.443322539329529, "l_mix": 2.754081070423126, "l_a": 2.8412258327007294, "l_c": 2.789578437805176, "z_loss": 37.613067626953125, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12403880804777145, "tokens_per_second": 1307701.343172884, "eta_seconds": 10054.756631278991, "data_wait_seconds": 2.0802021026611328e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.18668145, "gate_a2b/L1_absmax": 0.71614367, "gate_a2b/L1_rms": 0.19835493, "gate_a2b/L3_absmean": 0.18222633, "gate_a2b/L3_absmax": 0.55758828, "gate_a2b/L3_rms": 0.18836397, "gate_a2b/L5_absmean": 0.27872577, "gate_a2b/L5_absmax": 0.73452151, "gate_a2b/L5_rms": 0.28475291, "gate_c2b/L1_absmean": 0.02033176, "gate_c2b/L1_absmax": 0.62835139, "gate_c2b/L1_rms": 0.06630242, "gate_c2b/L3_absmean": 0.18734312, "gate_c2b/L3_absmax": 0.42343348, "gate_c2b/L3_rms": 0.19130118, "gate_c2b/L5_absmean": 0.53339517, "gate_c2b/L5_absmax": 0.75183851, "gate_c2b/L5_rms": 0.54459453, "gate_b2a/L7_absmean": 0.05095821, "gate_b2a/L7_absmax": 0.32341552, "gate_b2a/L7_rms": 0.05651152, "gate_b2a/L11_absmean": 0.08205354, "gate_b2a/L11_absmax": 0.59731764, "gate_b2a/L11_rms": 0.09097094, "gate_b2a/L14_absmean": 0.21168128, "gate_b2a/L14_absmax": 0.7864849, "gate_b2a/L14_rms": 0.22031412, "gate_b2c/L3_absmean": 0.03660467, "gate_b2c/L3_absmax": 0.56919867, "gate_b2c/L3_rms": 0.04772953, "gate_b2c/L5_absmean": 0.03922283, "gate_b2c/L5_absmax": 0.57682741, "gate_b2c/L5_rms": 0.04947256, "gate_b2c/L6_absmean": 0.05629364, "gate_b2c/L6_absmax": 0.37196285, "gate_b2c/L6_rms": 0.0630219}, "streams": {"cos_a2b/round0": 0.363753, "cos_c2b/round0": 0.300868, "cos_b2a/round0": 0.152091, "cos_b2c/round0": 0.133873, "rms_a/round0": 5.859441, "rms_b/round0": 10.681648, "rms_c/round0": 4.120946, "cos_a2b/round1": 0.410266, "cos_c2b/round1": 0.304765, "cos_b2a/round1": 0.254575, "cos_b2c/round1": 0.114498, "rms_a/round1": 11.148156, "rms_b/round1": 24.358435, "rms_c/round1": 8.442813, "cos_a2b/round2": 0.545631, "cos_c2b/round2": 0.52205, "cos_b2a/round2": 0.101459, "cos_b2c/round2": 0.182713, "rms_a/round2": 30.949173, "rms_b/round2": 64.292809, "rms_c/round2": 15.83347}, "alpha_mean": 0.430166177930964, "alpha_hist": [0.00905, 0.07169, 0.18724, 0.20056, 0.18575, 0.15236, 0.1158, 0.06635, 0.01111, 0.00011], "alpha_by_type": {"special": {"share": 0.00085, "mean": 0.56064, "std": 0.10141}, "whitespace": {"share": 0.02452, "mean": 0.51869, "std": 0.12445}, "digit": {"share": 0.03119, "mean": 0.49125, "std": 0.12296}, "word_start": {"share": 0.55962, "mean": 0.44959, "std": 0.1693}, "word_cont": {"share": 0.2634, "mean": 0.31771, "std": 0.12701}, "punct": {"share": 0.12042, "mean": 0.5511, "std": 0.16663}}, "alpha_p05": 0.18713, "alpha_p50": 0.40404, "alpha_p95": 0.708}
{"step": 47410, "total_steps": 72479, "tokens": 24856494080, "loss": 4.502663612365723, "l_mix": 2.7915011048316956, "l_a": 2.8788412511348724, "l_c": 2.825033575296402, "z_loss": 38.15793418884277, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.21519187092781067, "tokens_per_second": 1425445.241475908, "eta_seconds": 9220.540705156327, "data_wait_seconds": 1.6897916793823242e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 47420, "total_steps": 72479, "tokens": 24861736960, "loss": 4.518146455287933, "l_mix": 2.800358474254608, "l_a": 2.8880701065063477, "l_c": 2.8378894329071045, "z_loss": 37.16485118865967, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14946314692497253, "tokens_per_second": 1427089.7285683274, "eta_seconds": 9206.241716265678, "data_wait_seconds": 2.8580427169799805e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 47430, "total_steps": 72479, "tokens": 24866979840, "loss": 3.1551889777183533, "l_mix": 1.9417513459920883, "l_a": 2.0544494688510895, "l_c": 1.9903424084186554, "z_loss": 41.15020418167114, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14418935775756836, "tokens_per_second": 1317151.7978045426, "eta_seconds": 9970.673185801506, "data_wait_seconds": 2.5272369384765625e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 47440, "total_steps": 72479, "tokens": 24872222720, "loss": 3.1316177546977997, "l_mix": 1.9270304292440414, "l_a": 2.039740279316902, "l_c": 1.9755507111549377, "z_loss": 41.2012882232666, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14339646697044373, "tokens_per_second": 1406388.2965377057, "eta_seconds": 9334.297835326193, "data_wait_seconds": 2.4378299713134766e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 47450, "total_steps": 72479, "tokens": 24877465600, "loss": 4.626305878162384, "l_mix": 2.869740813970566, "l_a": 2.947096437215805, "l_c": 2.908120036125183, "z_loss": 37.830745697021484, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.0997556522488594, "tokens_per_second": 1398361.5595375549, "eta_seconds": 9384.128348278999, "data_wait_seconds": 2.530217170715332e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 47460, "total_steps": 72479, "tokens": 24882708480, "loss": 4.606620609760284, "l_mix": 2.8572623133659363, "l_a": 2.934068351984024, "l_c": 2.8971257507801056, "z_loss": 37.912883281707764, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1304829865694046, "tokens_per_second": 1424595.392659751, "eta_seconds": 9207.639965415001, "data_wait_seconds": 3.2007694244384766e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 47470, "total_steps": 72479, "tokens": 24887951360, "loss": 4.590162456035614, "l_mix": 2.8471908569335938, "l_a": 2.921170860528946, "l_c": 2.888734132051468, "z_loss": 37.81147766113281, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14867155253887177, "tokens_per_second": 1310277.5712656248, "eta_seconds": 10006.977818703652, "data_wait_seconds": 2.7835369110107422e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 47480, "total_steps": 72479, "tokens": 24893194240, "loss": 2.9163446724414825, "l_mix": 1.8048850148916245, "l_a": 1.8752027601003647, "l_c": 1.8296628147363663, "z_loss": 45.85305738449097, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13811592757701874, "tokens_per_second": 1381631.4422766583, "eta_seconds": 9486.37625849247, "data_wait_seconds": 2.8878450393676758e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 47490, "total_steps": 72479, "tokens": 24898437120, "loss": 4.572327971458435, "l_mix": 2.83454293012619, "l_a": 2.92178937792778, "l_c": 2.870827227830887, "z_loss": 37.34330415725708, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13002827763557434, "tokens_per_second": 1242206.075657741, "eta_seconds": 10546.907706165313, "data_wait_seconds": 2.1249055862426758e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 47500, "total_steps": 72479, "tokens": 24903680000, "loss": 4.673206210136414, "l_mix": 2.8978593349456787, "l_a": 2.979339689016342, "l_c": 2.938482940196991, "z_loss": 36.29355049133301, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13267163932323456, "tokens_per_second": 1288870.1789435772, "eta_seconds": 10160.984531998634, "data_wait_seconds": 1.7136335372924805e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.1869821, "gate_a2b/L1_absmax": 0.71502024, "gate_a2b/L1_rms": 0.19859882, "gate_a2b/L3_absmean": 0.18228884, "gate_a2b/L3_absmax": 0.5578059, "gate_a2b/L3_rms": 0.18842922, "gate_a2b/L5_absmean": 0.27804348, "gate_a2b/L5_absmax": 0.73518568, "gate_a2b/L5_rms": 0.28407019, "gate_c2b/L1_absmean": 0.01992575, "gate_c2b/L1_absmax": 0.62941408, "gate_c2b/L1_rms": 0.06599525, "gate_c2b/L3_absmean": 0.18690698, "gate_c2b/L3_absmax": 0.42556068, "gate_c2b/L3_rms": 0.19092551, "gate_c2b/L5_absmean": 0.53288907, "gate_c2b/L5_absmax": 0.75391936, "gate_c2b/L5_rms": 0.5440644, "gate_b2a/L7_absmean": 0.05099539, "gate_b2a/L7_absmax": 0.32344452, "gate_b2a/L7_rms": 0.05653749, "gate_b2a/L11_absmean": 0.08232802, "gate_b2a/L11_absmax": 0.59961373, "gate_b2a/L11_rms": 0.09125414, "gate_b2a/L14_absmean": 0.21192653, "gate_b2a/L14_absmax": 0.78486049, "gate_b2a/L14_rms": 0.22050093, "gate_b2c/L3_absmean": 0.03667644, "gate_b2c/L3_absmax": 0.56875443, "gate_b2c/L3_rms": 0.04777564, "gate_b2c/L5_absmean": 0.03932006, "gate_b2c/L5_absmax": 0.57655388, "gate_b2c/L5_rms": 0.04953275, "gate_b2c/L6_absmean": 0.05640056, "gate_b2c/L6_absmax": 0.37220573, "gate_b2c/L6_rms": 0.06312008}, "streams": {"cos_a2b/round0": 0.365668, "cos_c2b/round0": 0.303908, "cos_b2a/round0": 0.153636, "cos_b2c/round0": 0.132954, "rms_a/round0": 5.922724, "rms_b/round0": 10.747789, "rms_c/round0": 4.12745, "cos_a2b/round1": 0.416544, "cos_c2b/round1": 0.305401, "cos_b2a/round1": 0.253962, "cos_b2c/round1": 0.113327, "rms_a/round1": 11.391665, "rms_b/round1": 24.482731, "rms_c/round1": 8.564445, "cos_a2b/round2": 0.543956, "cos_c2b/round2": 0.528089, "cos_b2a/round2": 0.101929, "cos_b2c/round2": 0.18108, "rms_a/round2": 31.292053, "rms_b/round2": 65.266922, "rms_c/round2": 16.268103}, "alpha_mean": 0.4245255652400459, "alpha_hist": [0.00763, 0.06801, 0.18584, 0.21825, 0.19301, 0.15401, 0.10941, 0.05775, 0.00606, 5e-05], "alpha_by_type": {"special": {"share": 0.00107, "mean": 0.55157, "std": 0.10538}, "whitespace": {"share": 0.02748, "mean": 0.51293, "std": 0.12258}, "digit": {"share": 0.03493, "mean": 0.4298, "std": 0.12179}, "word_start": {"share": 0.56677, "mean": 0.44384, "std": 0.16166}, "word_cont": {"share": 0.25623, "mean": 0.3158, "std": 0.11969}, "punct": {"share": 0.11353, "mean": 0.54929, "std": 0.15946}}, "alpha_p05": 0.17413, "alpha_p50": 0.40592, "alpha_p95": 0.70882}
{"step": 47510, "total_steps": 72479, "tokens": 24908922880, "loss": 4.55639111995697, "l_mix": 2.8248948454856873, "l_a": 2.9086268544197083, "l_c": 2.8630272150039673, "z_loss": 37.661536693573, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12759380042552948, "tokens_per_second": 1468488.536295691, "eta_seconds": 8914.572193408012, "data_wait_seconds": 1.7255544662475586e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 47520, "total_steps": 72479, "tokens": 24914165760, "loss": 4.378998279571533, "l_mix": 2.7139606177806854, "l_a": 2.7991079688072205, "l_c": 2.7510175704956055, "z_loss": 38.26416254043579, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1316344141960144, "tokens_per_second": 1292444.5896261465, "eta_seconds": 10124.769987845422, "data_wait_seconds": 1.952052116394043e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 47530, "total_steps": 72479, "tokens": 24919408640, "loss": 4.366614282131195, "l_mix": 2.7060936093330383, "l_a": 2.792054295539856, "l_c": 2.7430143654346466, "z_loss": 37.35999155044556, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11858734488487244, "tokens_per_second": 1423345.3190505798, "eta_seconds": 9189.94226975441, "data_wait_seconds": 1.9311904907226562e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 47540, "total_steps": 72479, "tokens": 24924651520, "loss": 3.0877718031406403, "l_mix": 1.9114002734422684, "l_a": 1.980583056807518, "l_c": 1.940655305981636, "z_loss": 46.017897605895996, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17257355153560638, "tokens_per_second": 1359652.0116311775, "eta_seconds": 9616.591833901406, "data_wait_seconds": 3.594160079956055e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 47550, "total_steps": 72479, "tokens": 24929894400, "loss": 3.1713846921920776, "l_mix": 1.9513765424489975, "l_a": 2.0679363906383514, "l_c": 1.9987571388483047, "z_loss": 39.80595016479492, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18691286444664001, "tokens_per_second": 1376600.613417764, "eta_seconds": 9494.384518361092, "data_wait_seconds": 1.7970800399780273e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 47560, "total_steps": 72479, "tokens": 24935137280, "loss": 3.1239765882492065, "l_mix": 1.9217189699411392, "l_a": 2.0360897183418274, "l_c": 1.9714356362819672, "z_loss": 40.545183181762695, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17735427618026733, "tokens_per_second": 1462047.6039739882, "eta_seconds": 8935.91469695568, "data_wait_seconds": 2.92360782623291e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 47570, "total_steps": 72479, "tokens": 24940380160, "loss": 3.131819635629654, "l_mix": 1.9265935122966766, "l_a": 2.0400642305612564, "l_c": 1.9773560166358948, "z_loss": 40.738826751708984, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14729441702365875, "tokens_per_second": 1441778.9118375615, "eta_seconds": 9057.900406765939, "data_wait_seconds": 2.7358531951904297e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 47580, "total_steps": 72479, "tokens": 24945623040, "loss": 4.5921924114227295, "l_mix": 2.848393142223358, "l_a": 2.926234781742096, "l_c": 2.886429488658905, "z_loss": 37.97986602783203, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12723463773727417, "tokens_per_second": 1372287.1907437805, "eta_seconds": 9512.765986633302, "data_wait_seconds": 2.7924776077270508e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 47590, "total_steps": 72479, "tokens": 24950865920, "loss": 4.547399580478668, "l_mix": 2.820417582988739, "l_a": 2.8982552886009216, "l_c": 2.8583511412143707, "z_loss": 37.16239595413208, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14265979826450348, "tokens_per_second": 1419386.3897788737, "eta_seconds": 9193.412115240097, "data_wait_seconds": 4.246830940246582e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 47600, "total_steps": 72479, "tokens": 24956108800, "loss": 4.503400504589081, "l_mix": 2.791988104581833, "l_a": 2.8777845799922943, "l_c": 2.826923131942749, "z_loss": 37.233906269073486, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1174989566206932, "tokens_per_second": 1338950.7714911352, "eta_seconds": 9741.77798745632, "data_wait_seconds": 1.5348196029663086e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.18620379, "gate_a2b/L1_absmax": 0.7137956, "gate_a2b/L1_rms": 0.19788682, "gate_a2b/L3_absmean": 0.18206571, "gate_a2b/L3_absmax": 0.55494332, "gate_a2b/L3_rms": 0.18817081, "gate_a2b/L5_absmean": 0.27799997, "gate_a2b/L5_absmax": 0.73485041, "gate_a2b/L5_rms": 0.28396153, "gate_c2b/L1_absmean": 0.0206873, "gate_c2b/L1_absmax": 0.6296646, "gate_c2b/L1_rms": 0.06614412, "gate_c2b/L3_absmean": 0.18724093, "gate_c2b/L3_absmax": 0.42482352, "gate_c2b/L3_rms": 0.191184, "gate_c2b/L5_absmean": 0.53244537, "gate_c2b/L5_absmax": 0.75041169, "gate_c2b/L5_rms": 0.54364258, "gate_b2a/L7_absmean": 0.05137226, "gate_b2a/L7_absmax": 0.32308817, "gate_b2a/L7_rms": 0.05678508, "gate_b2a/L11_absmean": 0.08221515, "gate_b2a/L11_absmax": 0.59864497, "gate_b2a/L11_rms": 0.09113293, "gate_b2a/L14_absmean": 0.21215835, "gate_b2a/L14_absmax": 0.78349882, "gate_b2a/L14_rms": 0.22068243, "gate_b2c/L3_absmean": 0.03651264, "gate_b2c/L3_absmax": 0.56747061, "gate_b2c/L3_rms": 0.04761008, "gate_b2c/L5_absmean": 0.03904675, "gate_b2c/L5_absmax": 0.57780486, "gate_b2c/L5_rms": 0.04934332, "gate_b2c/L6_absmean": 0.05626874, "gate_b2c/L6_absmax": 0.37021372, "gate_b2c/L6_rms": 0.06297112}, "streams": {"cos_a2b/round0": 0.364021, "cos_c2b/round0": 0.301836, "cos_b2a/round0": 0.155542, "cos_b2c/round0": 0.134188, "rms_a/round0": 5.906526, "rms_b/round0": 10.710055, "rms_c/round0": 4.135854, "cos_a2b/round1": 0.417607, "cos_c2b/round1": 0.309, "cos_b2a/round1": 0.25732, "cos_b2c/round1": 0.114486, "rms_a/round1": 11.335293, "rms_b/round1": 24.371117, "rms_c/round1": 8.536975, "cos_a2b/round2": 0.550063, "cos_c2b/round2": 0.523472, "cos_b2a/round2": 0.099655, "cos_b2c/round2": 0.182821, "rms_a/round2": 31.167633, "rms_b/round2": 64.800423, "rms_c/round2": 16.279079}, "alpha_mean": 0.43849278799453373, "alpha_hist": [0.0081, 0.06267, 0.17194, 0.21107, 0.18367, 0.15672, 0.12427, 0.07254, 0.00897, 5e-05], "alpha_by_type": {"special": {"share": 0.00143, "mean": 0.57695, "std": 0.11334}, "whitespace": {"share": 0.02167, "mean": 0.52248, "std": 0.11864}, "digit": {"share": 0.02937, "mean": 0.4294, "std": 0.11859}, "word_start": {"share": 0.59026, "mean": 0.45934, "std": 0.16348}, "word_cont": {"share": 0.25491, "mean": 0.32281, "std": 0.12504}, "punct": {"share": 0.10236, "mean": 0.58926, "std": 0.16232}}, "alpha_p05": 0.18803, "alpha_p50": 0.41371, "alpha_p95": 0.73488}
{"step": 47610, "total_steps": 72479, "tokens": 24961351680, "loss": 4.3317776918411255, "l_mix": 2.6840240359306335, "l_a": 2.7715121507644653, "l_c": 2.7209998965263367, "z_loss": 37.82147932052612, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13619470596313477, "tokens_per_second": 1422656.6285697613, "eta_seconds": 9164.908812260628, "data_wait_seconds": 1.749396324157715e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 47620, "total_steps": 72479, "tokens": 24966594560, "loss": 4.414289087057114, "l_mix": 2.736097365617752, "l_a": 2.8213804066181183, "l_c": 2.7725917994976044, "z_loss": 37.07829284667969, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12167452275753021, "tokens_per_second": 1467173.201168602, "eta_seconds": 8883.25617017746, "data_wait_seconds": 2.1785497665405273e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 47630, "total_steps": 72479, "tokens": 24971837440, "loss": 4.3767412304878235, "l_mix": 2.7122032046318054, "l_a": 2.7978678047657013, "l_c": 2.7505919337272644, "z_loss": 38.022233963012695, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1277831792831421, "tokens_per_second": 1411661.4556765459, "eta_seconds": 9228.86465420723, "data_wait_seconds": 1.704692840576172e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 47640, "total_steps": 72479, "tokens": 24977080320, "loss": 4.388810753822327, "l_mix": 2.7200068831443787, "l_a": 2.8068171441555023, "l_c": 2.7558620870113373, "z_loss": 37.21998691558838, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13456077873706818, "tokens_per_second": 1456606.0230068455, "eta_seconds": 8940.502391386031, "data_wait_seconds": 1.609325408935547e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 47650, "total_steps": 72479, "tokens": 24982323200, "loss": 3.0896677672863007, "l_mix": 1.9127392172813416, "l_a": 1.9815844744443893, "l_c": 1.9415105879306793, "z_loss": 45.191399574279785, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14397700130939484, "tokens_per_second": 1473992.4560751207, "eta_seconds": 8831.48804347515, "data_wait_seconds": 2.7894973754882812e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 47660, "total_steps": 72479, "tokens": 24987566080, "loss": 3.172234833240509, "l_mix": 1.9506398290395737, "l_a": 2.069158837199211, "l_c": 2.0028243958950043, "z_loss": 42.311867237091064, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2204069048166275, "tokens_per_second": 1385333.234354767, "eta_seconds": 9392.905294775963, "data_wait_seconds": 3.579258918762207e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 47670, "total_steps": 72479, "tokens": 24992808960, "loss": 3.145557224750519, "l_mix": 1.9359086155891418, "l_a": 2.0467540323734283, "l_c": 1.9854078739881516, "z_loss": 39.76997470855713, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15650375187397003, "tokens_per_second": 1391316.4647446414, "eta_seconds": 9348.743669462203, "data_wait_seconds": 3.5703182220458984e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 47680, "total_steps": 72479, "tokens": 24998051840, "loss": 3.1659650206565857, "l_mix": 1.9475260078907013, "l_a": 2.0636754035949707, "l_c": 1.9977876991033554, "z_loss": 39.298551082611084, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.190046027302742, "tokens_per_second": 1378162.0811468794, "eta_seconds": 9434.171996068955, "data_wait_seconds": 2.1636486053466797e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 47690, "total_steps": 72479, "tokens": 25003294720, "loss": 3.149601072072983, "l_mix": 1.9379723817110062, "l_a": 2.054312825202942, "l_c": 1.9844492971897125, "z_loss": 41.825812339782715, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.21979756653308868, "tokens_per_second": 1362180.3969583372, "eta_seconds": 9541.008856844903, "data_wait_seconds": 2.962350845336914e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 47700, "total_steps": 72479, "tokens": 25008537600, "loss": 3.103611946105957, "l_mix": 1.9089945703744888, "l_a": 2.0181553065776825, "l_c": 1.963902473449707, "z_loss": 41.62963342666626, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2159171849489212, "tokens_per_second": 1365667.0194666665, "eta_seconds": 9512.811078262328, "data_wait_seconds": 2.384185791015625e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.18619321, "gate_a2b/L1_absmax": 0.71310264, "gate_a2b/L1_rms": 0.1978723, "gate_a2b/L3_absmean": 0.1817985, "gate_a2b/L3_absmax": 0.55791771, "gate_a2b/L3_rms": 0.18800974, "gate_a2b/L5_absmean": 0.27686808, "gate_a2b/L5_absmax": 0.73626119, "gate_a2b/L5_rms": 0.28290084, "gate_c2b/L1_absmean": 0.02112583, "gate_c2b/L1_absmax": 0.62728792, "gate_c2b/L1_rms": 0.06616605, "gate_c2b/L3_absmean": 0.18653013, "gate_c2b/L3_absmax": 0.42011133, "gate_c2b/L3_rms": 0.19053091, "gate_c2b/L5_absmean": 0.53189158, "gate_c2b/L5_absmax": 0.74461883, "gate_c2b/L5_rms": 0.54308736, "gate_b2a/L7_absmean": 0.0510573, "gate_b2a/L7_absmax": 0.32283705, "gate_b2a/L7_rms": 0.0566031, "gate_b2a/L11_absmean": 0.08238949, "gate_b2a/L11_absmax": 0.59670681, "gate_b2a/L11_rms": 0.09127433, "gate_b2a/L14_absmean": 0.21165207, "gate_b2a/L14_absmax": 0.78616506, "gate_b2a/L14_rms": 0.22030197, "gate_b2c/L3_absmean": 0.03628494, "gate_b2c/L3_absmax": 0.5668391, "gate_b2c/L3_rms": 0.04746448, "gate_b2c/L5_absmean": 0.03903591, "gate_b2c/L5_absmax": 0.57955617, "gate_b2c/L5_rms": 0.0493944, "gate_b2c/L6_absmean": 0.05602461, "gate_b2c/L6_absmax": 0.37059897, "gate_b2c/L6_rms": 0.06275374}, "streams": {"cos_a2b/round0": 0.362023, "cos_c2b/round0": 0.268398, "cos_b2a/round0": 0.134201, "cos_b2c/round0": 0.112479, "rms_a/round0": 6.180887, "rms_b/round0": 10.815573, "rms_c/round0": 4.218595, "cos_a2b/round1": 0.408931, "cos_c2b/round1": 0.298361, "cos_b2a/round1": 0.234711, "cos_b2c/round1": 0.115888, "rms_a/round1": 12.018837, "rms_b/round1": 25.723103, "rms_c/round1": 9.013041, "cos_a2b/round2": 0.513861, "cos_c2b/round2": 0.520056, "cos_b2a/round2": 0.085815, "cos_b2c/round2": 0.165918, "rms_a/round2": 31.709806, "rms_b/round2": 66.65641, "rms_c/round2": 17.194962}, "alpha_mean": 0.40378806540498147, "alpha_hist": [0.03302, 0.16788, 0.19595, 0.15491, 0.12257, 0.10529, 0.09534, 0.07855, 0.04369, 0.00281], "alpha_by_type": {"special": {"share": 0.00101, "mean": 0.62951, "std": 0.11844}, "whitespace": {"share": 0.03922, "mean": 0.61415, "std": 0.17415}, "digit": {"share": 0.01062, "mean": 0.4072, "std": 0.17042}, "word_start": {"share": 0.54941, "mean": 0.39815, "std": 0.20874}, "word_cont": {"share": 0.26794, "mean": 0.28018, "std": 0.13596}, "punct": {"share": 0.13181, "mean": 0.61398, "std": 0.18236}}, "alpha_p05": 0.12253, "alpha_p50": 0.35533, "alpha_p95": 0.79689}
{"step": 47710, "total_steps": 72479, "tokens": 25013780480, "loss": 4.33346700668335, "l_mix": 2.6854960322380066, "l_a": 2.773600995540619, "l_c": 2.719635248184204, "z_loss": 38.14905405044556, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16153064370155334, "tokens_per_second": 1472705.7112062008, "eta_seconds": 8817.844171571733, "data_wait_seconds": 1.8835067749023438e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 47720, "total_steps": 72479, "tokens": 25019023360, "loss": 4.432613015174866, "l_mix": 2.747061938047409, "l_a": 2.835226386785507, "l_c": 2.783277004957199, "z_loss": 38.77913570404053, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16453731060028076, "tokens_per_second": 1398263.930018616, "eta_seconds": 9283.545340275765, "data_wait_seconds": 1.8149614334106445e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 47730, "total_steps": 72479, "tokens": 25024266240, "loss": 4.449997007846832, "l_mix": 2.757711559534073, "l_a": 2.845652222633362, "l_c": 2.7952992618083954, "z_loss": 38.911673069000244, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12993770837783813, "tokens_per_second": 1434765.6385367059, "eta_seconds": 9043.709553313256, "data_wait_seconds": 1.6450881958007812e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 47740, "total_steps": 72479, "tokens": 25029509120, "loss": 4.375973403453827, "l_mix": 2.7121090292930603, "l_a": 2.797624111175537, "l_c": 2.7485900223255157, "z_loss": 37.40044355392456, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12858061492443085, "tokens_per_second": 1466869.9065262212, "eta_seconds": 8842.202552723886, "data_wait_seconds": 1.5348196029663086e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 47750, "total_steps": 72479, "tokens": 25034752000, "loss": 4.412088990211487, "l_mix": 2.7341780066490173, "l_a": 2.824101835489273, "l_c": 2.768934667110443, "z_loss": 38.05020332336426, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15285490453243256, "tokens_per_second": 1443333.233273789, "eta_seconds": 8982.76132850647, "data_wait_seconds": 1.9431114196777344e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 47760, "total_steps": 72479, "tokens": 25039994880, "loss": 3.6390455067157745, "l_mix": 2.2553086280822754, "l_a": 2.3545467406511307, "l_c": 2.2579091638326645, "z_loss": 44.77409744262695, "lr": 0.0023, "wd": 0.01, "grad_norm": 1.1634492874145508, "tokens_per_second": 1429835.5176957666, "eta_seconds": 9063.89225306511, "data_wait_seconds": 2.1636486053466797e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 47770, "total_steps": 72479, "tokens": 25045237760, "loss": 3.111124664545059, "l_mix": 1.9142110049724579, "l_a": 2.0278482139110565, "l_c": 1.9618637412786484, "z_loss": 40.656776428222656, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14082878828048706, "tokens_per_second": 1449046.6840159453, "eta_seconds": 8940.106854319572, "data_wait_seconds": 2.0205974578857422e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 47780, "total_steps": 72479, "tokens": 25050480640, "loss": 3.108652114868164, "l_mix": 1.911721110343933, "l_a": 2.028795376420021, "l_c": 1.960974633693695, "z_loss": 41.113365650177, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17610101401805878, "tokens_per_second": 1432559.322280876, "eta_seconds": 9039.33897228241, "data_wait_seconds": 1.6510486602783203e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 47790, "total_steps": 72479, "tokens": 25055723520, "loss": 3.137060135602951, "l_mix": 1.931146964430809, "l_a": 2.0428305566310883, "l_c": 1.9768800139427185, "z_loss": 39.57856559753418, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20241773128509521, "tokens_per_second": 1409134.7463013967, "eta_seconds": 9185.882660245896, "data_wait_seconds": 2.8401613235473633e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 47800, "total_steps": 72479, "tokens": 25060966400, "loss": 3.1904696822166443, "l_mix": 1.9631440043449402, "l_a": 2.077132761478424, "l_c": 2.0139524340629578, "z_loss": 40.63769340515137, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1691192090511322, "tokens_per_second": 1429652.7622445635, "eta_seconds": 9050.38194847107, "data_wait_seconds": 1.7076730728149414e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.18575136, "gate_a2b/L1_absmax": 0.71676773, "gate_a2b/L1_rms": 0.19729945, "gate_a2b/L3_absmean": 0.18100362, "gate_a2b/L3_absmax": 0.56190217, "gate_a2b/L3_rms": 0.18718271, "gate_a2b/L5_absmean": 0.2773726, "gate_a2b/L5_absmax": 0.73693329, "gate_a2b/L5_rms": 0.28341815, "gate_c2b/L1_absmean": 0.02077113, "gate_c2b/L1_absmax": 0.62904996, "gate_c2b/L1_rms": 0.06614185, "gate_c2b/L3_absmean": 0.18697913, "gate_c2b/L3_absmax": 0.42481783, "gate_c2b/L3_rms": 0.19096625, "gate_c2b/L5_absmean": 0.53043449, "gate_c2b/L5_absmax": 0.73882073, "gate_c2b/L5_rms": 0.54150105, "gate_b2a/L7_absmean": 0.05124588, "gate_b2a/L7_absmax": 0.32268405, "gate_b2a/L7_rms": 0.05667312, "gate_b2a/L11_absmean": 0.0825118, "gate_b2a/L11_absmax": 0.59720856, "gate_b2a/L11_rms": 0.09141948, "gate_b2a/L14_absmean": 0.21114346, "gate_b2a/L14_absmax": 0.79574406, "gate_b2a/L14_rms": 0.21992633, "gate_b2c/L3_absmean": 0.03651151, "gate_b2c/L3_absmax": 0.57148594, "gate_b2c/L3_rms": 0.0477272, "gate_b2c/L5_absmean": 0.03916988, "gate_b2c/L5_absmax": 0.57705706, "gate_b2c/L5_rms": 0.04938862, "gate_b2c/L6_absmean": 0.05636568, "gate_b2c/L6_absmax": 0.37496266, "gate_b2c/L6_rms": 0.0631588}, "streams": {"cos_a2b/round0": 0.359862, "cos_c2b/round0": 0.264454, "cos_b2a/round0": 0.135458, "cos_b2c/round0": 0.111591, "rms_a/round0": 6.187557, "rms_b/round0": 10.93333, "rms_c/round0": 4.229039, "cos_a2b/round1": 0.409331, "cos_c2b/round1": 0.295915, "cos_b2a/round1": 0.234534, "cos_b2c/round1": 0.117525, "rms_a/round1": 12.137818, "rms_b/round1": 26.03006, "rms_c/round1": 9.036042, "cos_a2b/round2": 0.517453, "cos_c2b/round2": 0.513499, "cos_b2a/round2": 0.085288, "cos_b2c/round2": 0.166424, "rms_a/round2": 32.078175, "rms_b/round2": 67.241508, "rms_c/round2": 17.350832}, "alpha_mean": 0.41859184614878586, "alpha_hist": [0.03648, 0.16113, 0.18547, 0.14619, 0.11351, 0.10378, 0.10048, 0.09193, 0.05542, 0.0056], "alpha_by_type": {"special": {"share": 0.00102, "mean": 0.6175, "std": 0.13605}, "whitespace": {"share": 0.03831, "mean": 0.63447, "std": 0.17526}, "digit": {"share": 0.01195, "mean": 0.40902, "std": 0.18032}, "word_start": {"share": 0.55002, "mean": 0.42345, "std": 0.2162}, "word_cont": {"share": 0.26735, "mean": 0.27459, "std": 0.14783}, "punct": {"share": 0.13135, "mean": 0.6277, "std": 0.18829}}, "alpha_p05": 0.11202, "alpha_p50": 0.36932, "alpha_p95": 0.80317}
{"step": 47810, "total_steps": 72479, "tokens": 25066209280, "loss": 3.156928062438965, "l_mix": 1.9428407400846481, "l_a": 2.0580109357833862, "l_c": 1.988946408033371, "z_loss": 40.54965019226074, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20358751714229584, "tokens_per_second": 1457112.8356977869, "eta_seconds": 8876.224514079095, "data_wait_seconds": 3.001093864440918e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 47820, "total_steps": 72479, "tokens": 25071452160, "loss": 4.335514068603516, "l_mix": 2.6861009895801544, "l_a": 2.7735495865345, "l_c": 2.7244938910007477, "z_loss": 36.950499057769775, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20292945206165314, "tokens_per_second": 1460250.2410496373, "eta_seconds": 8853.563196611403, "data_wait_seconds": 1.958012580871582e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 47830, "total_steps": 72479, "tokens": 25076695040, "loss": 4.37061208486557, "l_mix": 2.7082319259643555, "l_a": 2.7982871532440186, "l_c": 2.7429796159267426, "z_loss": 38.058151721954346, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.10829197615385056, "tokens_per_second": 1311968.6101865082, "eta_seconds": 9850.216546082496, "data_wait_seconds": 1.7762184143066406e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 47840, "total_steps": 72479, "tokens": 25081937920, "loss": 4.343530476093292, "l_mix": 2.691245347261429, "l_a": 2.7808490097522736, "l_c": 2.726768136024475, "z_loss": 38.15190124511719, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1267651617527008, "tokens_per_second": 1354265.3213282276, "eta_seconds": 9538.701042222978, "data_wait_seconds": 1.7255544662475586e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 47850, "total_steps": 72479, "tokens": 25087180800, "loss": 3.1416781544685364, "l_mix": 1.9329600781202316, "l_a": 2.0454627573490143, "l_c": 1.9835973531007767, "z_loss": 40.94404077529907, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.161304771900177, "tokens_per_second": 1423357.8485426044, "eta_seconds": 9071.990691041947, "data_wait_seconds": 1.8268823623657227e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 47860, "total_steps": 72479, "tokens": 25092423680, "loss": 4.623145937919617, "l_mix": 2.868586242198944, "l_a": 2.941244810819626, "l_c": 2.907287299633026, "z_loss": 36.98393964767456, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13012951612472534, "tokens_per_second": 1395888.6629343492, "eta_seconds": 9246.75915403366, "data_wait_seconds": 3.215670585632324e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 47870, "total_steps": 72479, "tokens": 25097666560, "loss": 4.580966234207153, "l_mix": 2.8422186076641083, "l_a": 2.9152027666568756, "l_c": 2.8806222677230835, "z_loss": 38.416213035583496, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11920129507780075, "tokens_per_second": 1401191.6155160672, "eta_seconds": 9208.022121405602, "data_wait_seconds": 2.16066837310791e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 47880, "total_steps": 72479, "tokens": 25102909440, "loss": 3.2011043429374695, "l_mix": 1.9693381488323212, "l_a": 2.0847762525081635, "l_c": 2.0211109817028046, "z_loss": 41.04010200500488, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16965964436531067, "tokens_per_second": 1466392.659893458, "eta_seconds": 8795.025278520583, "data_wait_seconds": 1.609325408935547e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 47890, "total_steps": 72479, "tokens": 25108152320, "loss": 3.170754313468933, "l_mix": 1.9508583843708038, "l_a": 2.067289710044861, "l_c": 1.999029889702797, "z_loss": 41.016769886016846, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1778329461812973, "tokens_per_second": 1440067.0318587394, "eta_seconds": 8952.164966487884, "data_wait_seconds": 2.282857894897461e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 47900, "total_steps": 72479, "tokens": 25113395200, "loss": 3.121736168861389, "l_mix": 1.9202793389558792, "l_a": 2.0348328948020935, "l_c": 1.9700229465961456, "z_loss": 40.52601337432861, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1869235336780548, "tokens_per_second": 1371780.7506773993, "eta_seconds": 9393.975491809844, "data_wait_seconds": 3.814697265625e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.18502109, "gate_a2b/L1_absmax": 0.71649891, "gate_a2b/L1_rms": 0.19677381, "gate_a2b/L3_absmean": 0.18075573, "gate_a2b/L3_absmax": 0.56085527, "gate_a2b/L3_rms": 0.18691088, "gate_a2b/L5_absmean": 0.27614537, "gate_a2b/L5_absmax": 0.73688418, "gate_a2b/L5_rms": 0.28218347, "gate_c2b/L1_absmean": 0.02064581, "gate_c2b/L1_absmax": 0.62839299, "gate_c2b/L1_rms": 0.06627993, "gate_c2b/L3_absmean": 0.1865458, "gate_c2b/L3_absmax": 0.42256829, "gate_c2b/L3_rms": 0.19051968, "gate_c2b/L5_absmean": 0.5309251, "gate_c2b/L5_absmax": 0.74700809, "gate_c2b/L5_rms": 0.54203087, "gate_b2a/L7_absmean": 0.05114271, "gate_b2a/L7_absmax": 0.32315004, "gate_b2a/L7_rms": 0.05666929, "gate_b2a/L11_absmean": 0.08237769, "gate_b2a/L11_absmax": 0.59730917, "gate_b2a/L11_rms": 0.09131125, "gate_b2a/L14_absmean": 0.21183631, "gate_b2a/L14_absmax": 0.79391277, "gate_b2a/L14_rms": 0.22047643, "gate_b2c/L3_absmean": 0.03611008, "gate_b2c/L3_absmax": 0.56895596, "gate_b2c/L3_rms": 0.04735729, "gate_b2c/L5_absmean": 0.03890769, "gate_b2c/L5_absmax": 0.57409769, "gate_b2c/L5_rms": 0.04912379, "gate_b2c/L6_absmean": 0.05623987, "gate_b2c/L6_absmax": 0.37114668, "gate_b2c/L6_rms": 0.06293419}, "streams": {"cos_a2b/round0": 0.358772, "cos_c2b/round0": 0.266249, "cos_b2a/round0": 0.13307, "cos_b2c/round0": 0.111251, "rms_a/round0": 6.182191, "rms_b/round0": 10.984187, "rms_c/round0": 4.236272, "cos_a2b/round1": 0.408241, "cos_c2b/round1": 0.299852, "cos_b2a/round1": 0.234351, "cos_b2c/round1": 0.115427, "rms_a/round1": 12.077939, "rms_b/round1": 25.991447, "rms_c/round1": 9.073596, "cos_a2b/round2": 0.517717, "cos_c2b/round2": 0.514207, "cos_b2a/round2": 0.084881, "cos_b2c/round2": 0.163942, "rms_a/round2": 32.166302, "rms_b/round2": 67.177544, "rms_c/round2": 17.443169}, "alpha_mean": 0.4166066907692709, "alpha_hist": [0.02884, 0.15718, 0.19017, 0.15112, 0.12302, 0.11006, 0.10242, 0.08504, 0.04752, 0.00464], "alpha_by_type": {"special": {"share": 0.00107, "mean": 0.64735, "std": 0.09707}, "whitespace": {"share": 0.04272, "mean": 0.63117, "std": 0.16504}, "digit": {"share": 0.01128, "mean": 0.40803, "std": 0.16963}, "word_start": {"share": 0.55573, "mean": 0.41906, "std": 0.20938}, "word_cont": {"share": 0.2692, "mean": 0.28283, "std": 0.14118}, "punct": {"share": 0.12001, "mean": 0.62769, "std": 0.18456}}, "alpha_p05": 0.12831, "alpha_p50": 0.36523, "alpha_p95": 0.79311}
{"step": 47910, "total_steps": 72479, "tokens": 25118638080, "loss": 3.1397512555122375, "l_mix": 1.9318799674510956, "l_a": 2.0462488383054733, "l_c": 1.979988545179367, "z_loss": 40.08380317687988, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17564252018928528, "tokens_per_second": 1249541.8000564475, "eta_seconds": 10308.764277768136, "data_wait_seconds": 2.7626752853393555e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 47920, "total_steps": 72479, "tokens": 25123880960, "loss": 3.130912095308304, "l_mix": 1.927107110619545, "l_a": 2.0375042259693146, "l_c": 1.975178837776184, "z_loss": 41.517253398895264, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1931910663843155, "tokens_per_second": 1407500.0046736954, "eta_seconds": 9148.127139782906, "data_wait_seconds": 2.2292137145996094e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 47930, "total_steps": 72479, "tokens": 25129123840, "loss": 4.444658517837524, "l_mix": 2.7545052468776703, "l_a": 2.8429639637470245, "l_c": 2.790879875421524, "z_loss": 37.376267433166504, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11883395165205002, "tokens_per_second": 1335008.2919906056, "eta_seconds": 9640.94844145775, "data_wait_seconds": 1.8417835235595703e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 47940, "total_steps": 72479, "tokens": 25134366720, "loss": 4.405543506145477, "l_mix": 2.730075031518936, "l_a": 2.8179695308208466, "l_c": 2.7669250667095184, "z_loss": 37.73536825180054, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13864558935165405, "tokens_per_second": 1406464.214851734, "eta_seconds": 9147.408868384362, "data_wait_seconds": 3.0219554901123047e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 47950, "total_steps": 72479, "tokens": 25139609600, "loss": 4.446046888828278, "l_mix": 2.755698949098587, "l_a": 2.8439188301563263, "l_c": 2.790574163198471, "z_loss": 38.15045976638794, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14076721668243408, "tokens_per_second": 1309144.3962973335, "eta_seconds": 9823.408623504638, "data_wait_seconds": 1.823902130126953e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 47960, "total_steps": 72479, "tokens": 25144852480, "loss": 3.109128922224045, "l_mix": 1.9132208079099655, "l_a": 2.028255969285965, "l_c": 1.9581043869256973, "z_loss": 40.17973518371582, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16184131801128387, "tokens_per_second": 1428471.8355311689, "eta_seconds": 8999.139606571198, "data_wait_seconds": 2.8252601623535156e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 47970, "total_steps": 72479, "tokens": 25150095360, "loss": 3.111055701971054, "l_mix": 1.9137887954711914, "l_a": 2.026068076491356, "l_c": 1.9648213535547256, "z_loss": 41.970983028411865, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17183417081832886, "tokens_per_second": 1343497.2430811692, "eta_seconds": 9564.422002482415, "data_wait_seconds": 1.665949821472168e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 47980, "total_steps": 72479, "tokens": 25155338240, "loss": 3.1490603387355804, "l_mix": 1.9374134093523026, "l_a": 2.05669105052948, "l_c": 1.9821318984031677, "z_loss": 40.70960330963135, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.22410817444324493, "tokens_per_second": 1465217.839109722, "eta_seconds": 8766.29492840767, "data_wait_seconds": 3.129243850708008e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 47990, "total_steps": 72479, "tokens": 25160581120, "loss": 3.136491149663925, "l_mix": 1.9293192923069, "l_a": 2.042946606874466, "l_c": 1.9809594303369522, "z_loss": 39.704317569732666, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.23547601699829102, "tokens_per_second": 1381964.079841612, "eta_seconds": 9290.609661483764, "data_wait_seconds": 2.995133399963379e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 48000, "total_steps": 72479, "tokens": 25165824000, "loss": 3.1318706572055817, "l_mix": 1.928093433380127, "l_a": 2.040673777461052, "l_c": 1.9719170033931732, "z_loss": 39.639939308166504, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16702520847320557, "tokens_per_second": 1459841.8325913048, "eta_seconds": 8791.39483845234, "data_wait_seconds": 2.0742416381835938e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.18501292, "gate_a2b/L1_absmax": 0.71517652, "gate_a2b/L1_rms": 0.19647309, "gate_a2b/L3_absmean": 0.18096463, "gate_a2b/L3_absmax": 0.55989909, "gate_a2b/L3_rms": 0.1870767, "gate_a2b/L5_absmean": 0.2752347, "gate_a2b/L5_absmax": 0.73700297, "gate_a2b/L5_rms": 0.28129455, "gate_c2b/L1_absmean": 0.02082343, "gate_c2b/L1_absmax": 0.62881047, "gate_c2b/L1_rms": 0.066209, "gate_c2b/L3_absmean": 0.18650909, "gate_c2b/L3_absmax": 0.41948774, "gate_c2b/L3_rms": 0.19041382, "gate_c2b/L5_absmean": 0.53041309, "gate_c2b/L5_absmax": 0.74677587, "gate_c2b/L5_rms": 0.54153699, "gate_b2a/L7_absmean": 0.05107224, "gate_b2a/L7_absmax": 0.32240209, "gate_b2a/L7_rms": 0.0565502, "gate_b2a/L11_absmean": 0.08233339, "gate_b2a/L11_absmax": 0.59724265, "gate_b2a/L11_rms": 0.09123923, "gate_b2a/L14_absmean": 0.21177758, "gate_b2a/L14_absmax": 0.79230326, "gate_b2a/L14_rms": 0.22041902, "gate_b2c/L3_absmean": 0.03647655, "gate_b2c/L3_absmax": 0.56856197, "gate_b2c/L3_rms": 0.04754756, "gate_b2c/L5_absmean": 0.03905601, "gate_b2c/L5_absmax": 0.57258147, "gate_b2c/L5_rms": 0.04915087, "gate_b2c/L6_absmean": 0.0564308, "gate_b2c/L6_absmax": 0.37080225, "gate_b2c/L6_rms": 0.06308978}, "streams": {"cos_a2b/round0": 0.361085, "cos_c2b/round0": 0.26609, "cos_b2a/round0": 0.143881, "cos_b2c/round0": 0.113985, "rms_a/round0": 6.210707, "rms_b/round0": 10.978834, "rms_c/round0": 4.280948, "cos_a2b/round1": 0.41445, "cos_c2b/round1": 0.306127, "cos_b2a/round1": 0.243242, "cos_b2c/round1": 0.123838, "rms_a/round1": 12.174123, "rms_b/round1": 26.098122, "rms_c/round1": 9.193591, "cos_a2b/round2": 0.523013, "cos_c2b/round2": 0.512811, "cos_b2a/round2": 0.09121, "cos_b2c/round2": 0.17369, "rms_a/round2": 32.485729, "rms_b/round2": 67.348534, "rms_c/round2": 17.632732}, "alpha_mean": 0.40878414770797633, "alpha_hist": [0.03233, 0.14812, 0.19348, 0.16243, 0.1299, 0.11081, 0.10574, 0.08287, 0.03302, 0.0013], "alpha_by_type": {"special": {"share": 0.00114, "mean": 0.58766, "std": 0.14628}, "whitespace": {"share": 0.03322, "mean": 0.62203, "std": 0.16139}, "digit": {"share": 0.00995, "mean": 0.38151, "std": 0.17689}, "word_start": {"share": 0.58362, "mean": 0.41426, "std": 0.20018}, "word_cont": {"share": 0.25999, "mean": 0.2815, "std": 0.14027}, "punct": {"share": 0.11208, "mean": 0.61293, "std": 0.17799}}, "alpha_p05": 0.12489, "alpha_p50": 0.37754, "alpha_p95": 0.77764}
{"step": 48010, "total_steps": 72479, "tokens": 25171066880, "loss": 3.137736588716507, "l_mix": 1.9304014593362808, "l_a": 2.0445221066474915, "l_c": 1.9799283891916275, "z_loss": 41.00860929489136, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1513434648513794, "tokens_per_second": 1462072.3919317862, "eta_seconds": 8774.39663233757, "data_wait_seconds": 1.582503318786621e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 48020, "total_steps": 72479, "tokens": 25176309760, "loss": 3.157959371805191, "l_mix": 1.9432528167963028, "l_a": 2.0563288927078247, "l_c": 1.9926927834749222, "z_loss": 40.363099098205566, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12631884217262268, "tokens_per_second": 1396788.1678823803, "eta_seconds": 9180.74800951481, "data_wait_seconds": 1.7017126083374023e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 48030, "total_steps": 72479, "tokens": 25181552640, "loss": 3.1482528746128082, "l_mix": 1.9362880289554596, "l_a": 2.0543648451566696, "l_c": 1.9855177998542786, "z_loss": 40.6500940322876, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18806710839271545, "tokens_per_second": 1356603.6971606442, "eta_seconds": 9448.829705262184, "data_wait_seconds": 2.422928810119629e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 48040, "total_steps": 72479, "tokens": 25186795520, "loss": 4.427836775779724, "l_mix": 2.7444416880607605, "l_a": 2.8303794860839844, "l_c": 2.7809369266033173, "z_loss": 38.359182357788086, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1411682516336441, "tokens_per_second": 1290833.6317082208, "eta_seconds": 9926.201268124581, "data_wait_seconds": 2.9325485229492188e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 48050, "total_steps": 72479, "tokens": 25192038400, "loss": 4.4601112604141235, "l_mix": 2.7647812366485596, "l_a": 2.8478868901729584, "l_c": 2.803212881088257, "z_loss": 37.721999645233154, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16651055216789246, "tokens_per_second": 1255509.785851348, "eta_seconds": 10201.299660372733, "data_wait_seconds": 2.2202730178833008e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 48060, "total_steps": 72479, "tokens": 25197281280, "loss": 4.443899929523468, "l_mix": 2.7545174658298492, "l_a": 2.841674715280533, "l_c": 2.7895999550819397, "z_loss": 38.13362407684326, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15604525804519653, "tokens_per_second": 1474818.2019426618, "eta_seconds": 8680.791066408157, "data_wait_seconds": 3.701448440551758e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 48070, "total_steps": 72479, "tokens": 25202524160, "loss": 3.1682035326957703, "l_mix": 1.9488147646188736, "l_a": 2.0642512142658234, "l_c": 2.0003778636455536, "z_loss": 41.80443716049194, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18939520418643951, "tokens_per_second": 1476132.720550289, "eta_seconds": 8669.50892276764, "data_wait_seconds": 3.218650817871094e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 48080, "total_steps": 72479, "tokens": 25207767040, "loss": 3.1227535605430603, "l_mix": 1.9215164631605148, "l_a": 2.0336553305387497, "l_c": 1.970468059182167, "z_loss": 40.91602611541748, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15326619148254395, "tokens_per_second": 1384202.753629182, "eta_seconds": 9241.495061659813, "data_wait_seconds": 6.261467933654785e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 48090, "total_steps": 72479, "tokens": 25213009920, "loss": 3.125407099723816, "l_mix": 1.9226210415363312, "l_a": 2.0369466096162796, "l_c": 1.972340166568756, "z_loss": 41.319292068481445, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1459483802318573, "tokens_per_second": 1402469.78475027, "eta_seconds": 9117.38717727661, "data_wait_seconds": 4.4286251068115234e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 48100, "total_steps": 72479, "tokens": 25218252800, "loss": 3.176515281200409, "l_mix": 1.954929232597351, "l_a": 2.068564251065254, "l_c": 2.003389060497284, "z_loss": 40.67974376678467, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14897386729717255, "tokens_per_second": 1372949.8252095396, "eta_seconds": 9309.602519559861, "data_wait_seconds": 2.1785497665405273e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.1850626, "gate_a2b/L1_absmax": 0.71521133, "gate_a2b/L1_rms": 0.19655542, "gate_a2b/L3_absmean": 0.18034264, "gate_a2b/L3_absmax": 0.56005436, "gate_a2b/L3_rms": 0.18649192, "gate_a2b/L5_absmean": 0.27468264, "gate_a2b/L5_absmax": 0.73677194, "gate_a2b/L5_rms": 0.28075844, "gate_c2b/L1_absmean": 0.02056525, "gate_c2b/L1_absmax": 0.62808108, "gate_c2b/L1_rms": 0.06597626, "gate_c2b/L3_absmean": 0.1860452, "gate_c2b/L3_absmax": 0.42293164, "gate_c2b/L3_rms": 0.19003621, "gate_c2b/L5_absmean": 0.53020638, "gate_c2b/L5_absmax": 0.75063509, "gate_c2b/L5_rms": 0.5413031, "gate_b2a/L7_absmean": 0.05085554, "gate_b2a/L7_absmax": 0.32193539, "gate_b2a/L7_rms": 0.05636859, "gate_b2a/L11_absmean": 0.0824507, "gate_b2a/L11_absmax": 0.59781587, "gate_b2a/L11_rms": 0.0913541, "gate_b2a/L14_absmean": 0.21170013, "gate_b2a/L14_absmax": 0.79363245, "gate_b2a/L14_rms": 0.22033344, "gate_b2c/L3_absmean": 0.0363339, "gate_b2c/L3_absmax": 0.56747276, "gate_b2c/L3_rms": 0.04741615, "gate_b2c/L5_absmean": 0.03905825, "gate_b2c/L5_absmax": 0.5747534, "gate_b2c/L5_rms": 0.04924481, "gate_b2c/L6_absmean": 0.05606242, "gate_b2c/L6_absmax": 0.37008286, "gate_b2c/L6_rms": 0.06271496}, "streams": {"cos_a2b/round0": 0.358185, "cos_c2b/round0": 0.265155, "cos_b2a/round0": 0.141985, "cos_b2c/round0": 0.113973, "rms_a/round0": 6.211824, "rms_b/round0": 11.042845, "rms_c/round0": 4.250248, "cos_a2b/round1": 0.412078, "cos_c2b/round1": 0.309803, "cos_b2a/round1": 0.238775, "cos_b2c/round1": 0.123498, "rms_a/round1": 12.190394, "rms_b/round1": 26.072754, "rms_c/round1": 9.137917, "cos_a2b/round2": 0.527281, "cos_c2b/round2": 0.512815, "cos_b2a/round2": 0.090862, "cos_b2c/round2": 0.170469, "rms_a/round2": 32.690708, "rms_b/round2": 67.254814, "rms_c/round2": 17.589685}, "alpha_mean": 0.41519673887236763, "alpha_hist": [0.03352, 0.15582, 0.18407, 0.14862, 0.12349, 0.11803, 0.10449, 0.08748, 0.04262, 0.00186], "alpha_by_type": {"special": {"share": 0.00122, "mean": 0.65751, "std": 0.11648}, "whitespace": {"share": 0.02928, "mean": 0.63049, "std": 0.17606}, "digit": {"share": 0.00772, "mean": 0.3629, "std": 0.17213}, "word_start": {"share": 0.60504, "mean": 0.42305, "std": 0.20364}, "word_cont": {"share": 0.24921, "mean": 0.27473, "std": 0.14312}, "punct": {"share": 0.10753, "mean": 0.63892, "std": 0.18017}}, "alpha_p05": 0.11839, "alpha_p50": 0.38191, "alpha_p95": 0.78923}
{"step": 48110, "total_steps": 72479, "tokens": 25223495680, "loss": 4.7547279596328735, "l_mix": 2.9511767625808716, "l_a": 3.024700403213501, "l_c": 2.9871364533901215, "z_loss": 36.67015314102173, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15252435207366943, "tokens_per_second": 1465733.1057896179, "eta_seconds": 8716.712627649307, "data_wait_seconds": 1.6987323760986328e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 48120, "total_steps": 72479, "tokens": 25228738560, "loss": 4.753385484218597, "l_mix": 2.9489135146141052, "l_a": 3.025853157043457, "l_c": 2.989052891731262, "z_loss": 37.84356164932251, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14823132753372192, "tokens_per_second": 1466597.6451932988, "eta_seconds": 8707.999384737015, "data_wait_seconds": 1.990795135498047e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 48130, "total_steps": 72479, "tokens": 25233981440, "loss": 4.974466443061829, "l_mix": 3.0885154008865356, "l_a": 3.1625378727912903, "l_c": 3.1239651441574097, "z_loss": 37.99282455444336, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.26144883036613464, "tokens_per_second": 1445023.8351184127, "eta_seconds": 8834.379199671746, "data_wait_seconds": 1.6361474990844727e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 48140, "total_steps": 72479, "tokens": 25239224320, "loss": 3.1446326971054077, "l_mix": 1.9345910400152206, "l_a": 2.049302637577057, "l_c": 1.9841694235801697, "z_loss": 41.08155298233032, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19113366305828094, "tokens_per_second": 1326201.745629717, "eta_seconds": 9621.949054169656, "data_wait_seconds": 3.75211238861084e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 48150, "total_steps": 72479, "tokens": 25244467200, "loss": 4.424005806446075, "l_mix": 2.74135360121727, "l_a": 2.8314729034900665, "l_c": 2.777367413043976, "z_loss": 38.27487659454346, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12455185502767563, "tokens_per_second": 1295498.837510934, "eta_seconds": 9845.939172363282, "data_wait_seconds": 1.9758939743041992e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 48160, "total_steps": 72479, "tokens": 25249710080, "loss": 4.394379138946533, "l_mix": 2.7229277789592743, "l_a": 2.8145441710948944, "l_c": 2.756960242986679, "z_loss": 38.726778507232666, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1448945552110672, "tokens_per_second": 1441681.9310597482, "eta_seconds": 8843.947889828683, "data_wait_seconds": 3.656744956970215e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 48170, "total_steps": 72479, "tokens": 25254952960, "loss": 4.369034588336945, "l_mix": 2.7074049413204193, "l_a": 2.7942327558994293, "l_c": 2.7445323169231415, "z_loss": 37.77393293380737, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18043377995491028, "tokens_per_second": 1157731.148688803, "eta_seconds": 11008.529058265687, "data_wait_seconds": 3.0040740966796875e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 48180, "total_steps": 72479, "tokens": 25260195840, "loss": 3.1628422141075134, "l_mix": 1.9460531026124954, "l_a": 2.0575400292873383, "l_c": 1.9984235316514969, "z_loss": 40.39036321640015, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15066461265087128, "tokens_per_second": 1468401.3622346923, "eta_seconds": 8675.880069065093, "data_wait_seconds": 2.3990869522094727e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 48190, "total_steps": 72479, "tokens": 25265438720, "loss": 3.1217457056045532, "l_mix": 1.9208886921405792, "l_a": 2.036087214946747, "l_c": 1.966769516468048, "z_loss": 41.91067314147949, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.228133425116539, "tokens_per_second": 1424085.2133738257, "eta_seconds": 8942.183453917503, "data_wait_seconds": 2.485513687133789e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 48200, "total_steps": 72479, "tokens": 25270681600, "loss": 3.159847915172577, "l_mix": 1.9460186213254929, "l_a": 2.0520417988300323, "l_c": 1.9940558671951294, "z_loss": 40.282588481903076, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.22964705526828766, "tokens_per_second": 1372664.010267593, "eta_seconds": 9273.346031355859, "data_wait_seconds": 3.692507743835449e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.1841079, "gate_a2b/L1_absmax": 0.71252769, "gate_a2b/L1_rms": 0.19563513, "gate_a2b/L3_absmean": 0.1801189, "gate_a2b/L3_absmax": 0.55510038, "gate_a2b/L3_rms": 0.18619755, "gate_a2b/L5_absmean": 0.27415252, "gate_a2b/L5_absmax": 0.73345441, "gate_a2b/L5_rms": 0.28012604, "gate_c2b/L1_absmean": 0.02024508, "gate_c2b/L1_absmax": 0.6254518, "gate_c2b/L1_rms": 0.06564794, "gate_c2b/L3_absmean": 0.18624109, "gate_c2b/L3_absmax": 0.42226592, "gate_c2b/L3_rms": 0.19019291, "gate_c2b/L5_absmean": 0.52977842, "gate_c2b/L5_absmax": 0.74676132, "gate_c2b/L5_rms": 0.54087031, "gate_b2a/L7_absmean": 0.05080984, "gate_b2a/L7_absmax": 0.32228154, "gate_b2a/L7_rms": 0.05629775, "gate_b2a/L11_absmean": 0.08200981, "gate_b2a/L11_absmax": 0.59874505, "gate_b2a/L11_rms": 0.09096465, "gate_b2a/L14_absmean": 0.21159589, "gate_b2a/L14_absmax": 0.79149336, "gate_b2a/L14_rms": 0.22020648, "gate_b2c/L3_absmean": 0.03600861, "gate_b2c/L3_absmax": 0.56614256, "gate_b2c/L3_rms": 0.04717614, "gate_b2c/L5_absmean": 0.03893084, "gate_b2c/L5_absmax": 0.57454991, "gate_b2c/L5_rms": 0.04911895, "gate_b2c/L6_absmean": 0.05600563, "gate_b2c/L6_absmax": 0.36945501, "gate_b2c/L6_rms": 0.06260488}, "streams": {"cos_a2b/round0": 0.359429, "cos_c2b/round0": 0.268247, "cos_b2a/round0": 0.142617, "cos_b2c/round0": 0.113944, "rms_a/round0": 6.24314, "rms_b/round0": 11.013792, "rms_c/round0": 4.26298, "cos_a2b/round1": 0.414166, "cos_c2b/round1": 0.307698, "cos_b2a/round1": 0.241749, "cos_b2c/round1": 0.121544, "rms_a/round1": 12.243752, "rms_b/round1": 26.002272, "rms_c/round1": 9.163279, "cos_a2b/round2": 0.523619, "cos_c2b/round2": 0.514602, "cos_b2a/round2": 0.087065, "cos_b2c/round2": 0.17298, "rms_a/round2": 32.566334, "rms_b/round2": 67.337746, "rms_c/round2": 17.628693}, "alpha_mean": 0.40893840352097754, "alpha_hist": [0.0287, 0.15376, 0.20215, 0.16087, 0.12292, 0.10561, 0.09804, 0.08534, 0.04066, 0.00194], "alpha_by_type": {"special": {"share": 0.00117, "mean": 0.58549, "std": 0.15135}, "whitespace": {"share": 0.03616, "mean": 0.60814, "std": 0.16246}, "digit": {"share": 0.01025, "mean": 0.39725, "std": 0.17461}, "word_start": {"share": 0.58154, "mean": 0.41036, "std": 0.20187}, "word_cont": {"share": 0.25298, "mean": 0.2757, "std": 0.13918}, "punct": {"share": 0.11789, "mean": 0.62596, "std": 0.17982}}, "alpha_p05": 0.1192, "alpha_p50": 0.38376, "alpha_p95": 0.79944}
{"step": 48210, "total_steps": 72479, "tokens": 25275924480, "loss": 3.1238451302051544, "l_mix": 1.920528531074524, "l_a": 2.0379281640052795, "l_c": 1.973127007484436, "z_loss": 41.38469743728638, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2481038123369217, "tokens_per_second": 1420885.3434235156, "eta_seconds": 8954.94174170494, "data_wait_seconds": 2.0951032638549805e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 48220, "total_steps": 72479, "tokens": 25281167360, "loss": 4.283125102519989, "l_mix": 2.6535918712615967, "l_a": 2.74172905087471, "l_c": 2.6900482773780823, "z_loss": 39.27474546432495, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15101435780525208, "tokens_per_second": 1413149.3055508162, "eta_seconds": 9000.253930735587, "data_wait_seconds": 1.5139579772949219e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 48230, "total_steps": 72479, "tokens": 25286410240, "loss": 4.39369335770607, "l_mix": 2.722585916519165, "l_a": 2.8054158091545105, "l_c": 2.7649421989917755, "z_loss": 38.547447204589844, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18294866383075714, "tokens_per_second": 1350502.2725550071, "eta_seconds": 9413.875096964837, "data_wait_seconds": 1.7434358596801758e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 48240, "total_steps": 72479, "tokens": 25291653120, "loss": 4.285916030406952, "l_mix": 2.656242072582245, "l_a": 2.743377774953842, "l_c": 2.68886861205101, "z_loss": 38.51999235153198, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15437421202659607, "tokens_per_second": 1323669.2587909761, "eta_seconds": 9600.749392342566, "data_wait_seconds": 1.6450881958007812e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 48250, "total_steps": 72479, "tokens": 25296896000, "loss": 4.35175359249115, "l_mix": 2.6962331235408783, "l_a": 2.782099574804306, "l_c": 2.736302077770233, "z_loss": 38.63755702972412, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1723388433456421, "tokens_per_second": 1415294.28159064, "eta_seconds": 8975.500090146064, "data_wait_seconds": 3.260374069213867e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 48260, "total_steps": 72479, "tokens": 25302138880, "loss": 4.409519016742706, "l_mix": 2.733897715806961, "l_a": 2.8167744278907776, "l_c": 2.7686294615268707, "z_loss": 37.485923767089844, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16584818065166473, "tokens_per_second": 1410031.7895339173, "eta_seconds": 9005.27999882698, "data_wait_seconds": 1.9729137420654297e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 48270, "total_steps": 72479, "tokens": 25307381760, "loss": 4.267389714717865, "l_mix": 2.642814427614212, "l_a": 2.733172059059143, "l_c": 2.682078868150711, "z_loss": 39.40351438522339, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20244349539279938, "tokens_per_second": 1152319.0085990175, "eta_seconds": 11014.73471953869, "data_wait_seconds": 2.0205974578857422e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20764, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 48280, "total_steps": 72479, "tokens": 25312624640, "loss": 4.3874505162239075, "l_mix": 2.719525784254074, "l_a": 2.8051519691944122, "l_c": 2.7545969486236572, "z_loss": 38.2042121887207, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18215225636959076, "tokens_per_second": 1446610.2875415082, "eta_seconds": 8770.327033662796, "data_wait_seconds": 2.333521842956543e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 48290, "total_steps": 72479, "tokens": 25317867520, "loss": 3.1046525835990906, "l_mix": 1.9107724875211716, "l_a": 2.0216636955738068, "l_c": 1.9579365849494934, "z_loss": 40.565253257751465, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13419300317764282, "tokens_per_second": 1312633.2574604712, "eta_seconds": 9661.497116518021, "data_wait_seconds": 7.325410842895508e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 48300, "total_steps": 72479, "tokens": 25323110400, "loss": 3.145808756351471, "l_mix": 1.9352879077196121, "l_a": 2.0502370595932007, "l_c": 1.984832376241684, "z_loss": 40.189844608306885, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16226395964622498, "tokens_per_second": 1410280.377032772, "eta_seconds": 8988.822193408012, "data_wait_seconds": 1.850724220275879e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.1842759, "gate_a2b/L1_absmax": 0.71380299, "gate_a2b/L1_rms": 0.1957646, "gate_a2b/L3_absmean": 0.17974809, "gate_a2b/L3_absmax": 0.55701905, "gate_a2b/L3_rms": 0.18581888, "gate_a2b/L5_absmean": 0.27352819, "gate_a2b/L5_absmax": 0.73525888, "gate_a2b/L5_rms": 0.27956432, "gate_c2b/L1_absmean": 0.02122122, "gate_c2b/L1_absmax": 0.62694645, "gate_c2b/L1_rms": 0.06609937, "gate_c2b/L3_absmean": 0.18598822, "gate_c2b/L3_absmax": 0.42157269, "gate_c2b/L3_rms": 0.1899147, "gate_c2b/L5_absmean": 0.52926004, "gate_c2b/L5_absmax": 0.74378186, "gate_c2b/L5_rms": 0.54041493, "gate_b2a/L7_absmean": 0.05104162, "gate_b2a/L7_absmax": 0.32213607, "gate_b2a/L7_rms": 0.05649868, "gate_b2a/L11_absmean": 0.08227967, "gate_b2a/L11_absmax": 0.5994665, "gate_b2a/L11_rms": 0.09124525, "gate_b2a/L14_absmean": 0.21172044, "gate_b2a/L14_absmax": 0.79095846, "gate_b2a/L14_rms": 0.22030661, "gate_b2c/L3_absmean": 0.03593333, "gate_b2c/L3_absmax": 0.56743526, "gate_b2c/L3_rms": 0.04714821, "gate_b2c/L5_absmean": 0.03880371, "gate_b2c/L5_absmax": 0.5763216, "gate_b2c/L5_rms": 0.0491062, "gate_b2c/L6_absmean": 0.05608242, "gate_b2c/L6_absmax": 0.36870989, "gate_b2c/L6_rms": 0.06268422}, "streams": {"cos_a2b/round0": 0.360304, "cos_c2b/round0": 0.26151, "cos_b2a/round0": 0.140985, "cos_b2c/round0": 0.112404, "rms_a/round0": 6.258213, "rms_b/round0": 11.053302, "rms_c/round0": 4.282713, "cos_a2b/round1": 0.41309, "cos_c2b/round1": 0.306139, "cos_b2a/round1": 0.242559, "cos_b2c/round1": 0.12225, "rms_a/round1": 12.182391, "rms_b/round1": 26.168428, "rms_c/round1": 9.141517, "cos_a2b/round2": 0.520991, "cos_c2b/round2": 0.507804, "cos_b2a/round2": 0.089376, "cos_b2c/round2": 0.171087, "rms_a/round2": 32.552536, "rms_b/round2": 67.505783, "rms_c/round2": 17.531761}, "alpha_mean": 0.3989518588787391, "alpha_hist": [0.04227, 0.18132, 0.19379, 0.1492, 0.10997, 0.09685, 0.09329, 0.08376, 0.04582, 0.00374], "alpha_by_type": {"special": {"share": 0.00107, "mean": 0.62116, "std": 0.10897}, "whitespace": {"share": 0.03247, "mean": 0.63618, "std": 0.17656}, "digit": {"share": 0.00945, "mean": 0.34968, "std": 0.18103}, "word_start": {"share": 0.57333, "mean": 0.39759, "std": 0.21415}, "word_cont": {"share": 0.26581, "mean": 0.27385, "std": 0.1417}, "punct": {"share": 0.11787, "mean": 0.62428, "std": 0.19107}}, "alpha_p05": 0.11086, "alpha_p50": 0.35488, "alpha_p95": 0.80317}
{"step": 48310, "total_steps": 72479, "tokens": 25328353280, "loss": 3.135242909193039, "l_mix": 1.9292252659797668, "l_a": 2.044449061155319, "l_c": 1.9756093919277191, "z_loss": 40.260390758514404, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14216406643390656, "tokens_per_second": 1372622.3692085422, "eta_seconds": 9231.611662650108, "data_wait_seconds": 1.7523765563964844e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 48320, "total_steps": 72479, "tokens": 25333596160, "loss": 3.1590735912323, "l_mix": 1.9438407123088837, "l_a": 2.0567111372947693, "l_c": 1.99406498670578, "z_loss": 40.57552433013916, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16583380103111267, "tokens_per_second": 1382363.6128228428, "eta_seconds": 9162.765624403954, "data_wait_seconds": 1.704692840576172e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 48330, "total_steps": 72479, "tokens": 25338839040, "loss": 4.468383550643921, "l_mix": 2.768755942583084, "l_a": 2.8599945306777954, "l_c": 2.805430442094803, "z_loss": 38.825276374816895, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1611277014017105, "tokens_per_second": 1461563.0953252742, "eta_seconds": 8662.664617419243, "data_wait_seconds": 1.9103288650512695e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 48340, "total_steps": 72479, "tokens": 25344081920, "loss": 4.30235481262207, "l_mix": 2.6660646498203278, "l_a": 2.751007229089737, "l_c": 2.7032930850982666, "z_loss": 38.200355529785156, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19203628599643707, "tokens_per_second": 1374192.7674931046, "eta_seconds": 9209.61624262333, "data_wait_seconds": 1.6361474990844727e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 48350, "total_steps": 72479, "tokens": 25349324800, "loss": 4.404885351657867, "l_mix": 2.7294325828552246, "l_a": 2.817002534866333, "l_c": 2.767840027809143, "z_loss": 37.68348979949951, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1728716492652893, "tokens_per_second": 1439922.3822284944, "eta_seconds": 8785.574353265762, "data_wait_seconds": 2.0891427993774414e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 48360, "total_steps": 72479, "tokens": 25354567680, "loss": 4.2075404822826385, "l_mix": 2.6074037551879883, "l_a": 2.692379593849182, "l_c": 2.641409456729889, "z_loss": 38.22990417480469, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19185137748718262, "tokens_per_second": 1445169.5115648154, "eta_seconds": 8750.047776961326, "data_wait_seconds": 1.5079975128173828e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 48370, "total_steps": 72479, "tokens": 25359810560, "loss": 4.236437439918518, "l_mix": 2.624261438846588, "l_a": 2.710281640291214, "l_c": 2.663638174533844, "z_loss": 40.69152498245239, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1416931003332138, "tokens_per_second": 1442156.3725304317, "eta_seconds": 8764.694060063362, "data_wait_seconds": 1.761317253112793e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 48380, "total_steps": 72479, "tokens": 25365053440, "loss": 4.330267608165741, "l_mix": 2.6826562881469727, "l_a": 2.771848112344742, "l_c": 2.7201895117759705, "z_loss": 38.99169063568115, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11194377392530441, "tokens_per_second": 1419523.0024214264, "eta_seconds": 8900.747991013526, "data_wait_seconds": 2.512335777282715e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 48390, "total_steps": 72479, "tokens": 25370296320, "loss": 4.815279543399811, "l_mix": 2.9888896048069, "l_a": 3.0621403455734253, "l_c": 3.025825619697571, "z_loss": 37.054654121398926, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1837320774793625, "tokens_per_second": 1443914.1831268298, "eta_seconds": 8746.761947202682, "data_wait_seconds": 3.504753112792969e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 48400, "total_steps": 72479, "tokens": 25375539200, "loss": 4.803779304027557, "l_mix": 2.9805515110492706, "l_a": 3.055779308080673, "l_c": 3.0216463208198547, "z_loss": 37.15754842758179, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19202187657356262, "tokens_per_second": 1294654.2856589442, "eta_seconds": 9751.121123099327, "data_wait_seconds": 2.276897430419922e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.18352303, "gate_a2b/L1_absmax": 0.71283871, "gate_a2b/L1_rms": 0.19500449, "gate_a2b/L3_absmean": 0.17949024, "gate_a2b/L3_absmax": 0.55232066, "gate_a2b/L3_rms": 0.18559051, "gate_a2b/L5_absmean": 0.27276626, "gate_a2b/L5_absmax": 0.73515409, "gate_a2b/L5_rms": 0.27875838, "gate_c2b/L1_absmean": 0.02130346, "gate_c2b/L1_absmax": 0.62737375, "gate_c2b/L1_rms": 0.06597871, "gate_c2b/L3_absmean": 0.18549338, "gate_c2b/L3_absmax": 0.42009097, "gate_c2b/L3_rms": 0.18942979, "gate_c2b/L5_absmean": 0.52919692, "gate_c2b/L5_absmax": 0.74925607, "gate_c2b/L5_rms": 0.54028732, "gate_b2a/L7_absmean": 0.05082124, "gate_b2a/L7_absmax": 0.32254124, "gate_b2a/L7_rms": 0.05634429, "gate_b2a/L11_absmean": 0.08218756, "gate_b2a/L11_absmax": 0.59797257, "gate_b2a/L11_rms": 0.09110121, "gate_b2a/L14_absmean": 0.21215528, "gate_b2a/L14_absmax": 0.79202974, "gate_b2a/L14_rms": 0.22076002, "gate_b2c/L3_absmean": 0.03630794, "gate_b2c/L3_absmax": 0.56614524, "gate_b2c/L3_rms": 0.04737556, "gate_b2c/L5_absmean": 0.0387213, "gate_b2c/L5_absmax": 0.57443023, "gate_b2c/L5_rms": 0.04892244, "gate_b2c/L6_absmean": 0.05584657, "gate_b2c/L6_absmax": 0.3684305, "gate_b2c/L6_rms": 0.06246957}, "streams": {"cos_a2b/round0": 0.357155, "cos_c2b/round0": 0.291271, "cos_b2a/round0": 0.15155, "cos_b2c/round0": 0.130913, "rms_a/round0": 5.971706, "rms_b/round0": 10.921448, "rms_c/round0": 4.159592, "cos_a2b/round1": 0.406608, "cos_c2b/round1": 0.307311, "cos_b2a/round1": 0.247596, "cos_b2c/round1": 0.11522, "rms_a/round1": 11.493943, "rms_b/round1": 24.89407, "rms_c/round1": 8.687813, "cos_a2b/round2": 0.551317, "cos_c2b/round2": 0.517894, "cos_b2a/round2": 0.100573, "cos_b2c/round2": 0.179941, "rms_a/round2": 31.878788, "rms_b/round2": 65.546211, "rms_c/round2": 16.395109}, "alpha_mean": 0.4343919178282789, "alpha_hist": [0.00682, 0.0652, 0.17676, 0.20168, 0.19261, 0.17094, 0.11629, 0.06236, 0.00691, 0.00043], "alpha_by_type": {"special": {"share": 0.00092, "mean": 0.54213, "std": 0.13099}, "whitespace": {"share": 0.0495, "mean": 0.52039, "std": 0.14161}, "digit": {"share": 0.03394, "mean": 0.41767, "std": 0.13259}, "word_start": {"share": 0.55231, "mean": 0.45689, "std": 0.15824}, "word_cont": {"share": 0.26347, "mean": 0.3227, "std": 0.12758}, "punct": {"share": 0.09987, "mean": 0.56669, "std": 0.16449}}, "alpha_p05": 0.19745, "alpha_p50": 0.42084, "alpha_p95": 0.72875}
{"step": 48410, "total_steps": 72479, "tokens": 25380782080, "loss": 4.824036657810211, "l_mix": 2.9931275248527527, "l_a": 3.069923609495163, "l_c": 3.033106714487076, "z_loss": 37.308438301086426, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14466430246829987, "tokens_per_second": 1439222.4586699146, "eta_seconds": 8767.989823937416, "data_wait_seconds": 1.621246337890625e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 48420, "total_steps": 72479, "tokens": 25386024960, "loss": 3.1387325823307037, "l_mix": 1.9315278679132462, "l_a": 2.045467346906662, "l_c": 1.9785481840372086, "z_loss": 42.05831956863403, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.22929823398590088, "tokens_per_second": 1485373.6857736344, "eta_seconds": 8492.034774017335, "data_wait_seconds": 3.272294998168945e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 48430, "total_steps": 72479, "tokens": 25391267840, "loss": 3.1486032009124756, "l_mix": 1.9381749331951141, "l_a": 2.0481761395931244, "l_c": 1.986584559082985, "z_loss": 40.49588108062744, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19251756370067596, "tokens_per_second": 1283539.9362614467, "eta_seconds": 9823.303315925597, "data_wait_seconds": 3.3855438232421875e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 48440, "total_steps": 72479, "tokens": 25396510720, "loss": 4.185003936290741, "l_mix": 2.592654138803482, "l_a": 2.6815621852874756, "l_c": 2.6262703239917755, "z_loss": 40.05771780014038, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17243212461471558, "tokens_per_second": 1426226.263086736, "eta_seconds": 8836.858188772203, "data_wait_seconds": 2.065300941467285e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 48450, "total_steps": 72479, "tokens": 25401753600, "loss": 4.247758150100708, "l_mix": 2.6322761476039886, "l_a": 2.716617166996002, "l_c": 2.6683225333690643, "z_loss": 39.68305826187134, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16538701951503754, "tokens_per_second": 1404168.436844377, "eta_seconds": 8971.940987586975, "data_wait_seconds": 1.9311904907226562e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 48460, "total_steps": 72479, "tokens": 25406996480, "loss": 4.311192631721497, "l_mix": 2.6706813275814056, "l_a": 2.7574447095394135, "l_c": 2.7109260261058807, "z_loss": 39.490675926208496, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2294619083404541, "tokens_per_second": 1355592.5495300787, "eta_seconds": 9289.571174144745, "data_wait_seconds": 2.244114875793457e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 48470, "total_steps": 72479, "tokens": 25412239360, "loss": 4.372786819934845, "l_mix": 2.71001473069191, "l_a": 2.7962856590747833, "l_c": 2.7462878823280334, "z_loss": 39.13444185256958, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17387917637825012, "tokens_per_second": 1445207.9773269705, "eta_seconds": 8709.909431362152, "data_wait_seconds": 2.086162567138672e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 48480, "total_steps": 72479, "tokens": 25417482240, "loss": 4.262596786022186, "l_mix": 2.639932692050934, "l_a": 2.7298184037208557, "l_c": 2.679062068462372, "z_loss": 39.44711256027222, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2032385766506195, "tokens_per_second": 1431191.280402812, "eta_seconds": 8791.54861009121, "data_wait_seconds": 1.84476375579834e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 48490, "total_steps": 72479, "tokens": 25422725120, "loss": 4.431085288524628, "l_mix": 2.745760917663574, "l_a": 2.8314882814884186, "l_c": 2.786259025335312, "z_loss": 38.072786808013916, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.32786813378334045, "tokens_per_second": 1388733.590383371, "eta_seconds": 9056.556937265395, "data_wait_seconds": 2.810359001159668e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 48500, "total_steps": 72479, "tokens": 25427968000, "loss": 4.7236491441726685, "l_mix": 2.9317077100276947, "l_a": 3.005475252866745, "l_c": 2.967662662267685, "z_loss": 36.20695734024048, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16596324741840363, "tokens_per_second": 1393524.9843282378, "eta_seconds": 9021.65522210598, "data_wait_seconds": 1.7404556274414062e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.18342829, "gate_a2b/L1_absmax": 0.71298069, "gate_a2b/L1_rms": 0.19490354, "gate_a2b/L3_absmean": 0.1788885, "gate_a2b/L3_absmax": 0.55469453, "gate_a2b/L3_rms": 0.18504937, "gate_a2b/L5_absmean": 0.27284542, "gate_a2b/L5_absmax": 0.73491693, "gate_a2b/L5_rms": 0.27882761, "gate_c2b/L1_absmean": 0.0208337, "gate_c2b/L1_absmax": 0.62835687, "gate_c2b/L1_rms": 0.06594019, "gate_c2b/L3_absmean": 0.18546978, "gate_c2b/L3_absmax": 0.42471293, "gate_c2b/L3_rms": 0.1894484, "gate_c2b/L5_absmean": 0.52825969, "gate_c2b/L5_absmax": 0.74895072, "gate_c2b/L5_rms": 0.5393613, "gate_b2a/L7_absmean": 0.05076918, "gate_b2a/L7_absmax": 0.32162589, "gate_b2a/L7_rms": 0.05628411, "gate_b2a/L11_absmean": 0.08200739, "gate_b2a/L11_absmax": 0.59869236, "gate_b2a/L11_rms": 0.0910143, "gate_b2a/L14_absmean": 0.21176867, "gate_b2a/L14_absmax": 0.79210716, "gate_b2a/L14_rms": 0.22042041, "gate_b2c/L3_absmean": 0.03602776, "gate_b2c/L3_absmax": 0.56724221, "gate_b2c/L3_rms": 0.04718624, "gate_b2c/L5_absmean": 0.03854882, "gate_b2c/L5_absmax": 0.57626539, "gate_b2c/L5_rms": 0.04889537, "gate_b2c/L6_absmean": 0.05574716, "gate_b2c/L6_absmax": 0.36899841, "gate_b2c/L6_rms": 0.06240331}, "streams": {"cos_a2b/round0": 0.3598, "cos_c2b/round0": 0.302919, "cos_b2a/round0": 0.159734, "cos_b2c/round0": 0.137221, "rms_a/round0": 6.029968, "rms_b/round0": 10.896245, "rms_c/round0": 4.161496, "cos_a2b/round1": 0.419549, "cos_c2b/round1": 0.312927, "cos_b2a/round1": 0.257569, "cos_b2c/round1": 0.117108, "rms_a/round1": 11.628438, "rms_b/round1": 24.610888, "rms_c/round1": 8.672223, "cos_a2b/round2": 0.561784, "cos_c2b/round2": 0.523051, "cos_b2a/round2": 0.109426, "cos_b2c/round2": 0.188526, "rms_a/round2": 32.182293, "rms_b/round2": 65.440857, "rms_c/round2": 16.503111}, "alpha_mean": 0.4553983037578462, "alpha_hist": [0.00351, 0.04555, 0.1548, 0.19615, 0.19159, 0.19624, 0.13213, 0.07202, 0.00783, 0.00018], "alpha_by_type": {"special": {"share": 0.00085, "mean": 0.59216, "std": 0.09879}, "whitespace": {"share": 0.0479, "mean": 0.54811, "std": 0.11658}, "digit": {"share": 0.01758, "mean": 0.44454, "std": 0.11977}, "word_start": {"share": 0.58949, "mean": 0.47235, "std": 0.15576}, "word_cont": {"share": 0.23642, "mean": 0.34042, "std": 0.12335}, "punct": {"share": 0.10776, "mean": 0.57442, "std": 0.16364}}, "alpha_p05": 0.20786, "alpha_p50": 0.46102, "alpha_p95": 0.72467}
{"step": 48510, "total_steps": 72479, "tokens": 25433210880, "loss": 4.785693407058716, "l_mix": 2.9694789350032806, "l_a": 3.0447335243225098, "l_c": 3.009314388036728, "z_loss": 36.89220714569092, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12710249423980713, "tokens_per_second": 1379833.2355529882, "eta_seconds": 9107.375259709357, "data_wait_seconds": 2.5600194931030273e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 48520, "total_steps": 72479, "tokens": 25438453760, "loss": 4.7379642724990845, "l_mix": 2.939366728067398, "l_a": 3.0191319584846497, "l_c": 2.976192682981491, "z_loss": 37.47077226638794, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18400825560092926, "tokens_per_second": 1379440.615177588, "eta_seconds": 9106.166698145866, "data_wait_seconds": 2.5779008865356445e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 48530, "total_steps": 72479, "tokens": 25443696640, "loss": 4.781539618968964, "l_mix": 2.966224431991577, "l_a": 3.044807255268097, "l_c": 3.0062430799007416, "z_loss": 38.05916357040405, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.141541987657547, "tokens_per_second": 1428929.4498088479, "eta_seconds": 8787.119135713578, "data_wait_seconds": 2.720952033996582e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 48540, "total_steps": 72479, "tokens": 25448939520, "loss": 4.842986106872559, "l_mix": 3.006214290857315, "l_a": 3.0797384083271027, "l_c": 3.0428338944911957, "z_loss": 36.350791454315186, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17315934598445892, "tokens_per_second": 1397692.3863193612, "eta_seconds": 8979.751592588425, "data_wait_seconds": 1.722574234008789e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 48550, "total_steps": 72479, "tokens": 25454182400, "loss": 3.1687792241573334, "l_mix": 1.9515268206596375, "l_a": 2.0613574385643005, "l_c": 1.9961502403020859, "z_loss": 39.46036958694458, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2523198425769806, "tokens_per_second": 1427372.069351504, "eta_seconds": 8789.360406708718, "data_wait_seconds": 1.627206802368164e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 48560, "total_steps": 72479, "tokens": 25459425280, "loss": 4.279865860939026, "l_mix": 2.6519093215465546, "l_a": 2.7407761216163635, "l_c": 2.685745596885681, "z_loss": 38.869415283203125, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13140393793582916, "tokens_per_second": 1469468.0679091548, "eta_seconds": 8534.002844882012, "data_wait_seconds": 1.874566078186035e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 48570, "total_steps": 72479, "tokens": 25464668160, "loss": 4.184972733259201, "l_mix": 2.5922568440437317, "l_a": 2.682234674692154, "l_c": 2.6268181204795837, "z_loss": 39.8419771194458, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2270023375749588, "tokens_per_second": 1406069.2414013764, "eta_seconds": 8915.067212128639, "data_wait_seconds": 2.187490463256836e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 48580, "total_steps": 72479, "tokens": 25469911040, "loss": 4.197039425373077, "l_mix": 2.6001880168914795, "l_a": 2.684916079044342, "l_c": 2.637921541929245, "z_loss": 39.05845022201538, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14383794367313385, "tokens_per_second": 1394250.3698160858, "eta_seconds": 8986.878672051429, "data_wait_seconds": 1.7970800399780273e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 48590, "total_steps": 72479, "tokens": 25475153920, "loss": 4.088062226772308, "l_mix": 2.5321223735809326, "l_a": 2.6193447709083557, "l_c": 2.567121386528015, "z_loss": 39.60783624649048, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13175541162490845, "tokens_per_second": 1385053.3199276202, "eta_seconds": 9042.768138813972, "data_wait_seconds": 1.6897916793823242e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 48600, "total_steps": 72479, "tokens": 25480396800, "loss": 4.151572287082672, "l_mix": 2.570853978395462, "l_a": 2.6609035432338715, "l_c": 2.6081570982933044, "z_loss": 41.628453731536865, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16848759353160858, "tokens_per_second": 1391112.1811946523, "eta_seconds": 8999.614352631568, "data_wait_seconds": 3.820657730102539e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.18303481, "gate_a2b/L1_absmax": 0.71724248, "gate_a2b/L1_rms": 0.1946691, "gate_a2b/L3_absmean": 0.17862211, "gate_a2b/L3_absmax": 0.55892968, "gate_a2b/L3_rms": 0.18478005, "gate_a2b/L5_absmean": 0.27189189, "gate_a2b/L5_absmax": 0.73649633, "gate_a2b/L5_rms": 0.27787736, "gate_c2b/L1_absmean": 0.02185097, "gate_c2b/L1_absmax": 0.62919623, "gate_c2b/L1_rms": 0.06590044, "gate_c2b/L3_absmean": 0.18511081, "gate_c2b/L3_absmax": 0.41378883, "gate_c2b/L3_rms": 0.18905316, "gate_c2b/L5_absmean": 0.52848685, "gate_c2b/L5_absmax": 0.75113302, "gate_c2b/L5_rms": 0.5396769, "gate_b2a/L7_absmean": 0.05100295, "gate_b2a/L7_absmax": 0.32222, "gate_b2a/L7_rms": 0.05654107, "gate_b2a/L11_absmean": 0.08250893, "gate_b2a/L11_absmax": 0.60450137, "gate_b2a/L11_rms": 0.0916067, "gate_b2a/L14_absmean": 0.21244571, "gate_b2a/L14_absmax": 0.80017465, "gate_b2a/L14_rms": 0.22113959, "gate_b2c/L3_absmean": 0.0362142, "gate_b2c/L3_absmax": 0.56711119, "gate_b2c/L3_rms": 0.04732286, "gate_b2c/L5_absmean": 0.03849427, "gate_b2c/L5_absmax": 0.57416737, "gate_b2c/L5_rms": 0.04879814, "gate_b2c/L6_absmean": 0.05557751, "gate_b2c/L6_absmax": 0.3708863, "gate_b2c/L6_rms": 0.06224201}, "streams": {"cos_a2b/round0": 0.364538, "cos_c2b/round0": 0.301117, "cos_b2a/round0": 0.144942, "cos_b2c/round0": 0.134822, "rms_a/round0": 6.070678, "rms_b/round0": 10.923542, "rms_c/round0": 4.191862, "cos_a2b/round1": 0.411994, "cos_c2b/round1": 0.292141, "cos_b2a/round1": 0.241083, "cos_b2c/round1": 0.108567, "rms_a/round1": 11.88453, "rms_b/round1": 25.367403, "rms_c/round1": 8.805727, "cos_a2b/round2": 0.536177, "cos_c2b/round2": 0.524973, "cos_b2a/round2": 0.095088, "cos_b2c/round2": 0.17268, "rms_a/round2": 32.30135, "rms_b/round2": 67.466705, "rms_c/round2": 16.771837}, "alpha_mean": 0.4282497744071776, "alpha_hist": [0.0146, 0.0784, 0.19075, 0.19304, 0.16725, 0.15358, 0.12262, 0.06831, 0.01093, 0.00052], "alpha_by_type": {"special": {"share": 0.00122, "mean": 0.54771, "std": 0.13308}, "whitespace": {"share": 0.02826, "mean": 0.53215, "std": 0.1348}, "digit": {"share": 0.04761, "mean": 0.44033, "std": 0.12849}, "word_start": {"share": 0.55296, "mean": 0.44721, "std": 0.17366}, "word_cont": {"share": 0.26123, "mean": 0.3095, "std": 0.13047}, "punct": {"share": 0.10872, "mean": 0.58348, "std": 0.16088}}, "alpha_p05": 0.17078, "alpha_p50": 0.40898, "alpha_p95": 0.73754}
{"step": 48610, "total_steps": 72479, "tokens": 25485639680, "loss": 3.356396347284317, "l_mix": 2.0753593146800995, "l_a": 2.161585509777069, "l_c": 2.108537897467613, "z_loss": 44.89424133300781, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.36302080750465393, "tokens_per_second": 1320670.597818436, "eta_seconds": 9475.663570213319, "data_wait_seconds": 1.7881393432617188e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 48620, "total_steps": 72479, "tokens": 25490882560, "loss": 4.7994319796562195, "l_mix": 2.9775085747241974, "l_a": 3.0566115379333496, "l_c": 3.016466051340103, "z_loss": 36.735215187072754, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12873612344264984, "tokens_per_second": 1382211.2087859178, "eta_seconds": 9049.982602143287, "data_wait_seconds": 1.8596649169921875e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 48630, "total_steps": 72479, "tokens": 25496125440, "loss": 4.9337477684021, "l_mix": 3.0623122453689575, "l_a": 3.137163609266281, "l_c": 3.1009543538093567, "z_loss": 36.156731605529785, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.10359276831150055, "tokens_per_second": 1280347.677705604, "eta_seconds": 9765.897755527498, "data_wait_seconds": 5.844235420227051e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 48640, "total_steps": 72479, "tokens": 25501368320, "loss": 4.81809014081955, "l_mix": 2.989730268716812, "l_a": 3.0635866224765778, "l_c": 3.0309462547302246, "z_loss": 36.58271360397339, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18793226778507233, "tokens_per_second": 1278969.68336035, "eta_seconds": 9772.320481562614, "data_wait_seconds": 1.8537044525146484e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 48650, "total_steps": 72479, "tokens": 25506611200, "loss": 4.780735671520233, "l_mix": 2.966425836086273, "l_a": 3.0418303310871124, "l_c": 3.0058690905570984, "z_loss": 37.22172689437866, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1903766691684723, "tokens_per_second": 1362371.797648233, "eta_seconds": 9170.227080130577, "data_wait_seconds": 2.0503997802734375e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 48660, "total_steps": 72479, "tokens": 25511854080, "loss": 4.792544305324554, "l_mix": 2.9736268520355225, "l_a": 3.050777554512024, "l_c": 3.012280285358429, "z_loss": 37.92386531829834, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1738746017217636, "tokens_per_second": 1407204.3078758966, "eta_seconds": 8874.34454407692, "data_wait_seconds": 2.130866050720215e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 48670, "total_steps": 72479, "tokens": 25517096960, "loss": 4.320204377174377, "l_mix": 2.676383376121521, "l_a": 2.7639118432998657, "l_c": 2.7154915034770966, "z_loss": 39.467262744903564, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11318682879209518, "tokens_per_second": 1396575.1788676975, "eta_seconds": 8938.131781864167, "data_wait_seconds": 1.659989356994629e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 48680, "total_steps": 72479, "tokens": 25522339840, "loss": 4.1861980855464935, "l_mix": 2.5928314328193665, "l_a": 2.6803492307662964, "l_c": 2.6308723390102386, "z_loss": 39.790263652801514, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13242489099502563, "tokens_per_second": 1369799.4224677745, "eta_seconds": 9109.019837021828, "data_wait_seconds": 1.6808509826660156e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 48690, "total_steps": 72479, "tokens": 25527582720, "loss": 4.374193370342255, "l_mix": 2.7109873592853546, "l_a": 2.7956758737564087, "l_c": 2.7483440339565277, "z_loss": 39.31044578552246, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12752750515937805, "tokens_per_second": 1383609.4758893654, "eta_seconds": 9014.311804986, "data_wait_seconds": 1.7642974853515625e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 48700, "total_steps": 72479, "tokens": 25532825600, "loss": 4.313295125961304, "l_mix": 2.673244893550873, "l_a": 2.758736252784729, "l_c": 2.708097219467163, "z_loss": 38.72529983520508, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1645651012659073, "tokens_per_second": 1412411.475077656, "eta_seconds": 8826.7792863369, "data_wait_seconds": 2.73287296295166e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.18327658, "gate_a2b/L1_absmax": 0.71591651, "gate_a2b/L1_rms": 0.19477193, "gate_a2b/L3_absmean": 0.17795615, "gate_a2b/L3_absmax": 0.55845475, "gate_a2b/L3_rms": 0.18410218, "gate_a2b/L5_absmean": 0.27131858, "gate_a2b/L5_absmax": 0.73702472, "gate_a2b/L5_rms": 0.27728736, "gate_c2b/L1_absmean": 0.02138791, "gate_c2b/L1_absmax": 0.62903404, "gate_c2b/L1_rms": 0.06559921, "gate_c2b/L3_absmean": 0.18469939, "gate_c2b/L3_absmax": 0.41863716, "gate_c2b/L3_rms": 0.18868133, "gate_c2b/L5_absmean": 0.52815723, "gate_c2b/L5_absmax": 0.74844259, "gate_c2b/L5_rms": 0.53924876, "gate_b2a/L7_absmean": 0.05047455, "gate_b2a/L7_absmax": 0.32398948, "gate_b2a/L7_rms": 0.05612534, "gate_b2a/L11_absmean": 0.08192116, "gate_b2a/L11_absmax": 0.60321617, "gate_b2a/L11_rms": 0.09103393, "gate_b2a/L14_absmean": 0.21231492, "gate_b2a/L14_absmax": 0.79821861, "gate_b2a/L14_rms": 0.22092077, "gate_b2c/L3_absmean": 0.03628093, "gate_b2c/L3_absmax": 0.56720567, "gate_b2c/L3_rms": 0.04733838, "gate_b2c/L5_absmean": 0.03860683, "gate_b2c/L5_absmax": 0.57346421, "gate_b2c/L5_rms": 0.0488293, "gate_b2c/L6_absmean": 0.05570196, "gate_b2c/L6_absmax": 0.36906219, "gate_b2c/L6_rms": 0.06219625}, "streams": {"cos_a2b/round0": 0.366001, "cos_c2b/round0": 0.32875, "cos_b2a/round0": 0.152724, "cos_b2c/round0": 0.151188, "rms_a/round0": 6.082165, "rms_b/round0": 10.816833, "rms_c/round0": 4.068887, "cos_a2b/round1": 0.430868, "cos_c2b/round1": 0.308035, "cos_b2a/round1": 0.24834, "cos_b2c/round1": 0.103071, "rms_a/round1": 11.664843, "rms_b/round1": 24.573446, "rms_c/round1": 8.456862, "cos_a2b/round2": 0.56348, "cos_c2b/round2": 0.516538, "cos_b2a/round2": 0.114177, "cos_b2c/round2": 0.175001, "rms_a/round2": 32.529381, "rms_b/round2": 66.407753, "rms_c/round2": 15.972442}, "alpha_mean": 0.448492504096599, "alpha_hist": [0.00694, 0.05696, 0.16006, 0.18941, 0.18803, 0.18202, 0.14603, 0.06348, 0.00703, 3e-05], "alpha_by_type": {"special": {"share": 0.00046, "mean": 0.57836, "std": 0.13211}, "whitespace": {"share": 0.01427, "mean": 0.53342, "std": 0.12607}, "digit": {"share": 0.02705, "mean": 0.44366, "std": 0.14512}, "word_start": {"share": 0.67029, "mean": 0.45594, "std": 0.1589}, "word_cont": {"share": 0.1841, "mean": 0.32825, "std": 0.13022}, "punct": {"share": 0.10384, "mean": 0.60259, "std": 0.14071}}, "alpha_p05": 0.17753, "alpha_p50": 0.42895, "alpha_p95": 0.72037}
{"step": 48710, "total_steps": 72479, "tokens": 25538068480, "loss": 4.339511573314667, "l_mix": 2.6892430782318115, "l_a": 2.7713871896266937, "l_c": 2.7295075356960297, "z_loss": 39.01670265197754, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1987922191619873, "tokens_per_second": 1322476.4649802132, "eta_seconds": 9423.079957938195, "data_wait_seconds": 1.5139579772949219e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 48720, "total_steps": 72479, "tokens": 25543311360, "loss": 3.0971945226192474, "l_mix": 1.9170526266098022, "l_a": 1.9871956259012222, "l_c": 1.9466106593608856, "z_loss": 46.587502002716064, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1975286453962326, "tokens_per_second": 1360450.7811571332, "eta_seconds": 9156.199374890328, "data_wait_seconds": 1.621246337890625e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 48730, "total_steps": 72479, "tokens": 25548554240, "loss": 3.0665176808834076, "l_mix": 1.898552492260933, "l_a": 1.9663895815610886, "l_c": 1.9268274456262589, "z_loss": 46.02477312088013, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20640887320041656, "tokens_per_second": 1432418.3229262088, "eta_seconds": 8692.513571429252, "data_wait_seconds": 1.7523765563964844e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 48740, "total_steps": 72479, "tokens": 25553797120, "loss": 4.353414595127106, "l_mix": 2.6978666484355927, "l_a": 2.782039850950241, "l_c": 2.73645281791687, "z_loss": 38.3321213722229, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14018097519874573, "tokens_per_second": 1426992.0283656279, "eta_seconds": 8721.893734931946, "data_wait_seconds": 1.9669532775878906e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 48750, "total_steps": 72479, "tokens": 25559040000, "loss": 4.253391861915588, "l_mix": 2.6353565752506256, "l_a": 2.720817655324936, "l_c": 2.6726330518722534, "z_loss": 37.868722915649414, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19242672622203827, "tokens_per_second": 1449635.014605627, "eta_seconds": 8582.042946434021, "data_wait_seconds": 2.0951032638549805e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 48760, "total_steps": 72479, "tokens": 25564282880, "loss": 4.280756175518036, "l_mix": 2.652316391468048, "l_a": 2.738306760787964, "l_c": 2.6898260414600372, "z_loss": 39.236417293548584, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12904758751392365, "tokens_per_second": 1408612.2987476871, "eta_seconds": 8828.253936910629, "data_wait_seconds": 3.463029861450195e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 48770, "total_steps": 72479, "tokens": 25569525760, "loss": 4.328922390937805, "l_mix": 2.68262842297554, "l_a": 2.76795294880867, "l_c": 2.7196933031082153, "z_loss": 39.391297817230225, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12456440180540085, "tokens_per_second": 1405741.2547722142, "eta_seconds": 8842.554879713058, "data_wait_seconds": 1.800060272216797e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 48780, "total_steps": 72479, "tokens": 25574768640, "loss": 4.330097079277039, "l_mix": 2.682898700237274, "l_a": 2.7691771388053894, "l_c": 2.721484065055847, "z_loss": 39.058104038238525, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13420113921165466, "tokens_per_second": 1459410.5036439416, "eta_seconds": 8513.780927968026, "data_wait_seconds": 2.276897430419922e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 48790, "total_steps": 72479, "tokens": 25580011520, "loss": 4.368904709815979, "l_mix": 2.7071030139923096, "l_a": 2.7956916093826294, "l_c": 2.743646889925003, "z_loss": 39.8300347328186, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18095612525939941, "tokens_per_second": 1473789.1519622735, "eta_seconds": 8427.160978531838, "data_wait_seconds": 2.753734588623047e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 48800, "total_steps": 72479, "tokens": 25585254400, "loss": 4.372899234294891, "l_mix": 2.7104427814483643, "l_a": 2.7945895195007324, "l_c": 2.746931701898575, "z_loss": 38.93312215805054, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14652955532073975, "tokens_per_second": 1389536.1737823142, "eta_seconds": 8934.359382820128, "data_wait_seconds": 1.558661460876465e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.1827319, "gate_a2b/L1_absmax": 0.72013837, "gate_a2b/L1_rms": 0.19444716, "gate_a2b/L3_absmean": 0.17781425, "gate_a2b/L3_absmax": 0.56331563, "gate_a2b/L3_rms": 0.18406689, "gate_a2b/L5_absmean": 0.26996785, "gate_a2b/L5_absmax": 0.73529601, "gate_a2b/L5_rms": 0.27616161, "gate_c2b/L1_absmean": 0.02166431, "gate_c2b/L1_absmax": 0.63348907, "gate_c2b/L1_rms": 0.06611466, "gate_c2b/L3_absmean": 0.1851683, "gate_c2b/L3_absmax": 0.42810646, "gate_c2b/L3_rms": 0.18926862, "gate_c2b/L5_absmean": 0.52686745, "gate_c2b/L5_absmax": 0.74035871, "gate_c2b/L5_rms": 0.53801996, "gate_b2a/L7_absmean": 0.05085568, "gate_b2a/L7_absmax": 0.32246909, "gate_b2a/L7_rms": 0.05652012, "gate_b2a/L11_absmean": 0.08171585, "gate_b2a/L11_absmax": 0.6017527, "gate_b2a/L11_rms": 0.09116965, "gate_b2a/L14_absmean": 0.21057907, "gate_b2a/L14_absmax": 0.8061381, "gate_b2a/L14_rms": 0.21971244, "gate_b2c/L3_absmean": 0.03585368, "gate_b2c/L3_absmax": 0.57366937, "gate_b2c/L3_rms": 0.04734674, "gate_b2c/L5_absmean": 0.03874127, "gate_b2c/L5_absmax": 0.58191466, "gate_b2c/L5_rms": 0.04936235, "gate_b2c/L6_absmean": 0.05508634, "gate_b2c/L6_absmax": 0.37822568, "gate_b2c/L6_rms": 0.0620017}, "streams": {"cos_a2b/round0": 0.365391, "cos_c2b/round0": 0.298763, "cos_b2a/round0": 0.149015, "cos_b2c/round0": 0.137174, "rms_a/round0": 6.165959, "rms_b/round0": 11.123311, "rms_c/round0": 4.240808, "cos_a2b/round1": 0.413374, "cos_c2b/round1": 0.296897, "cos_b2a/round1": 0.248233, "cos_b2c/round1": 0.111863, "rms_a/round1": 11.886985, "rms_b/round1": 25.445957, "rms_c/round1": 8.821374, "cos_a2b/round2": 0.538149, "cos_c2b/round2": 0.526987, "cos_b2a/round2": 0.101747, "cos_b2c/round2": 0.179554, "rms_a/round2": 32.870888, "rms_b/round2": 68.615631, "rms_c/round2": 17.084166}, "alpha_mean": 0.43490477328333554, "alpha_hist": [0.01331, 0.08203, 0.16812, 0.18626, 0.18138, 0.16209, 0.12576, 0.06783, 0.01302, 0.0002], "alpha_by_type": {"special": {"share": 0.00151, "mean": 0.60612, "std": 0.12052}, "whitespace": {"share": 0.0294, "mean": 0.53072, "std": 0.14236}, "digit": {"share": 0.04778, "mean": 0.44834, "std": 0.12328}, "word_start": {"share": 0.56697, "mean": 0.44344, "std": 0.17338}, "word_cont": {"share": 0.25412, "mean": 0.34045, "std": 0.1511}, "punct": {"share": 0.10022, "mean": 0.58904, "std": 0.16088}}, "alpha_p05": 0.17554, "alpha_p50": 0.4206, "alpha_p95": 0.7531}
{"step": 48810, "total_steps": 72479, "tokens": 25590497280, "loss": 4.365022420883179, "l_mix": 2.704042822122574, "l_a": 2.7926447987556458, "l_c": 2.74395352602005, "z_loss": 38.949204444885254, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13213084638118744, "tokens_per_second": 1464031.643222657, "eta_seconds": 8476.164247846604, "data_wait_seconds": 2.7120113372802734e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 48820, "total_steps": 72479, "tokens": 25595740160, "loss": 4.289557993412018, "l_mix": 2.6578412652015686, "l_a": 2.7449587285518646, "l_c": 2.694096624851227, "z_loss": 39.32808256149292, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16132083535194397, "tokens_per_second": 1354871.0089270822, "eta_seconds": 9155.21087267399, "data_wait_seconds": 2.473592758178711e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 48830, "total_steps": 72479, "tokens": 25600983040, "loss": 4.46990042924881, "l_mix": 2.769836276769638, "l_a": 2.857183516025543, "l_c": 2.8096969425678253, "z_loss": 39.18013381958008, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17611731588840485, "tokens_per_second": 1343391.530804424, "eta_seconds": 9229.540776228903, "data_wait_seconds": 2.0802021026611328e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 48840, "total_steps": 72479, "tokens": 25606225920, "loss": 2.7486164271831512, "l_mix": 1.7008128762245178, "l_a": 1.7657365947961807, "l_c": 1.7269419133663177, "z_loss": 46.662118434906006, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18367266654968262, "tokens_per_second": 1480622.3365925981, "eta_seconds": 8370.564002513885, "data_wait_seconds": 2.5272369384765625e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 48850, "total_steps": 72479, "tokens": 25611468800, "loss": 4.356405735015869, "l_mix": 2.6989735066890717, "l_a": 2.7881557047367096, "l_c": 2.736618220806122, "z_loss": 40.30376672744751, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.23610574007034302, "tokens_per_second": 1439692.6433086263, "eta_seconds": 8604.893002390862, "data_wait_seconds": 2.41696834564209e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 48860, "total_steps": 72479, "tokens": 25616711680, "loss": 4.428222835063934, "l_mix": 2.7444223761558533, "l_a": 2.8298171162605286, "l_c": 2.7828508019447327, "z_loss": 39.161513328552246, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13297858834266663, "tokens_per_second": 1411345.89151672, "eta_seconds": 8774.00667436123, "data_wait_seconds": 2.828240394592285e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 48870, "total_steps": 72479, "tokens": 25621954560, "loss": 4.362262725830078, "l_mix": 2.7035599052906036, "l_a": 2.784997522830963, "l_c": 2.744011700153351, "z_loss": 38.08736801147461, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18236324191093445, "tokens_per_second": 1470438.1896589769, "eta_seconds": 8417.841347599031, "data_wait_seconds": 2.8461217880249023e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 48880, "total_steps": 72479, "tokens": 25627197440, "loss": 4.391825556755066, "l_mix": 2.7208534479141235, "l_a": 2.809648722410202, "l_c": 2.7602583467960358, "z_loss": 39.06264781951904, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15727247297763824, "tokens_per_second": 1423766.2833385367, "eta_seconds": 8690.1007958889, "data_wait_seconds": 1.7315149307250977e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 48890, "total_steps": 72479, "tokens": 25632440320, "loss": 4.389610230922699, "l_mix": 2.7205464839935303, "l_a": 2.8057310581207275, "l_c": 2.7578141689300537, "z_loss": 38.7906756401062, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11344806849956512, "tokens_per_second": 1436274.753876121, "eta_seconds": 8610.76865594387, "data_wait_seconds": 2.4944543838500977e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 48900, "total_steps": 72479, "tokens": 25637683200, "loss": 4.803503215312958, "l_mix": 2.9802485704421997, "l_a": 3.0559918582439423, "l_c": 3.021523743867874, "z_loss": 36.85538387298584, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14998942613601685, "tokens_per_second": 1247158.762795294, "eta_seconds": 9912.279912376403, "data_wait_seconds": 2.5331974029541016e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.18268052, "gate_a2b/L1_absmax": 0.72068179, "gate_a2b/L1_rms": 0.19421305, "gate_a2b/L3_absmean": 0.17704216, "gate_a2b/L3_absmax": 0.55520719, "gate_a2b/L3_rms": 0.18323842, "gate_a2b/L5_absmean": 0.27000329, "gate_a2b/L5_absmax": 0.73521304, "gate_a2b/L5_rms": 0.27602762, "gate_c2b/L1_absmean": 0.02187381, "gate_c2b/L1_absmax": 0.63319677, "gate_c2b/L1_rms": 0.06616895, "gate_c2b/L3_absmean": 0.18531001, "gate_c2b/L3_absmax": 0.42048612, "gate_c2b/L3_rms": 0.18927822, "gate_c2b/L5_absmean": 0.52718228, "gate_c2b/L5_absmax": 0.75378513, "gate_c2b/L5_rms": 0.53828967, "gate_b2a/L7_absmean": 0.05079455, "gate_b2a/L7_absmax": 0.32602617, "gate_b2a/L7_rms": 0.0563175, "gate_b2a/L11_absmean": 0.08246023, "gate_b2a/L11_absmax": 0.60403794, "gate_b2a/L11_rms": 0.09153666, "gate_b2a/L14_absmean": 0.21199518, "gate_b2a/L14_absmax": 0.80038631, "gate_b2a/L14_rms": 0.22083086, "gate_b2c/L3_absmean": 0.03602855, "gate_b2c/L3_absmax": 0.57962883, "gate_b2c/L3_rms": 0.04758362, "gate_b2c/L5_absmean": 0.03865214, "gate_b2c/L5_absmax": 0.58161312, "gate_b2c/L5_rms": 0.04909454, "gate_b2c/L6_absmean": 0.05539852, "gate_b2c/L6_absmax": 0.36566058, "gate_b2c/L6_rms": 0.06197193}, "streams": {"cos_a2b/round0": 0.364462, "cos_c2b/round0": 0.30485, "cos_b2a/round0": 0.154973, "cos_b2c/round0": 0.138476, "rms_a/round0": 6.148126, "rms_b/round0": 10.96557, "rms_c/round0": 4.161806, "cos_a2b/round1": 0.421145, "cos_c2b/round1": 0.309743, "cos_b2a/round1": 0.251251, "cos_b2c/round1": 0.114802, "rms_a/round1": 11.738815, "rms_b/round1": 24.561453, "rms_c/round1": 8.595545, "cos_a2b/round2": 0.566303, "cos_c2b/round2": 0.515262, "cos_b2a/round2": 0.111212, "cos_b2c/round2": 0.184293, "rms_a/round2": 32.616314, "rms_b/round2": 65.820091, "rms_c/round2": 16.098932}, "alpha_mean": 0.4770975052109492, "alpha_hist": [0.00302, 0.03481, 0.12715, 0.17642, 0.19736, 0.20943, 0.15891, 0.08131, 0.01146, 0.00014], "alpha_by_type": {"special": {"share": 0.00024, "mean": 0.5279, "std": 0.15624}, "whitespace": {"share": 0.05086, "mean": 0.53368, "std": 0.12601}, "digit": {"share": 0.02171, "mean": 0.4394, "std": 0.12598}, "word_start": {"share": 0.55522, "mean": 0.49552, "std": 0.14848}, "word_cont": {"share": 0.24294, "mean": 0.35506, "std": 0.13154}, "punct": {"share": 0.12903, "mean": 0.61153, "std": 0.13967}}, "alpha_p05": 0.19683, "alpha_p50": 0.46235, "alpha_p95": 0.74017}
{"step": 48910, "total_steps": 72479, "tokens": 25642926080, "loss": 4.807629585266113, "l_mix": 2.982863038778305, "l_a": 3.064393639564514, "l_c": 3.018161028623581, "z_loss": 36.88810729980469, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15639100968837738, "tokens_per_second": 1345135.9386805561, "eta_seconds": 9186.390398669242, "data_wait_seconds": 2.8699636459350586e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 48920, "total_steps": 72479, "tokens": 25648168960, "loss": 4.752431929111481, "l_mix": 2.9483804404735565, "l_a": 3.0249692797660828, "l_c": 2.988535076379776, "z_loss": 37.47454643249512, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1644565612077713, "tokens_per_second": 1370991.0193568917, "eta_seconds": 9009.32304997444, "data_wait_seconds": 2.0802021026611328e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 48930, "total_steps": 72479, "tokens": 25653411840, "loss": 4.838184058666229, "l_mix": 3.0026509761810303, "l_a": 3.0790465772151947, "l_c": 3.03939688205719, "z_loss": 35.68545055389404, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.163554847240448, "tokens_per_second": 1415599.1294524355, "eta_seconds": 8721.71920363903, "data_wait_seconds": 3.1322240829467773e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 48940, "total_steps": 72479, "tokens": 25658654720, "loss": 4.761287152767181, "l_mix": 2.9538147151470184, "l_a": 3.031649500131607, "l_c": 2.993258684873581, "z_loss": 37.003525733947754, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17524845898151398, "tokens_per_second": 1269740.0955268666, "eta_seconds": 9719.481392669679, "data_wait_seconds": 1.722574234008789e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 48950, "total_steps": 72479, "tokens": 25663897600, "loss": 2.909957528114319, "l_mix": 1.8005018681287766, "l_a": 1.8694124668836594, "l_c": 1.8287728875875473, "z_loss": 48.32374286651611, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14912746846675873, "tokens_per_second": 1431068.0584770264, "eta_seconds": 8620.115779209138, "data_wait_seconds": 1.6450881958007812e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 48960, "total_steps": 72479, "tokens": 25669140480, "loss": 4.349937379360199, "l_mix": 2.695224106311798, "l_a": 2.7813844978809357, "l_c": 2.734326183795929, "z_loss": 39.34595727920532, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13464263081550598, "tokens_per_second": 1468967.3446147586, "eta_seconds": 8394.148118543624, "data_wait_seconds": 2.9653310775756836e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 48970, "total_steps": 72479, "tokens": 25674383360, "loss": 4.409533798694611, "l_mix": 2.7324250042438507, "l_a": 2.817706435918808, "l_c": 2.7726559937000275, "z_loss": 38.828675746917725, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14155955612659454, "tokens_per_second": 1391028.7599283706, "eta_seconds": 8860.698604559899, "data_wait_seconds": 2.5391578674316406e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 48980, "total_steps": 72479, "tokens": 25679626240, "loss": 4.389949321746826, "l_mix": 2.721771001815796, "l_a": 2.805711805820465, "l_c": 2.754882514476776, "z_loss": 38.882155895233154, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.32309845089912415, "tokens_per_second": 1395931.6390190697, "eta_seconds": 8825.821671795846, "data_wait_seconds": 2.16066837310791e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 48990, "total_steps": 72479, "tokens": 25684869120, "loss": 4.240624189376831, "l_mix": 2.627208322286606, "l_a": 2.714176118373871, "l_c": 2.663876473903656, "z_loss": 38.61830139160156, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17967846989631653, "tokens_per_second": 1415399.6788653648, "eta_seconds": 8700.723206233979, "data_wait_seconds": 1.7702579498291016e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 49000, "total_steps": 72479, "tokens": 25690112000, "loss": 4.387688100337982, "l_mix": 2.718467593193054, "l_a": 2.8044563233852386, "l_c": 2.759611964225769, "z_loss": 38.13833999633789, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15520446002483368, "tokens_per_second": 1307288.0720897245, "eta_seconds": 9416.255081653595, "data_wait_seconds": 2.8967857360839844e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.18208812, "gate_a2b/L1_absmax": 0.72000891, "gate_a2b/L1_rms": 0.19368477, "gate_a2b/L3_absmean": 0.17685728, "gate_a2b/L3_absmax": 0.55789626, "gate_a2b/L3_rms": 0.18301074, "gate_a2b/L5_absmean": 0.26957992, "gate_a2b/L5_absmax": 0.7328316, "gate_a2b/L5_rms": 0.27558792, "gate_c2b/L1_absmean": 0.02130781, "gate_c2b/L1_absmax": 0.63570982, "gate_c2b/L1_rms": 0.06594177, "gate_c2b/L3_absmean": 0.18487816, "gate_c2b/L3_absmax": 0.41583869, "gate_c2b/L3_rms": 0.18879636, "gate_c2b/L5_absmean": 0.52600998, "gate_c2b/L5_absmax": 0.75049043, "gate_c2b/L5_rms": 0.53722745, "gate_b2a/L7_absmean": 0.05038281, "gate_b2a/L7_absmax": 0.32615918, "gate_b2a/L7_rms": 0.05603852, "gate_b2a/L11_absmean": 0.08224418, "gate_b2a/L11_absmax": 0.6004681, "gate_b2a/L11_rms": 0.09126647, "gate_b2a/L14_absmean": 0.21207467, "gate_b2a/L14_absmax": 0.7969467, "gate_b2a/L14_rms": 0.2208046, "gate_b2c/L3_absmean": 0.03596585, "gate_b2c/L3_absmax": 0.57580537, "gate_b2c/L3_rms": 0.04734108, "gate_b2c/L5_absmean": 0.03863418, "gate_b2c/L5_absmax": 0.56972069, "gate_b2c/L5_rms": 0.04870385, "gate_b2c/L6_absmean": 0.05560326, "gate_b2c/L6_absmax": 0.36487046, "gate_b2c/L6_rms": 0.06207113}, "streams": {"cos_a2b/round0": 0.363459, "cos_c2b/round0": 0.292421, "cos_b2a/round0": 0.146603, "cos_b2c/round0": 0.134686, "rms_a/round0": 6.167937, "rms_b/round0": 11.147182, "rms_c/round0": 4.254823, "cos_a2b/round1": 0.409074, "cos_c2b/round1": 0.295042, "cos_b2a/round1": 0.245444, "cos_b2c/round1": 0.111917, "rms_a/round1": 12.018809, "rms_b/round1": 25.807741, "rms_c/round1": 8.926777, "cos_a2b/round2": 0.531353, "cos_c2b/round2": 0.523168, "cos_b2a/round2": 0.097012, "cos_b2c/round2": 0.17404, "rms_a/round2": 32.81403, "rms_b/round2": 68.416985, "rms_c/round2": 17.245436}, "alpha_mean": 0.4204802886268908, "alpha_hist": [0.01578, 0.08298, 0.1833, 0.20743, 0.18047, 0.14868, 0.11226, 0.05933, 0.00932, 0.00046], "alpha_by_type": {"special": {"share": 0.00186, "mean": 0.54633, "std": 0.13048}, "whitespace": {"share": 0.03223, "mean": 0.495, "std": 0.13527}, "digit": {"share": 0.04956, "mean": 0.40413, "std": 0.14042}, "word_start": {"share": 0.54491, "mean": 0.44133, "std": 0.16934}, "word_cont": {"share": 0.27168, "mean": 0.31673, "std": 0.13577}, "punct": {"share": 0.09976, "mean": 0.57081, "std": 0.16493}}, "alpha_p05": 0.17053, "alpha_p50": 0.413, "alpha_p95": 0.70028}
{"step": 49010, "total_steps": 72479, "tokens": 25695354880, "loss": 3.1089926064014435, "l_mix": 1.9242624789476395, "l_a": 1.9943864047527313, "l_c": 1.9547138512134552, "z_loss": 46.80108404159546, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1919218748807907, "tokens_per_second": 1260364.875814679, "eta_seconds": 9762.661042141915, "data_wait_seconds": 2.6047229766845703e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 49020, "total_steps": 72479, "tokens": 25700597760, "loss": 4.891523659229279, "l_mix": 3.0365094542503357, "l_a": 3.109797716140747, "l_c": 3.0735829174518585, "z_loss": 36.0051383972168, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15390995144844055, "tokens_per_second": 1349088.3109464138, "eta_seconds": 9116.728750967979, "data_wait_seconds": 1.7404556274414062e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 49030, "total_steps": 72479, "tokens": 25705840640, "loss": 4.8733009696006775, "l_mix": 3.0239152312278748, "l_a": 3.10127654671669, "l_c": 3.063342332839966, "z_loss": 35.484171867370605, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2541021704673767, "tokens_per_second": 1374215.1811688892, "eta_seconds": 8946.218525648117, "data_wait_seconds": 1.823902130126953e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 49040, "total_steps": 72479, "tokens": 25711083520, "loss": 4.7528926730155945, "l_mix": 2.9492335617542267, "l_a": 3.0270437002182007, "l_c": 2.9851534962654114, "z_loss": 36.66784715652466, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15581092238426208, "tokens_per_second": 1202944.658977822, "eta_seconds": 10215.587508773802, "data_wait_seconds": 3.719329833984375e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 49050, "total_steps": 72479, "tokens": 25716326400, "loss": 4.627280563116074, "l_mix": 2.8702951967716217, "l_a": 2.947100967168808, "l_c": 2.909516543149948, "z_loss": 38.14971590042114, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18965445458889008, "tokens_per_second": 1420136.7550444352, "eta_seconds": 8649.549776363372, "data_wait_seconds": 3.17990779876709e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 49060, "total_steps": 72479, "tokens": 25721569280, "loss": 2.806627571582794, "l_mix": 1.7372865229845047, "l_a": 1.799679383635521, "l_c": 1.7647904604673386, "z_loss": 46.158233642578125, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.22879041731357574, "tokens_per_second": 1402841.17023658, "eta_seconds": 8752.452474665643, "data_wait_seconds": 2.5480985641479492e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 49070, "total_steps": 72479, "tokens": 25726812160, "loss": 4.357788562774658, "l_mix": 2.7006295919418335, "l_a": 2.787998080253601, "l_c": 2.735864818096161, "z_loss": 38.71208906173706, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1254100501537323, "tokens_per_second": 1460502.4962453395, "eta_seconds": 8403.31175301075, "data_wait_seconds": 1.5676021575927734e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 49080, "total_steps": 72479, "tokens": 25732055040, "loss": 4.455708980560303, "l_mix": 2.7621727287769318, "l_a": 2.84621062874794, "l_c": 2.798910081386566, "z_loss": 38.340735912323, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1230330765247345, "tokens_per_second": 1474070.3152606895, "eta_seconds": 8322.408222317696, "data_wait_seconds": 1.6868114471435547e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 49090, "total_steps": 72479, "tokens": 25737297920, "loss": 4.5092214941978455, "l_mix": 2.7945417761802673, "l_a": 2.882992058992386, "l_c": 2.832606792449951, "z_loss": 37.92265224456787, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12377020716667175, "tokens_per_second": 1347381.6072481135, "eta_seconds": 9101.038611507414, "data_wait_seconds": 1.5795230865478516e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 49100, "total_steps": 72479, "tokens": 25742540800, "loss": 4.361878752708435, "l_mix": 2.703136682510376, "l_a": 2.7867473661899567, "l_c": 2.7423925399780273, "z_loss": 38.89524984359741, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13312746584415436, "tokens_per_second": 1443851.4218089513, "eta_seconds": 8489.32858800888, "data_wait_seconds": 1.5556812286376953e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.18220355, "gate_a2b/L1_absmax": 0.71771395, "gate_a2b/L1_rms": 0.19367962, "gate_a2b/L3_absmean": 0.17684111, "gate_a2b/L3_absmax": 0.55781311, "gate_a2b/L3_rms": 0.18301168, "gate_a2b/L5_absmean": 0.26872274, "gate_a2b/L5_absmax": 0.73449987, "gate_a2b/L5_rms": 0.2747547, "gate_c2b/L1_absmean": 0.02176755, "gate_c2b/L1_absmax": 0.63330269, "gate_c2b/L1_rms": 0.06594756, "gate_c2b/L3_absmean": 0.18472745, "gate_c2b/L3_absmax": 0.42028341, "gate_c2b/L3_rms": 0.18873401, "gate_c2b/L5_absmean": 0.52566415, "gate_c2b/L5_absmax": 0.75011086, "gate_c2b/L5_rms": 0.53674614, "gate_b2a/L7_absmean": 0.05085663, "gate_b2a/L7_absmax": 0.32509634, "gate_b2a/L7_rms": 0.0564333, "gate_b2a/L11_absmean": 0.08216649, "gate_b2a/L11_absmax": 0.60169166, "gate_b2a/L11_rms": 0.09127875, "gate_b2a/L14_absmean": 0.21168207, "gate_b2a/L14_absmax": 0.79785889, "gate_b2a/L14_rms": 0.22044869, "gate_b2c/L3_absmean": 0.03603037, "gate_b2c/L3_absmax": 0.57309985, "gate_b2c/L3_rms": 0.04727447, "gate_b2c/L5_absmean": 0.03837353, "gate_b2c/L5_absmax": 0.56787562, "gate_b2c/L5_rms": 0.0484588, "gate_b2c/L6_absmean": 0.05530688, "gate_b2c/L6_absmax": 0.36762625, "gate_b2c/L6_rms": 0.0618999}, "streams": {"cos_a2b/round0": 0.365963, "cos_c2b/round0": 0.298693, "cos_b2a/round0": 0.148101, "cos_b2c/round0": 0.134614, "rms_a/round0": 6.139877, "rms_b/round0": 11.042615, "rms_c/round0": 4.259754, "cos_a2b/round1": 0.41343, "cos_c2b/round1": 0.299689, "cos_b2a/round1": 0.247721, "cos_b2c/round1": 0.110155, "rms_a/round1": 11.877516, "rms_b/round1": 25.437977, "rms_c/round1": 8.840513, "cos_a2b/round2": 0.534752, "cos_c2b/round2": 0.529075, "cos_b2a/round2": 0.093132, "cos_b2c/round2": 0.175923, "rms_a/round2": 32.613758, "rms_b/round2": 67.88372, "rms_c/round2": 16.746439}, "alpha_mean": 0.4216763266695196, "alpha_hist": [0.00755, 0.06316, 0.18907, 0.22748, 0.19202, 0.15814, 0.1096, 0.04784, 0.00499, 0.00015], "alpha_by_type": {"special": {"share": 0.00102, "mean": 0.52017, "std": 0.12638}, "whitespace": {"share": 0.02763, "mean": 0.50182, "std": 0.13866}, "digit": {"share": 0.04185, "mean": 0.44917, "std": 0.10658}, "word_start": {"share": 0.56125, "mean": 0.43431, "std": 0.15954}, "word_cont": {"share": 0.26712, "mean": 0.3311, "std": 0.12351}, "punct": {"share": 0.10112, "mean": 0.55654, "std": 0.15496}}, "alpha_p05": 0.17955, "alpha_p50": 0.40757, "alpha_p95": 0.70436}
{"step": 49110, "total_steps": 72479, "tokens": 25747783680, "loss": 4.4302056431770325, "l_mix": 2.745670735836029, "l_a": 2.8329531848430634, "l_c": 2.7821629643440247, "z_loss": 37.79129695892334, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15229402482509613, "tokens_per_second": 1419748.548696602, "eta_seconds": 8629.75791258812, "data_wait_seconds": 1.7076730728149414e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 49120, "total_steps": 72479, "tokens": 25753026560, "loss": 3.0414803624153137, "l_mix": 1.8831132501363754, "l_a": 1.9506072849035263, "l_c": 1.9106163084506989, "z_loss": 45.476107597351074, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13655902445316315, "tokens_per_second": 1315870.2358338851, "eta_seconds": 9307.0297195673, "data_wait_seconds": 1.8417835235595703e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 49130, "total_steps": 72479, "tokens": 25758269440, "loss": 2.926829993724823, "l_mix": 1.8124350905418396, "l_a": 1.8741864711046219, "l_c": 1.8404632657766342, "z_loss": 46.15750694274902, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20613926649093628, "tokens_per_second": 1437956.6554828957, "eta_seconds": 8513.191594004631, "data_wait_seconds": 2.2023916244506836e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 49140, "total_steps": 72479, "tokens": 25763512320, "loss": 4.7560805678367615, "l_mix": 2.9509566724300385, "l_a": 3.0291957557201385, "l_c": 2.987883508205414, "z_loss": 37.29823398590088, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11526836454868317, "tokens_per_second": 1458233.6882318736, "eta_seconds": 8391.218589138985, "data_wait_seconds": 3.057718276977539e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 49150, "total_steps": 72479, "tokens": 25768755200, "loss": 4.799859344959259, "l_mix": 2.9775936603546143, "l_a": 3.0564360916614532, "l_c": 3.0177824795246124, "z_loss": 37.132668018341064, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17865124344825745, "tokens_per_second": 1358057.1993265986, "eta_seconds": 9006.33254480362, "data_wait_seconds": 1.8090009689331055e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 49160, "total_steps": 72479, "tokens": 25773998080, "loss": 4.747921168804169, "l_mix": 2.9464783370494843, "l_a": 3.0205219984054565, "l_c": 2.9842870235443115, "z_loss": 36.53285551071167, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.21119192242622375, "tokens_per_second": 1448598.9979489925, "eta_seconds": 8439.790369391441, "data_wait_seconds": 2.065300941467285e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 49170, "total_steps": 72479, "tokens": 25779240960, "loss": 4.803139626979828, "l_mix": 2.980292946100235, "l_a": 3.0548303425312042, "l_c": 3.0213249921798706, "z_loss": 37.766597270965576, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18917377293109894, "tokens_per_second": 1453289.0514775445, "eta_seconds": 8408.945886969566, "data_wait_seconds": 2.6553869247436523e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 49180, "total_steps": 72479, "tokens": 25784483840, "loss": 4.544872403144836, "l_mix": 2.817631781101227, "l_a": 2.9026152789592743, "l_c": 2.8548534214496613, "z_loss": 38.29961824417114, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14781615138053894, "tokens_per_second": 1432361.035288404, "eta_seconds": 8528.147450995446, "data_wait_seconds": 2.22623348236084e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 49190, "total_steps": 72479, "tokens": 25789726720, "loss": 4.470604181289673, "l_mix": 2.7709286510944366, "l_a": 2.8553668558597565, "l_c": 2.8102179169654846, "z_loss": 38.15761041641235, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13871115446090698, "tokens_per_second": 1410690.8421613376, "eta_seconds": 8655.435242843629, "data_wait_seconds": 2.014636993408203e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 49200, "total_steps": 72479, "tokens": 25794969600, "loss": 4.525574803352356, "l_mix": 2.8052539229393005, "l_a": 2.8936062455177307, "l_c": 2.840796560049057, "z_loss": 37.916687965393066, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12083788961172104, "tokens_per_second": 1386175.8413550819, "eta_seconds": 8804.727357006073, "data_wait_seconds": 1.9758939743041992e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.18210997, "gate_a2b/L1_absmax": 0.71625268, "gate_a2b/L1_rms": 0.19358757, "gate_a2b/L3_absmean": 0.17638445, "gate_a2b/L3_absmax": 0.55598402, "gate_a2b/L3_rms": 0.182567, "gate_a2b/L5_absmean": 0.26873097, "gate_a2b/L5_absmax": 0.73438001, "gate_a2b/L5_rms": 0.27475241, "gate_c2b/L1_absmean": 0.0216608, "gate_c2b/L1_absmax": 0.63198191, "gate_c2b/L1_rms": 0.06602393, "gate_c2b/L3_absmean": 0.18409835, "gate_c2b/L3_absmax": 0.41708025, "gate_c2b/L3_rms": 0.18800828, "gate_c2b/L5_absmean": 0.5253076, "gate_c2b/L5_absmax": 0.74130386, "gate_c2b/L5_rms": 0.53633636, "gate_b2a/L7_absmean": 0.05054093, "gate_b2a/L7_absmax": 0.32423964, "gate_b2a/L7_rms": 0.05620098, "gate_b2a/L11_absmean": 0.08207555, "gate_b2a/L11_absmax": 0.60212946, "gate_b2a/L11_rms": 0.09117344, "gate_b2a/L14_absmean": 0.21132173, "gate_b2a/L14_absmax": 0.79731482, "gate_b2a/L14_rms": 0.22010246, "gate_b2c/L3_absmean": 0.03588354, "gate_b2c/L3_absmax": 0.57252854, "gate_b2c/L3_rms": 0.04722719, "gate_b2c/L5_absmean": 0.03825074, "gate_b2c/L5_absmax": 0.56977516, "gate_b2c/L5_rms": 0.0484485, "gate_b2c/L6_absmean": 0.05519094, "gate_b2c/L6_absmax": 0.3662644, "gate_b2c/L6_rms": 0.0617749}, "streams": {"cos_a2b/round0": 0.364515, "cos_c2b/round0": 0.293623, "cos_b2a/round0": 0.152608, "cos_b2c/round0": 0.136471, "rms_a/round0": 6.166985, "rms_b/round0": 11.133959, "rms_c/round0": 4.243438, "cos_a2b/round1": 0.410196, "cos_c2b/round1": 0.30418, "cos_b2a/round1": 0.256022, "cos_b2c/round1": 0.114103, "rms_a/round1": 11.829858, "rms_b/round1": 25.286705, "rms_c/round1": 8.868876, "cos_a2b/round2": 0.547606, "cos_c2b/round2": 0.518994, "cos_b2a/round2": 0.10385, "cos_b2c/round2": 0.179604, "rms_a/round2": 32.987759, "rms_b/round2": 67.403603, "rms_c/round2": 16.721643}, "alpha_mean": 0.43166033319145214, "alpha_hist": [0.00978, 0.06528, 0.17647, 0.20467, 0.18864, 0.16772, 0.12411, 0.05905, 0.00426, 2e-05], "alpha_by_type": {"special": {"share": 0.00124, "mean": 0.52967, "std": 0.12997}, "whitespace": {"share": 0.03403, "mean": 0.5036, "std": 0.11567}, "digit": {"share": 0.04308, "mean": 0.40657, "std": 0.12493}, "word_start": {"share": 0.54773, "mean": 0.46113, "std": 0.15537}, "word_cont": {"share": 0.26433, "mean": 0.31012, "std": 0.12812}, "punct": {"share": 0.1096, "mean": 0.56393, "std": 0.1568}}, "alpha_p05": 0.17329, "alpha_p50": 0.42728, "alpha_p95": 0.71083}
{"step": 49210, "total_steps": 72479, "tokens": 25800212480, "loss": 4.443951845169067, "l_mix": 2.7540654838085175, "l_a": 2.842246264219284, "l_c": 2.790708005428314, "z_loss": 38.41592264175415, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13652068376541138, "tokens_per_second": 1353661.5082707598, "eta_seconds": 9012.339789128302, "data_wait_seconds": 3.8504600524902344e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 49220, "total_steps": 72479, "tokens": 25805455360, "loss": 4.4807921051979065, "l_mix": 2.777031660079956, "l_a": 2.8630901277065277, "l_c": 2.8161113560199738, "z_loss": 37.828617572784424, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12251116335391998, "tokens_per_second": 1443802.600689722, "eta_seconds": 8446.040051579475, "data_wait_seconds": 1.868605613708496e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 49230, "total_steps": 72479, "tokens": 25810698240, "loss": 2.9028663635253906, "l_mix": 1.7965869307518005, "l_a": 1.864746779203415, "l_c": 1.8228510618209839, "z_loss": 47.29122591018677, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18693184852600098, "tokens_per_second": 1412185.2610466527, "eta_seconds": 8631.425386047364, "data_wait_seconds": 2.86102294921875e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 49240, "total_steps": 72479, "tokens": 25815941120, "loss": 2.9963377714157104, "l_mix": 1.8549611270427704, "l_a": 1.9234333485364914, "l_c": 1.881155326962471, "z_loss": 47.706950187683105, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18255093693733215, "tokens_per_second": 1447862.588661387, "eta_seconds": 8415.114063596726, "data_wait_seconds": 2.1517276763916016e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 49250, "total_steps": 72479, "tokens": 25821184000, "loss": 4.437268853187561, "l_mix": 2.750158578157425, "l_a": 2.8358605206012726, "l_c": 2.7878403067588806, "z_loss": 39.635231018066406, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16243848204612732, "tokens_per_second": 1464732.3981205798, "eta_seconds": 8314.614988803864, "data_wait_seconds": 1.9729137420654297e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 49260, "total_steps": 72479, "tokens": 25826426880, "loss": 4.46906840801239, "l_mix": 2.7698640525341034, "l_a": 2.8525874614715576, "l_c": 2.8114266395568848, "z_loss": 38.5163779258728, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.21527467668056488, "tokens_per_second": 1429712.250243159, "eta_seconds": 8514.610593795776, "data_wait_seconds": 3.299117088317871e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 49270, "total_steps": 72479, "tokens": 25831669760, "loss": 4.42894434928894, "l_mix": 2.7456107437610626, "l_a": 2.8306655883789062, "l_c": 2.780446410179138, "z_loss": 38.75636053085327, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16014747321605682, "tokens_per_second": 1464237.3338830657, "eta_seconds": 8310.26494846344, "data_wait_seconds": 2.0682811737060547e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 49280, "total_steps": 72479, "tokens": 25836912640, "loss": 4.590621888637543, "l_mix": 2.8454991281032562, "l_a": 2.9282606840133667, "l_c": 2.8888146579265594, "z_loss": 38.93965482711792, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19129127264022827, "tokens_per_second": 1441336.3659096814, "eta_seconds": 8438.666781520844, "data_wait_seconds": 1.9431114196777344e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 49290, "total_steps": 72479, "tokens": 25842155520, "loss": 4.5354920625686646, "l_mix": 2.8118503987789154, "l_a": 2.8970638811588287, "l_c": 2.848408132791519, "z_loss": 38.741682052612305, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15714986622333527, "tokens_per_second": 1393844.9974246784, "eta_seconds": 8722.429290533066, "data_wait_seconds": 1.8328428268432617e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 49300, "total_steps": 72479, "tokens": 25847398400, "loss": 4.488706052303314, "l_mix": 2.7823750376701355, "l_a": 2.8681986927986145, "l_c": 2.819571167230606, "z_loss": 38.58519983291626, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.119114451110363, "tokens_per_second": 1460450.6027247112, "eta_seconds": 8321.04251203537, "data_wait_seconds": 1.5109777450561523e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.18153778, "gate_a2b/L1_absmax": 0.71582699, "gate_a2b/L1_rms": 0.19309375, "gate_a2b/L3_absmean": 0.17682162, "gate_a2b/L3_absmax": 0.55663198, "gate_a2b/L3_rms": 0.18287519, "gate_a2b/L5_absmean": 0.26828885, "gate_a2b/L5_absmax": 0.73275423, "gate_a2b/L5_rms": 0.27428043, "gate_c2b/L1_absmean": 0.02192562, "gate_c2b/L1_absmax": 0.63232756, "gate_c2b/L1_rms": 0.06619349, "gate_c2b/L3_absmean": 0.18396682, "gate_c2b/L3_absmax": 0.41891617, "gate_c2b/L3_rms": 0.18787658, "gate_c2b/L5_absmean": 0.52475238, "gate_c2b/L5_absmax": 0.74159598, "gate_c2b/L5_rms": 0.53572929, "gate_b2a/L7_absmean": 0.05045764, "gate_b2a/L7_absmax": 0.3244127, "gate_b2a/L7_rms": 0.05604716, "gate_b2a/L11_absmean": 0.08206355, "gate_b2a/L11_absmax": 0.60012448, "gate_b2a/L11_rms": 0.09108497, "gate_b2a/L14_absmean": 0.21187329, "gate_b2a/L14_absmax": 0.79440922, "gate_b2a/L14_rms": 0.22053413, "gate_b2c/L3_absmean": 0.03576034, "gate_b2c/L3_absmax": 0.57122737, "gate_b2c/L3_rms": 0.0470589, "gate_b2c/L5_absmean": 0.03845209, "gate_b2c/L5_absmax": 0.56799906, "gate_b2c/L5_rms": 0.04853604, "gate_b2c/L6_absmean": 0.05538618, "gate_b2c/L6_absmax": 0.36572969, "gate_b2c/L6_rms": 0.06184429}, "streams": {"cos_a2b/round0": 0.361043, "cos_c2b/round0": 0.293864, "cos_b2a/round0": 0.149773, "cos_b2c/round0": 0.134562, "rms_a/round0": 6.164021, "rms_b/round0": 11.087608, "rms_c/round0": 4.265913, "cos_a2b/round1": 0.411697, "cos_c2b/round1": 0.29777, "cos_b2a/round1": 0.247705, "cos_b2c/round1": 0.114069, "rms_a/round1": 11.942088, "rms_b/round1": 25.533287, "rms_c/round1": 8.806983, "cos_a2b/round2": 0.544196, "cos_c2b/round2": 0.517514, "cos_b2a/round2": 0.098627, "cos_b2c/round2": 0.177627, "rms_a/round2": 33.202263, "rms_b/round2": 68.16217, "rms_c/round2": 16.819937}, "alpha_mean": 0.4447302888650029, "alpha_hist": [0.01039, 0.06197, 0.15994, 0.19759, 0.18398, 0.17241, 0.13252, 0.07172, 0.00929, 0.0002], "alpha_by_type": {"special": {"share": 0.00131, "mean": 0.53765, "std": 0.10108}, "whitespace": {"share": 0.03128, "mean": 0.53507, "std": 0.12093}, "digit": {"share": 0.04251, "mean": 0.4798, "std": 0.11222}, "word_start": {"share": 0.55351, "mean": 0.46282, "std": 0.16736}, "word_cont": {"share": 0.26712, "mean": 0.33368, "std": 0.13585}, "punct": {"share": 0.10426, "mean": 0.59061, "std": 0.14908}}, "alpha_p05": 0.17867, "alpha_p50": 0.42752, "alpha_p95": 0.73297}
{"step": 49310, "total_steps": 72479, "tokens": 25852641280, "loss": 4.466507792472839, "l_mix": 2.76914781332016, "l_a": 2.850988268852234, "l_c": 2.8068777918815613, "z_loss": 37.40443801879883, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19579347968101501, "tokens_per_second": 1488532.6361234824, "eta_seconds": 8160.539028310775, "data_wait_seconds": 1.531839370727539e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 49320, "total_steps": 72479, "tokens": 25857884160, "loss": 4.514706611633301, "l_mix": 2.798562616109848, "l_a": 2.885318249464035, "l_c": 2.835161566734314, "z_loss": 37.598116397857666, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1460312455892563, "tokens_per_second": 1431334.1810716875, "eta_seconds": 8482.984583592415, "data_wait_seconds": 1.749396324157715e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 49330, "total_steps": 72479, "tokens": 25863127040, "loss": 4.421947121620178, "l_mix": 2.740383952856064, "l_a": 2.826245605945587, "l_c": 2.7789647579193115, "z_loss": 39.687448501586914, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1321057826280594, "tokens_per_second": 1325984.8713612305, "eta_seconds": 9153.002552390099, "data_wait_seconds": 1.7583370208740234e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 49340, "total_steps": 72479, "tokens": 25868369920, "loss": 2.9954719245433807, "l_mix": 1.8534330427646637, "l_a": 1.9270338416099548, "l_c": 1.8797623366117477, "z_loss": 47.33095359802246, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18900975584983826, "tokens_per_second": 1468968.9146699077, "eta_seconds": 8258.51378531456, "data_wait_seconds": 3.224611282348633e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 49350, "total_steps": 72479, "tokens": 25873612800, "loss": 2.7989217787981033, "l_mix": 1.7331421300768852, "l_a": 1.7940293103456497, "l_c": 1.7585695162415504, "z_loss": 45.957083225250244, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19501125812530518, "tokens_per_second": 1410738.626161682, "eta_seconds": 8595.679544830322, "data_wait_seconds": 2.4318695068359375e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 49360, "total_steps": 72479, "tokens": 25878855680, "loss": 4.422912120819092, "l_mix": 2.739948183298111, "l_a": 2.83013454079628, "l_c": 2.7797451317310333, "z_loss": 39.05204772949219, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14331857860088348, "tokens_per_second": 1433401.229956061, "eta_seconds": 8456.121020889283, "data_wait_seconds": 1.6480684280395508e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 49370, "total_steps": 72479, "tokens": 25884098560, "loss": 4.478164613246918, "l_mix": 2.7754471600055695, "l_a": 2.862057864665985, "l_c": 2.8136666119098663, "z_loss": 38.6721134185791, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1443774402141571, "tokens_per_second": 1450520.4638977915, "eta_seconds": 8352.70628271103, "data_wait_seconds": 1.3619661331176758e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 49380, "total_steps": 72479, "tokens": 25889341440, "loss": 4.4817575216293335, "l_mix": 2.777682840824127, "l_a": 2.861673504114151, "l_c": 2.818575143814087, "z_loss": 39.208622455596924, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16075199842453003, "tokens_per_second": 1449655.6564003038, "eta_seconds": 8354.072540283203, "data_wait_seconds": 3.9905309677124023e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 49390, "total_steps": 72479, "tokens": 25894584320, "loss": 4.359103500843048, "l_mix": 2.701446920633316, "l_a": 2.78392431139946, "l_c": 2.7415974140167236, "z_loss": 39.09954023361206, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19698473811149597, "tokens_per_second": 1439670.1164385544, "eta_seconds": 8408.374594831466, "data_wait_seconds": 1.8268823623657227e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 49400, "total_steps": 72479, "tokens": 25899827200, "loss": 4.422596037387848, "l_mix": 2.740607887506485, "l_a": 2.8299205899238586, "l_c": 2.776706337928772, "z_loss": 38.66438102722168, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14179876446723938, "tokens_per_second": 1334088.2301368255, "eta_seconds": 9069.896936845778, "data_wait_seconds": 1.823902130126953e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.18125997, "gate_a2b/L1_absmax": 0.71623498, "gate_a2b/L1_rms": 0.19285344, "gate_a2b/L3_absmean": 0.17595766, "gate_a2b/L3_absmax": 0.56090581, "gate_a2b/L3_rms": 0.18217672, "gate_a2b/L5_absmean": 0.26787755, "gate_a2b/L5_absmax": 0.74037534, "gate_a2b/L5_rms": 0.27401158, "gate_c2b/L1_absmean": 0.02254158, "gate_c2b/L1_absmax": 0.63433933, "gate_c2b/L1_rms": 0.06660126, "gate_c2b/L3_absmean": 0.18456654, "gate_c2b/L3_absmax": 0.41505301, "gate_c2b/L3_rms": 0.18854389, "gate_c2b/L5_absmean": 0.52428305, "gate_c2b/L5_absmax": 0.75209832, "gate_c2b/L5_rms": 0.53525877, "gate_b2a/L7_absmean": 0.0508396, "gate_b2a/L7_absmax": 0.32399324, "gate_b2a/L7_rms": 0.05644209, "gate_b2a/L11_absmean": 0.08225708, "gate_b2a/L11_absmax": 0.60606521, "gate_b2a/L11_rms": 0.09154219, "gate_b2a/L14_absmean": 0.21041672, "gate_b2a/L14_absmax": 0.79983598, "gate_b2a/L14_rms": 0.21939132, "gate_b2c/L3_absmean": 0.03606036, "gate_b2c/L3_absmax": 0.57323903, "gate_b2c/L3_rms": 0.04734525, "gate_b2c/L5_absmean": 0.03819023, "gate_b2c/L5_absmax": 0.57182878, "gate_b2c/L5_rms": 0.04848415, "gate_b2c/L6_absmean": 0.05486379, "gate_b2c/L6_absmax": 0.3710407, "gate_b2c/L6_rms": 0.06159346}, "streams": {"cos_a2b/round0": 0.362008, "cos_c2b/round0": 0.295052, "cos_b2a/round0": 0.15086, "cos_b2c/round0": 0.135365, "rms_a/round0": 6.219534, "rms_b/round0": 11.20087, "rms_c/round0": 4.273425, "cos_a2b/round1": 0.413611, "cos_c2b/round1": 0.303244, "cos_b2a/round1": 0.250301, "cos_b2c/round1": 0.114263, "rms_a/round1": 12.031991, "rms_b/round1": 25.610966, "rms_c/round1": 8.840809, "cos_a2b/round2": 0.548278, "cos_c2b/round2": 0.518592, "cos_b2a/round2": 0.100696, "cos_b2c/round2": 0.18268, "rms_a/round2": 33.257557, "rms_b/round2": 68.448593, "rms_c/round2": 16.627253}, "alpha_mean": 0.4491619237474396, "alpha_hist": [0.00792, 0.06308, 0.16988, 0.18762, 0.17499, 0.1629, 0.13695, 0.08334, 0.01318, 0.00014], "alpha_by_type": {"special": {"share": 0.00095, "mean": 0.57422, "std": 0.11176}, "whitespace": {"share": 0.02626, "mean": 0.54852, "std": 0.12692}, "digit": {"share": 0.02817, "mean": 0.4543, "std": 0.117}, "word_start": {"share": 0.56842, "mean": 0.47099, "std": 0.1686}, "word_cont": {"share": 0.26509, "mean": 0.32808, "std": 0.13531}, "punct": {"share": 0.11111, "mean": 0.60053, "std": 0.15472}}, "alpha_p05": 0.18713, "alpha_p50": 0.44408, "alpha_p95": 0.74909}
{"step": 49410, "total_steps": 72479, "tokens": 25905070080, "loss": 5.091095447540283, "l_mix": 3.1601204574108124, "l_a": 3.234582245349884, "l_c": 3.202000916004181, "z_loss": 36.806777000427246, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.26488593220710754, "tokens_per_second": 1415756.6157607003, "eta_seconds": 8542.99371612072, "data_wait_seconds": 2.0295381546020508e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 49420, "total_steps": 72479, "tokens": 25910312960, "loss": 4.721466779708862, "l_mix": 2.929658442735672, "l_a": 3.004654884338379, "l_c": 2.96803942322731, "z_loss": 36.777451515197754, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16673783957958221, "tokens_per_second": 1479738.7986182251, "eta_seconds": 8170.06150226593, "data_wait_seconds": 2.014636993408203e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 49430, "total_steps": 72479, "tokens": 25915555840, "loss": 4.798548221588135, "l_mix": 2.978058099746704, "l_a": 3.05310195684433, "l_c": 3.015198290348053, "z_loss": 36.94230842590332, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14986194670200348, "tokens_per_second": 1445450.691263214, "eta_seconds": 8360.23960211277, "data_wait_seconds": 2.0831823348999023e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 49440, "total_steps": 72479, "tokens": 25920798720, "loss": 4.698574125766754, "l_mix": 2.9155930876731873, "l_a": 2.990421324968338, "l_c": 2.952848494052887, "z_loss": 37.54033660888672, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1335524320602417, "tokens_per_second": 1292424.0043632807, "eta_seconds": 9346.059181213379, "data_wait_seconds": 2.2143125534057617e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 49450, "total_steps": 72479, "tokens": 25926041600, "loss": 4.733660936355591, "l_mix": 2.93709796667099, "l_a": 3.009563684463501, "l_c": 2.97897931933403, "z_loss": 38.813973903656006, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13261643052101135, "tokens_per_second": 1456477.0355952203, "eta_seconds": 8289.748521208763, "data_wait_seconds": 2.682209014892578e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 49460, "total_steps": 72479, "tokens": 25931284480, "loss": 2.719053342938423, "l_mix": 1.6825065314769745, "l_a": 1.748042955994606, "l_c": 1.707112967967987, "z_loss": 45.44785833358765, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2184363454580307, "tokens_per_second": 1450602.9441268486, "eta_seconds": 8319.702866220474, "data_wait_seconds": 2.2649765014648438e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 49470, "total_steps": 72479, "tokens": 25936527360, "loss": 4.5026437640190125, "l_mix": 2.7899534702301025, "l_a": 2.878552943468094, "l_c": 2.830414295196533, "z_loss": 39.10069179534912, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14929184317588806, "tokens_per_second": 1336898.5543361139, "eta_seconds": 9023.379188251494, "data_wait_seconds": 2.0295381546020508e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 49480, "total_steps": 72479, "tokens": 25941770240, "loss": 4.517519235610962, "l_mix": 2.7999234199523926, "l_a": 2.889393389225006, "l_c": 2.8359254598617554, "z_loss": 38.33553218841553, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.136525958776474, "tokens_per_second": 1402985.7156969793, "eta_seconds": 8594.599059057235, "data_wait_seconds": 1.5676021575927734e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 49490, "total_steps": 72479, "tokens": 25947013120, "loss": 4.521161258220673, "l_mix": 2.8023961186408997, "l_a": 2.889402389526367, "l_c": 2.8398144841194153, "z_loss": 37.98687744140625, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1546086221933365, "tokens_per_second": 1443782.1252418836, "eta_seconds": 8348.113348460198, "data_wait_seconds": 2.4557113647460938e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 49500, "total_steps": 72479, "tokens": 25952256000, "loss": 4.403654158115387, "l_mix": 2.7287614345550537, "l_a": 2.8138545751571655, "l_c": 2.7691208720207214, "z_loss": 38.9735221862793, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15062768757343292, "tokens_per_second": 1400651.3090613666, "eta_seconds": 8601.436970114708, "data_wait_seconds": 1.5348196029663086e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.18044899, "gate_a2b/L1_absmax": 0.7163614, "gate_a2b/L1_rms": 0.19210228, "gate_a2b/L3_absmean": 0.17509823, "gate_a2b/L3_absmax": 0.55843824, "gate_a2b/L3_rms": 0.18129766, "gate_a2b/L5_absmean": 0.26650986, "gate_a2b/L5_absmax": 0.73959458, "gate_a2b/L5_rms": 0.27264419, "gate_c2b/L1_absmean": 0.02183849, "gate_c2b/L1_absmax": 0.6360873, "gate_c2b/L1_rms": 0.06628665, "gate_c2b/L3_absmean": 0.18469365, "gate_c2b/L3_absmax": 0.41570795, "gate_c2b/L3_rms": 0.18860921, "gate_c2b/L5_absmean": 0.52418154, "gate_c2b/L5_absmax": 0.74007034, "gate_c2b/L5_rms": 0.53510308, "gate_b2a/L7_absmean": 0.0507132, "gate_b2a/L7_absmax": 0.32444695, "gate_b2a/L7_rms": 0.05633887, "gate_b2a/L11_absmean": 0.08183989, "gate_b2a/L11_absmax": 0.60406667, "gate_b2a/L11_rms": 0.09102589, "gate_b2a/L14_absmean": 0.21143997, "gate_b2a/L14_absmax": 0.79845428, "gate_b2a/L14_rms": 0.22021495, "gate_b2c/L3_absmean": 0.03573617, "gate_b2c/L3_absmax": 0.57151175, "gate_b2c/L3_rms": 0.0470519, "gate_b2c/L5_absmean": 0.0384611, "gate_b2c/L5_absmax": 0.56868833, "gate_b2c/L5_rms": 0.0485752, "gate_b2c/L6_absmean": 0.05505867, "gate_b2c/L6_absmax": 0.37088162, "gate_b2c/L6_rms": 0.06168473}, "streams": {"cos_a2b/round0": 0.365314, "cos_c2b/round0": 0.300332, "cos_b2a/round0": 0.152039, "cos_b2c/round0": 0.136139, "rms_a/round0": 6.208759, "rms_b/round0": 11.14213, "rms_c/round0": 4.291819, "cos_a2b/round1": 0.416915, "cos_c2b/round1": 0.304091, "cos_b2a/round1": 0.250604, "cos_b2c/round1": 0.111258, "rms_a/round1": 12.130482, "rms_b/round1": 25.634443, "rms_c/round1": 8.832849, "cos_a2b/round2": 0.541082, "cos_c2b/round2": 0.524151, "cos_b2a/round2": 0.10415, "cos_b2c/round2": 0.180954, "rms_a/round2": 33.131016, "rms_b/round2": 68.467804, "rms_c/round2": 16.836317}, "alpha_mean": 0.4321247868894602, "alpha_hist": [0.00818, 0.07571, 0.19455, 0.19099, 0.16731, 0.15659, 0.12575, 0.07143, 0.00946, 3e-05], "alpha_by_type": {"special": {"share": 0.00102, "mean": 0.5292, "std": 0.11213}, "whitespace": {"share": 0.02545, "mean": 0.54478, "std": 0.12097}, "digit": {"share": 0.03021, "mean": 0.46383, "std": 0.13722}, "word_start": {"share": 0.59412, "mean": 0.44291, "std": 0.17117}, "word_cont": {"share": 0.24294, "mean": 0.32065, "std": 0.12944}, "punct": {"share": 0.10626, "mean": 0.58975, "std": 0.15718}}, "alpha_p05": 0.18418, "alpha_p50": 0.4225, "alpha_p95": 0.73259}
{"step": 49510, "total_steps": 72479, "tokens": 25957498880, "loss": 4.458071768283844, "l_mix": 2.7630495727062225, "l_a": 2.850665360689163, "l_c": 2.79940864443779, "z_loss": 38.46933031082153, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13695098459720612, "tokens_per_second": 1477562.2587367587, "eta_seconds": 8150.161525034905, "data_wait_seconds": 1.519918441772461e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 49520, "total_steps": 72479, "tokens": 25962741760, "loss": 3.147230237722397, "l_mix": 1.8840951025485992, "l_a": 2.379095286130905, "l_c": 1.8313550502061844, "z_loss": 43.36744832992554, "lr": 0.0023, "wd": 0.01, "grad_norm": 1.0039366483688354, "tokens_per_second": 1436325.0373443826, "eta_seconds": 8380.504328083993, "data_wait_seconds": 4.5746564865112305e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 49530, "total_steps": 72479, "tokens": 25967984640, "loss": 4.711679041385651, "l_mix": 2.9229533970355988, "l_a": 3.0021535754203796, "l_c": 2.9602649211883545, "z_loss": 38.28880500793457, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15185588598251343, "tokens_per_second": 1379250.7903337055, "eta_seconds": 8723.493505549432, "data_wait_seconds": 1.704692840576172e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 49540, "total_steps": 72479, "tokens": 25973227520, "loss": 4.751956462860107, "l_mix": 2.948716104030609, "l_a": 3.023600548505783, "l_c": 2.987200379371643, "z_loss": 37.29392147064209, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11492238938808441, "tokens_per_second": 1408144.338680554, "eta_seconds": 8540.773911905288, "data_wait_seconds": 1.7315149307250977e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 49550, "total_steps": 72479, "tokens": 25978470400, "loss": 4.765646755695343, "l_mix": 2.957260936498642, "l_a": 3.0323232412338257, "l_c": 2.9956294894218445, "z_loss": 36.741058349609375, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13534365594387054, "tokens_per_second": 1321042.1377659966, "eta_seconds": 9099.93648827076, "data_wait_seconds": 2.5957822799682617e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 49560, "total_steps": 72479, "tokens": 25983713280, "loss": 4.72627329826355, "l_mix": 2.932548373937607, "l_a": 3.009233444929123, "l_c": 2.969849109649658, "z_loss": 37.3942985534668, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1260939985513687, "tokens_per_second": 1393971.0821265955, "eta_seconds": 8620.090349125861, "data_wait_seconds": 3.263354301452637e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 49570, "total_steps": 72479, "tokens": 25988956160, "loss": 4.8000301122665405, "l_mix": 2.9784616827964783, "l_a": 3.053597390651703, "l_c": 3.018296927213669, "z_loss": 37.51200246810913, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14355501532554626, "tokens_per_second": 1449109.0383724496, "eta_seconds": 8288.48173184395, "data_wait_seconds": 4.571676254272461e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 49580, "total_steps": 72479, "tokens": 25994199040, "loss": 4.524631023406982, "l_mix": 2.8049913346767426, "l_a": 2.8913914263248444, "l_c": 2.840740978717804, "z_loss": 38.6826114654541, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15705671906471252, "tokens_per_second": 1454608.0792483797, "eta_seconds": 8253.543399953842, "data_wait_seconds": 1.704692840576172e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 49590, "total_steps": 72479, "tokens": 25999441920, "loss": 4.439787030220032, "l_mix": 2.7510285079479218, "l_a": 2.840684711933136, "l_c": 2.788510352373123, "z_loss": 38.64954471588135, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1294250786304474, "tokens_per_second": 1463546.5981600105, "eta_seconds": 8199.55309047699, "data_wait_seconds": 2.110004425048828e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 49600, "total_steps": 72479, "tokens": 26004684800, "loss": 4.508801579475403, "l_mix": 2.7942083179950714, "l_a": 2.8815593123435974, "l_c": 2.833751618862152, "z_loss": 38.76685094833374, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13296431303024292, "tokens_per_second": 1388659.2233163964, "eta_seconds": 8637.961675977707, "data_wait_seconds": 1.722574234008789e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.18090422, "gate_a2b/L1_absmax": 0.7195394, "gate_a2b/L1_rms": 0.19239181, "gate_a2b/L3_absmean": 0.17532697, "gate_a2b/L3_absmax": 0.55122435, "gate_a2b/L3_rms": 0.1814786, "gate_a2b/L5_absmean": 0.265856, "gate_a2b/L5_absmax": 0.73696023, "gate_a2b/L5_rms": 0.27191392, "gate_c2b/L1_absmean": 0.02204673, "gate_c2b/L1_absmax": 0.63736814, "gate_c2b/L1_rms": 0.06641851, "gate_c2b/L3_absmean": 0.18419833, "gate_c2b/L3_absmax": 0.41442847, "gate_c2b/L3_rms": 0.18810494, "gate_c2b/L5_absmean": 0.52433437, "gate_c2b/L5_absmax": 0.74109918, "gate_c2b/L5_rms": 0.53514451, "gate_b2a/L7_absmean": 0.05064382, "gate_b2a/L7_absmax": 0.32419902, "gate_b2a/L7_rms": 0.05622975, "gate_b2a/L11_absmean": 0.08214091, "gate_b2a/L11_absmax": 0.6067102, "gate_b2a/L11_rms": 0.09134223, "gate_b2a/L14_absmean": 0.21191409, "gate_b2a/L14_absmax": 0.80236214, "gate_b2a/L14_rms": 0.22068805, "gate_b2c/L3_absmean": 0.03565045, "gate_b2c/L3_absmax": 0.56712848, "gate_b2c/L3_rms": 0.04681235, "gate_b2c/L5_absmean": 0.0382022, "gate_b2c/L5_absmax": 0.5659259, "gate_b2c/L5_rms": 0.04826732, "gate_b2c/L6_absmean": 0.05522529, "gate_b2c/L6_absmax": 0.37039876, "gate_b2c/L6_rms": 0.06178671}, "streams": {"cos_a2b/round0": 0.360903, "cos_c2b/round0": 0.295532, "cos_b2a/round0": 0.154347, "cos_b2c/round0": 0.136469, "rms_a/round0": 6.252609, "rms_b/round0": 11.328854, "rms_c/round0": 4.265123, "cos_a2b/round1": 0.413786, "cos_c2b/round1": 0.30468, "cos_b2a/round1": 0.249679, "cos_b2c/round1": 0.116448, "rms_a/round1": 12.253647, "rms_b/round1": 25.867687, "rms_c/round1": 8.969712, "cos_a2b/round2": 0.549823, "cos_c2b/round2": 0.516008, "cos_b2a/round2": 0.100961, "cos_b2c/round2": 0.180448, "rms_a/round2": 33.869942, "rms_b/round2": 68.76532, "rms_c/round2": 17.033182}, "alpha_mean": 0.43869006128828403, "alpha_hist": [0.00789, 0.06772, 0.17076, 0.20287, 0.18471, 0.16109, 0.1196, 0.07396, 0.01125, 0.00017], "alpha_by_type": {"special": {"share": 0.00128, "mean": 0.59203, "std": 0.11735}, "whitespace": {"share": 0.02484, "mean": 0.5256, "std": 0.11958}, "digit": {"share": 0.03632, "mean": 0.43755, "std": 0.12489}, "word_start": {"share": 0.5762, "mean": 0.45579, "std": 0.16668}, "word_cont": {"share": 0.24767, "mean": 0.32384, "std": 0.132}, "punct": {"share": 0.11369, "mean": 0.58185, "std": 0.15889}}, "alpha_p05": 0.15974, "alpha_p50": 0.4383, "alpha_p95": 0.73335}
{"step": 49610, "total_steps": 72479, "tokens": 26009927680, "loss": 4.502786099910736, "l_mix": 2.7916820645332336, "l_a": 2.8747956454753876, "l_c": 2.8288843631744385, "z_loss": 38.46341371536255, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11742525547742844, "tokens_per_second": 1458498.6931366457, "eta_seconds": 8220.742554259301, "data_wait_seconds": 1.6421079635620117e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 49620, "total_steps": 72479, "tokens": 26015170560, "loss": 4.490887224674225, "l_mix": 2.7837061882019043, "l_a": 2.8703715205192566, "l_c": 2.820231944322586, "z_loss": 38.52076816558838, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1377793848514557, "tokens_per_second": 1480229.9545912794, "eta_seconds": 8096.511866164208, "data_wait_seconds": 1.665949821472168e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 49630, "total_steps": 72479, "tokens": 26020413440, "loss": 4.476004540920258, "l_mix": 2.774286538362503, "l_a": 2.863087385892868, "l_c": 2.809305429458618, "z_loss": 38.878952503204346, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1862170398235321, "tokens_per_second": 1406149.6210195085, "eta_seconds": 8519.33274590969, "data_wait_seconds": 1.6897916793823242e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 49640, "total_steps": 72479, "tokens": 26025656320, "loss": 3.0024768114089966, "l_mix": 1.8582032769918442, "l_a": 1.9242203831672668, "l_c": 1.8900244235992432, "z_loss": 46.42194652557373, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19344791769981384, "tokens_per_second": 1434270.9754380921, "eta_seconds": 8348.641112494468, "data_wait_seconds": 2.6971101760864258e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 49650, "total_steps": 72479, "tokens": 26030899200, "loss": 3.1861215829849243, "l_mix": 1.9615174233913422, "l_a": 2.071418583393097, "l_c": 2.0105951726436615, "z_loss": 39.97362184524536, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1677900105714798, "tokens_per_second": 1439087.4904582114, "eta_seconds": 8317.055656003951, "data_wait_seconds": 1.9818544387817383e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 49660, "total_steps": 72479, "tokens": 26036142080, "loss": 4.7192739844322205, "l_mix": 2.9287867546081543, "l_a": 3.001992851495743, "l_c": 2.9662978053092957, "z_loss": 37.496899127960205, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11140552908182144, "tokens_per_second": 1380144.7382786218, "eta_seconds": 8668.458850860596, "data_wait_seconds": 1.6897916793823242e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 49670, "total_steps": 72479, "tokens": 26041384960, "loss": 4.764073431491852, "l_mix": 2.955251455307007, "l_a": 3.0295897722244263, "l_c": 2.9998165369033813, "z_loss": 38.77950477600098, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18909333646297455, "tokens_per_second": 1313580.6045823707, "eta_seconds": 9103.73139667511, "data_wait_seconds": 1.5616416931152344e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 49680, "total_steps": 72479, "tokens": 26046627840, "loss": 4.756107032299042, "l_mix": 2.9515064656734467, "l_a": 3.0241156220436096, "l_c": 2.9912195801734924, "z_loss": 38.43943929672241, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13715635240077972, "tokens_per_second": 1468512.8566653784, "eta_seconds": 8139.691837048531, "data_wait_seconds": 1.7583370208740234e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 49690, "total_steps": 72479, "tokens": 26051870720, "loss": 4.475368320941925, "l_mix": 2.7736961245536804, "l_a": 2.8605974316596985, "l_c": 2.811643362045288, "z_loss": 38.74362134933472, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11713560670614243, "tokens_per_second": 1422545.3625136414, "eta_seconds": 8399.028633356094, "data_wait_seconds": 2.2023916244506836e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 49700, "total_steps": 72479, "tokens": 26057113600, "loss": 4.495371460914612, "l_mix": 2.7868461310863495, "l_a": 2.8700950741767883, "l_c": 2.8249890506267548, "z_loss": 38.778708934783936, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11407574266195297, "tokens_per_second": 1478259.8313336142, "eta_seconds": 8078.929088687897, "data_wait_seconds": 1.7374753952026367e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.18084989, "gate_a2b/L1_absmax": 0.71644217, "gate_a2b/L1_rms": 0.19225438, "gate_a2b/L3_absmean": 0.17481697, "gate_a2b/L3_absmax": 0.55409837, "gate_a2b/L3_rms": 0.18104187, "gate_a2b/L5_absmean": 0.26577145, "gate_a2b/L5_absmax": 0.7354095, "gate_a2b/L5_rms": 0.27182004, "gate_c2b/L1_absmean": 0.0223611, "gate_c2b/L1_absmax": 0.63412279, "gate_c2b/L1_rms": 0.06619862, "gate_c2b/L3_absmean": 0.18366539, "gate_c2b/L3_absmax": 0.40882638, "gate_c2b/L3_rms": 0.18758608, "gate_c2b/L5_absmean": 0.52314121, "gate_c2b/L5_absmax": 0.73593742, "gate_c2b/L5_rms": 0.53399694, "gate_b2a/L7_absmean": 0.05064742, "gate_b2a/L7_absmax": 0.32299155, "gate_b2a/L7_rms": 0.05613778, "gate_b2a/L11_absmean": 0.08214071, "gate_b2a/L11_absmax": 0.60343397, "gate_b2a/L11_rms": 0.09126188, "gate_b2a/L14_absmean": 0.21138361, "gate_b2a/L14_absmax": 0.79820251, "gate_b2a/L14_rms": 0.22017463, "gate_b2c/L3_absmean": 0.03586802, "gate_b2c/L3_absmax": 0.56579578, "gate_b2c/L3_rms": 0.04691933, "gate_b2c/L5_absmean": 0.03833384, "gate_b2c/L5_absmax": 0.5654124, "gate_b2c/L5_rms": 0.04835251, "gate_b2c/L6_absmean": 0.05519643, "gate_b2c/L6_absmax": 0.36905152, "gate_b2c/L6_rms": 0.06176927}, "streams": {"cos_a2b/round0": 0.363845, "cos_c2b/round0": 0.293786, "cos_b2a/round0": 0.151326, "cos_b2c/round0": 0.1366, "rms_a/round0": 6.336586, "rms_b/round0": 11.33925, "rms_c/round0": 4.254431, "cos_a2b/round1": 0.414289, "cos_c2b/round1": 0.305213, "cos_b2a/round1": 0.248757, "cos_b2c/round1": 0.118243, "rms_a/round1": 12.277117, "rms_b/round1": 25.712879, "rms_c/round1": 8.936913, "cos_a2b/round2": 0.55094, "cos_c2b/round2": 0.516523, "cos_b2a/round2": 0.104023, "cos_b2c/round2": 0.183096, "rms_a/round2": 33.948502, "rms_b/round2": 68.400703, "rms_c/round2": 16.887999}, "alpha_mean": 0.4513760765227488, "alpha_hist": [0.0096, 0.07185, 0.15892, 0.16757, 0.16869, 0.19228, 0.1451, 0.07484, 0.01094, 0.00021], "alpha_by_type": {"special": {"share": 0.00122, "mean": 0.56949, "std": 0.13428}, "whitespace": {"share": 0.03293, "mean": 0.53877, "std": 0.1198}, "digit": {"share": 0.03528, "mean": 0.46187, "std": 0.13264}, "word_start": {"share": 0.56572, "mean": 0.46626, "std": 0.16734}, "word_cont": {"share": 0.24878, "mean": 0.33322, "std": 0.14592}, "punct": {"share": 0.11607, "mean": 0.60283, "std": 0.15133}}, "alpha_p05": 0.17526, "alpha_p50": 0.4615, "alpha_p95": 0.74466}
{"step": 49710, "total_steps": 72479, "tokens": 26062356480, "loss": 4.505991816520691, "l_mix": 2.793225109577179, "l_a": 2.8777061700820923, "l_c": 2.8315160870552063, "z_loss": 38.629902362823486, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12819431722164154, "tokens_per_second": 1458916.126048755, "eta_seconds": 8182.453575539589, "data_wait_seconds": 1.5914440155029297e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 49720, "total_steps": 72479, "tokens": 26067599360, "loss": 4.404252082109451, "l_mix": 2.7294504046440125, "l_a": 2.815762996673584, "l_c": 2.766908884048462, "z_loss": 39.688021183013916, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13330908119678497, "tokens_per_second": 1431317.6910766128, "eta_seconds": 8336.563340473176, "data_wait_seconds": 1.609325408935547e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 49730, "total_steps": 72479, "tokens": 26072842240, "loss": 4.474347114562988, "l_mix": 2.7736470997333527, "l_a": 2.8603866398334503, "l_c": 2.8086128532886505, "z_loss": 38.588130950927734, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13476379215717316, "tokens_per_second": 1427239.2218792227, "eta_seconds": 8356.712406134604, "data_wait_seconds": 1.9997358322143555e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 49740, "total_steps": 72479, "tokens": 26078085120, "loss": 4.546102225780487, "l_mix": 2.817823678255081, "l_a": 2.9037339985370636, "l_c": 2.857194274663925, "z_loss": 39.11769723892212, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1505632847547531, "tokens_per_second": 1435145.9917997187, "eta_seconds": 8307.018867850304, "data_wait_seconds": 2.6881694793701172e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 49750, "total_steps": 72479, "tokens": 26083328000, "loss": 2.7467991411685944, "l_mix": 1.6995654106140137, "l_a": 1.7634510695934296, "l_c": 1.7273280024528503, "z_loss": 48.78908634185791, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1558254361152649, "tokens_per_second": 1414013.3709027804, "eta_seconds": 8427.460586452484, "data_wait_seconds": 3.141164779663086e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 49760, "total_steps": 72479, "tokens": 26088570880, "loss": 3.128271758556366, "l_mix": 1.92408387362957, "l_a": 2.041202038526535, "l_c": 1.9727574288845062, "z_loss": 40.63105010986328, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18051792681217194, "tokens_per_second": 1298300.982749724, "eta_seconds": 9174.528272151947, "data_wait_seconds": 1.9252300262451172e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 49770, "total_steps": 72479, "tokens": 26093813760, "loss": 3.1339304745197296, "l_mix": 1.9282439798116684, "l_a": 2.043027639389038, "l_c": 1.975927159190178, "z_loss": 41.291985511779785, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1436614692211151, "tokens_per_second": 1375618.1336656301, "eta_seconds": 8655.059060811996, "data_wait_seconds": 2.8401613235473633e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 49780, "total_steps": 72479, "tokens": 26099056640, "loss": 3.1480498015880585, "l_mix": 1.9363820403814316, "l_a": 2.0533304512500763, "l_c": 1.9855618178844452, "z_loss": 42.25551176071167, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14314377307891846, "tokens_per_second": 1389896.347665337, "eta_seconds": 8562.374692177773, "data_wait_seconds": 2.3752450942993164e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 49790, "total_steps": 72479, "tokens": 26104299520, "loss": 4.462319731712341, "l_mix": 2.7654701471328735, "l_a": 2.8515032529830933, "l_c": 2.8046618700027466, "z_loss": 39.21231269836426, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1231088936328888, "tokens_per_second": 1441792.0515625018, "eta_seconds": 8250.545159482956, "data_wait_seconds": 2.491474151611328e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 49800, "total_steps": 72479, "tokens": 26109542400, "loss": 4.462990701198578, "l_mix": 2.766006290912628, "l_a": 2.854334533214569, "l_c": 2.8022799491882324, "z_loss": 38.565999031066895, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12582378089427948, "tokens_per_second": 1362756.53391282, "eta_seconds": 8725.203113031386, "data_wait_seconds": 1.7970800399780273e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.18110906, "gate_a2b/L1_absmax": 0.7182529, "gate_a2b/L1_rms": 0.19252531, "gate_a2b/L3_absmean": 0.17464861, "gate_a2b/L3_absmax": 0.55293024, "gate_a2b/L3_rms": 0.18088956, "gate_a2b/L5_absmean": 0.26565608, "gate_a2b/L5_absmax": 0.73493707, "gate_a2b/L5_rms": 0.27171376, "gate_c2b/L1_absmean": 0.02257922, "gate_c2b/L1_absmax": 0.63420022, "gate_c2b/L1_rms": 0.06638168, "gate_c2b/L3_absmean": 0.18355389, "gate_c2b/L3_absmax": 0.41915485, "gate_c2b/L3_rms": 0.18745099, "gate_c2b/L5_absmean": 0.52262598, "gate_c2b/L5_absmax": 0.73723739, "gate_c2b/L5_rms": 0.53353941, "gate_b2a/L7_absmean": 0.05075864, "gate_b2a/L7_absmax": 0.32423162, "gate_b2a/L7_rms": 0.05635563, "gate_b2a/L11_absmean": 0.08187336, "gate_b2a/L11_absmax": 0.60181534, "gate_b2a/L11_rms": 0.09100241, "gate_b2a/L14_absmean": 0.21179299, "gate_b2a/L14_absmax": 0.79743099, "gate_b2a/L14_rms": 0.22046055, "gate_b2c/L3_absmean": 0.03588084, "gate_b2c/L3_absmax": 0.5665518, "gate_b2c/L3_rms": 0.04699253, "gate_b2c/L5_absmean": 0.03845448, "gate_b2c/L5_absmax": 0.56693578, "gate_b2c/L5_rms": 0.04849956, "gate_b2c/L6_absmean": 0.05512201, "gate_b2c/L6_absmax": 0.36800566, "gate_b2c/L6_rms": 0.06168559}, "streams": {"cos_a2b/round0": 0.360857, "cos_c2b/round0": 0.300216, "cos_b2a/round0": 0.15339, "cos_b2c/round0": 0.13741, "rms_a/round0": 6.289112, "rms_b/round0": 11.319722, "rms_c/round0": 4.320052, "cos_a2b/round1": 0.418856, "cos_c2b/round1": 0.305388, "cos_b2a/round1": 0.251234, "cos_b2c/round1": 0.113204, "rms_a/round1": 12.253594, "rms_b/round1": 25.785791, "rms_c/round1": 8.955471, "cos_a2b/round2": 0.552764, "cos_c2b/round2": 0.523661, "cos_b2a/round2": 0.097965, "cos_b2c/round2": 0.181312, "rms_a/round2": 33.952187, "rms_b/round2": 69.328514, "rms_c/round2": 16.997831}, "alpha_mean": 0.4368608063748809, "alpha_hist": [0.00754, 0.05714, 0.17226, 0.21574, 0.18941, 0.16225, 0.12679, 0.062, 0.00685, 3e-05], "alpha_by_type": {"special": {"share": 0.00114, "mean": 0.51151, "std": 0.12728}, "whitespace": {"share": 0.02454, "mean": 0.47201, "std": 0.10408}, "digit": {"share": 0.02502, "mean": 0.45791, "std": 0.11513}, "word_start": {"share": 0.57617, "mean": 0.4587, "std": 0.15862}, "word_cont": {"share": 0.26471, "mean": 0.32351, "std": 0.12358}, "punct": {"share": 0.10841, "mean": 0.58396, "std": 0.15124}}, "alpha_p05": 0.19621, "alpha_p50": 0.43494, "alpha_p95": 0.72135}
{"step": 49810, "total_steps": 72479, "tokens": 26114785280, "loss": 4.487643837928772, "l_mix": 2.7817173302173615, "l_a": 2.868328779935837, "l_c": 2.81809264421463, "z_loss": 38.63036870956421, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12072128057479858, "tokens_per_second": 1367531.2798806934, "eta_seconds": 8690.905171132088, "data_wait_seconds": 1.5735626220703125e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 49820, "total_steps": 72479, "tokens": 26120028160, "loss": 4.473001062870026, "l_mix": 2.7722043991088867, "l_a": 2.86073899269104, "l_c": 2.8085830211639404, "z_loss": 38.75813341140747, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11801564693450928, "tokens_per_second": 1475853.8405733635, "eta_seconds": 8049.4703915834425, "data_wait_seconds": 1.683831214904785e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 49830, "total_steps": 72479, "tokens": 26125271040, "loss": 4.412159979343414, "l_mix": 2.734730541706085, "l_a": 2.820652723312378, "l_c": 2.7707785069942474, "z_loss": 38.65062665939331, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12001314759254456, "tokens_per_second": 1453243.4315955176, "eta_seconds": 8171.101037740707, "data_wait_seconds": 2.2143125534057617e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 49840, "total_steps": 72479, "tokens": 26130513920, "loss": 4.555270969867706, "l_mix": 2.8245158791542053, "l_a": 2.9075050354003906, "l_c": 2.861678272485733, "z_loss": 38.50830698013306, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14619819819927216, "tokens_per_second": 1434699.4578159107, "eta_seconds": 8273.060930871963, "data_wait_seconds": 1.7881393432617188e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 49850, "total_steps": 72479, "tokens": 26135756800, "loss": 4.460061550140381, "l_mix": 2.7642225325107574, "l_a": 2.8515532314777374, "l_c": 2.801243394613266, "z_loss": 38.310606479644775, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14706888794898987, "tokens_per_second": 1422692.8927323548, "eta_seconds": 8339.194785189628, "data_wait_seconds": 2.2292137145996094e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 49860, "total_steps": 72479, "tokens": 26140999680, "loss": 3.1087570190429688, "l_mix": 1.9227058738470078, "l_a": 1.9735200852155685, "l_c": 1.9799836575984955, "z_loss": 44.77004957199097, "lr": 0.0023, "wd": 0.01, "grad_norm": 1.039583683013916, "tokens_per_second": 1442992.9362023412, "eta_seconds": 8218.245546793938, "data_wait_seconds": 2.092123031616211e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 49870, "total_steps": 72479, "tokens": 26146242560, "loss": 3.143247902393341, "l_mix": 1.9341684430837631, "l_a": 2.048725515604019, "l_c": 1.9815390557050705, "z_loss": 41.69123935699463, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15766972303390503, "tokens_per_second": 1479415.95479211, "eta_seconds": 8012.36957976818, "data_wait_seconds": 1.9341707229614258e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 49880, "total_steps": 72479, "tokens": 26151485440, "loss": 3.1186127066612244, "l_mix": 1.9178852140903473, "l_a": 2.032344177365303, "l_c": 1.9700807929039001, "z_loss": 42.0529465675354, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16484694182872772, "tokens_per_second": 1411417.0011832928, "eta_seconds": 8394.67322702408, "data_wait_seconds": 1.9341707229614258e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 49890, "total_steps": 72479, "tokens": 26156728320, "loss": 3.1268530786037445, "l_mix": 1.9239431768655777, "l_a": 2.0399944484233856, "l_c": 1.969705045223236, "z_loss": 41.79596757888794, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1426859050989151, "tokens_per_second": 1273871.3703316352, "eta_seconds": 9296.96820874214, "data_wait_seconds": 3.4809112548828125e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 49900, "total_steps": 72479, "tokens": 26161971200, "loss": 4.577044725418091, "l_mix": 2.83776718378067, "l_a": 2.921231687068939, "l_c": 2.87635999917984, "z_loss": 38.189712047576904, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11985858529806137, "tokens_per_second": 1437429.3462916317, "eta_seconds": 8235.464777827263, "data_wait_seconds": 1.609325408935547e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.17951588, "gate_a2b/L1_absmax": 0.71764547, "gate_a2b/L1_rms": 0.19111043, "gate_a2b/L3_absmean": 0.17446384, "gate_a2b/L3_absmax": 0.55199963, "gate_a2b/L3_rms": 0.18065353, "gate_a2b/L5_absmean": 0.26492026, "gate_a2b/L5_absmax": 0.73606026, "gate_a2b/L5_rms": 0.27100468, "gate_c2b/L1_absmean": 0.02319628, "gate_c2b/L1_absmax": 0.6336031, "gate_c2b/L1_rms": 0.06628837, "gate_c2b/L3_absmean": 0.18310355, "gate_c2b/L3_absmax": 0.41155061, "gate_c2b/L3_rms": 0.18704066, "gate_c2b/L5_absmean": 0.52224392, "gate_c2b/L5_absmax": 0.73008031, "gate_c2b/L5_rms": 0.53304255, "gate_b2a/L7_absmean": 0.05087586, "gate_b2a/L7_absmax": 0.32706067, "gate_b2a/L7_rms": 0.05636267, "gate_b2a/L11_absmean": 0.0825756, "gate_b2a/L11_absmax": 0.60528237, "gate_b2a/L11_rms": 0.09167626, "gate_b2a/L14_absmean": 0.21158108, "gate_b2a/L14_absmax": 0.80236584, "gate_b2a/L14_rms": 0.22045675, "gate_b2c/L3_absmean": 0.03607325, "gate_b2c/L3_absmax": 0.56727618, "gate_b2c/L3_rms": 0.04708138, "gate_b2c/L5_absmean": 0.03836852, "gate_b2c/L5_absmax": 0.56271219, "gate_b2c/L5_rms": 0.04831444, "gate_b2c/L6_absmean": 0.05524781, "gate_b2c/L6_absmax": 0.3708289, "gate_b2c/L6_rms": 0.06184709}, "streams": {"cos_a2b/round0": 0.360572, "cos_c2b/round0": 0.290439, "cos_b2a/round0": 0.149525, "cos_b2c/round0": 0.136167, "rms_a/round0": 6.302916, "rms_b/round0": 11.387616, "rms_c/round0": 4.339781, "cos_a2b/round1": 0.411059, "cos_c2b/round1": 0.302262, "cos_b2a/round1": 0.249215, "cos_b2c/round1": 0.110018, "rms_a/round1": 12.215675, "rms_b/round1": 25.992678, "rms_c/round1": 8.981283, "cos_a2b/round2": 0.542421, "cos_c2b/round2": 0.520721, "cos_b2a/round2": 0.099986, "cos_b2c/round2": 0.179548, "rms_a/round2": 33.935467, "rms_b/round2": 69.22258, "rms_c/round2": 17.14765}, "alpha_mean": 0.4352154809596911, "alpha_hist": [0.01292, 0.07309, 0.17409, 0.19052, 0.18274, 0.16434, 0.12042, 0.07127, 0.01033, 0.00027], "alpha_by_type": {"special": {"share": 0.00119, "mean": 0.58243, "std": 0.12038}, "whitespace": {"share": 0.0202, "mean": 0.52352, "std": 0.13359}, "digit": {"share": 0.03455, "mean": 0.45242, "std": 0.13511}, "word_start": {"share": 0.57394, "mean": 0.45575, "std": 0.16845}, "word_cont": {"share": 0.25165, "mean": 0.31574, "std": 0.13374}, "punct": {"share": 0.11847, "mean": 0.56798, "std": 0.16297}}, "alpha_p05": 0.15922, "alpha_p50": 0.42012, "alpha_p95": 0.73905}
{"step": 49910, "total_steps": 72479, "tokens": 26167214080, "loss": 4.470882177352905, "l_mix": 2.771449714899063, "l_a": 2.856389045715332, "l_c": 2.808385193347931, "z_loss": 37.87105131149292, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1827128529548645, "tokens_per_second": 1413863.2717710298, "eta_seconds": 8369.02415406704, "data_wait_seconds": 1.874566078186035e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 49920, "total_steps": 72479, "tokens": 26172456960, "loss": 4.497626900672913, "l_mix": 2.787321627140045, "l_a": 2.8753791749477386, "l_c": 2.8256381452083588, "z_loss": 38.54103136062622, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1458577811717987, "tokens_per_second": 1352454.8305786864, "eta_seconds": 8745.144550919533, "data_wait_seconds": 2.0772218704223633e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 49930, "total_steps": 72479, "tokens": 26177699840, "loss": 3.1373239159584045, "l_mix": 1.9306785315275192, "l_a": 2.044672131538391, "l_c": 1.9774788320064545, "z_loss": 41.62896251678467, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16246706247329712, "tokens_per_second": 1471303.171124599, "eta_seconds": 8035.169327449798, "data_wait_seconds": 1.6987323760986328e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 49940, "total_steps": 72479, "tokens": 26182942720, "loss": 4.817193627357483, "l_mix": 2.988804042339325, "l_a": 3.066666603088379, "l_c": 3.02796471118927, "z_loss": 38.284329891204834, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13295066356658936, "tokens_per_second": 1430332.9866825633, "eta_seconds": 8261.661684393883, "data_wait_seconds": 3.159046173095703e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 49950, "total_steps": 72479, "tokens": 26188185600, "loss": 4.734711289405823, "l_mix": 2.938121199607849, "l_a": 3.0118055641651154, "l_c": 2.9768278002738953, "z_loss": 36.760584354400635, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15818752348423004, "tokens_per_second": 1427913.164026657, "eta_seconds": 8271.990657114982, "data_wait_seconds": 1.84476375579834e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 49960, "total_steps": 72479, "tokens": 26193428480, "loss": 4.679062068462372, "l_mix": 2.902112603187561, "l_a": 2.9808183908462524, "l_c": 2.9423464834690094, "z_loss": 37.728886127471924, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19661933183670044, "tokens_per_second": 1406761.310540075, "eta_seconds": 8392.640161156654, "data_wait_seconds": 0.005864322185516357, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 49970, "total_steps": 72479, "tokens": 26198671360, "loss": 4.594994068145752, "l_mix": 2.850950449705124, "l_a": 2.9263924658298492, "l_c": 2.88708558678627, "z_loss": 38.16153335571289, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17999061942100525, "tokens_per_second": 1453318.8259978078, "eta_seconds": 8120.171830773354, "data_wait_seconds": 2.9355287551879883e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 49980, "total_steps": 72479, "tokens": 26203914240, "loss": 3.1455246210098267, "l_mix": 1.9345472604036331, "l_a": 2.0498763024806976, "l_c": 1.986714705824852, "z_loss": 41.19368886947632, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19505444169044495, "tokens_per_second": 1427996.709701794, "eta_seconds": 8260.492220926286, "data_wait_seconds": 3.0219554901123047e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 49990, "total_steps": 72479, "tokens": 26209157120, "loss": 3.115804374217987, "l_mix": 1.9170918762683868, "l_a": 2.0324384719133377, "l_c": 1.9632696211338043, "z_loss": 41.85109043121338, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18295201659202576, "tokens_per_second": 1435588.0241184072, "eta_seconds": 8213.159091544152, "data_wait_seconds": 1.800060272216797e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 50000, "total_steps": 72479, "tokens": 26214400000, "loss": 3.1219153702259064, "l_mix": 1.9206082671880722, "l_a": 2.0356917828321457, "l_c": 1.968664988875389, "z_loss": 41.3200478553772, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15202398598194122, "tokens_per_second": 1421461.9581966614, "eta_seconds": 8291.090650749207, "data_wait_seconds": 1.683831214904785e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.17906408, "gate_a2b/L1_absmax": 0.72075373, "gate_a2b/L1_rms": 0.19060588, "gate_a2b/L3_absmean": 0.17418526, "gate_a2b/L3_absmax": 0.55997622, "gate_a2b/L3_rms": 0.18037225, "gate_a2b/L5_absmean": 0.26417023, "gate_a2b/L5_absmax": 0.73425633, "gate_a2b/L5_rms": 0.27021286, "gate_c2b/L1_absmean": 0.02251538, "gate_c2b/L1_absmax": 0.63612914, "gate_c2b/L1_rms": 0.06628842, "gate_c2b/L3_absmean": 0.18329242, "gate_c2b/L3_absmax": 0.41734231, "gate_c2b/L3_rms": 0.18721852, "gate_c2b/L5_absmean": 0.52187252, "gate_c2b/L5_absmax": 0.73547798, "gate_c2b/L5_rms": 0.53272122, "gate_b2a/L7_absmean": 0.05085426, "gate_b2a/L7_absmax": 0.32462347, "gate_b2a/L7_rms": 0.05632498, "gate_b2a/L11_absmean": 0.08194033, "gate_b2a/L11_absmax": 0.60280114, "gate_b2a/L11_rms": 0.09103241, "gate_b2a/L14_absmean": 0.21217096, "gate_b2a/L14_absmax": 0.8019504, "gate_b2a/L14_rms": 0.22093204, "gate_b2c/L3_absmean": 0.03585258, "gate_b2c/L3_absmax": 0.56739533, "gate_b2c/L3_rms": 0.04699976, "gate_b2c/L5_absmean": 0.03812521, "gate_b2c/L5_absmax": 0.56329519, "gate_b2c/L5_rms": 0.04810568, "gate_b2c/L6_absmean": 0.05492879, "gate_b2c/L6_absmax": 0.36909008, "gate_b2c/L6_rms": 0.06146212}, "streams": {"cos_a2b/round0": 0.356984, "cos_c2b/round0": 0.255373, "cos_b2a/round0": 0.142194, "cos_b2c/round0": 0.1152, "rms_a/round0": 6.588633, "rms_b/round0": 11.610291, "rms_c/round0": 4.439736, "cos_a2b/round1": 0.408574, "cos_c2b/round1": 0.305448, "cos_b2a/round1": 0.236765, "cos_b2c/round1": 0.119449, "rms_a/round1": 13.025813, "rms_b/round1": 27.376261, "rms_c/round1": 9.620459, "cos_a2b/round2": 0.522057, "cos_c2b/round2": 0.511056, "cos_b2a/round2": 0.091046, "cos_b2c/round2": 0.16879, "rms_a/round2": 34.899612, "rms_b/round2": 70.905388, "rms_c/round2": 18.476414}, "alpha_mean": 0.41975983206882717, "alpha_hist": [0.03229, 0.14648, 0.18562, 0.15135, 0.12979, 0.11172, 0.10141, 0.0927, 0.04654, 0.00209], "alpha_by_type": {"special": {"share": 0.00117, "mean": 0.65948, "std": 0.11218}, "whitespace": {"share": 0.03455, "mean": 0.60005, "std": 0.16743}, "digit": {"share": 0.01068, "mean": 0.36945, "std": 0.15937}, "word_start": {"share": 0.57837, "mean": 0.42682, "std": 0.20794}, "word_cont": {"share": 0.25301, "mean": 0.2815, "std": 0.14556}, "punct": {"share": 0.12222, "mean": 0.62367, "std": 0.1854}}, "alpha_p05": 0.11556, "alpha_p50": 0.36978, "alpha_p95": 0.7947}
{"step": 50010, "total_steps": 72479, "tokens": 26219642880, "loss": 4.651992499828339, "l_mix": 2.884848266839981, "l_a": 2.9699029326438904, "l_c": 2.9205774664878845, "z_loss": 37.559760093688965, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13737300038337708, "tokens_per_second": 1456803.9381598237, "eta_seconds": 8086.3503752470015, "data_wait_seconds": 2.4020671844482422e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 50020, "total_steps": 72479, "tokens": 26224885760, "loss": 4.452943682670593, "l_mix": 2.760293573141098, "l_a": 2.8466287553310394, "l_c": 2.795537829399109, "z_loss": 40.0214729309082, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.24064503610134125, "tokens_per_second": 1422445.98323485, "eta_seconds": 8277.983368635178, "data_wait_seconds": 1.9043684005737305e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 50030, "total_steps": 72479, "tokens": 26230128640, "loss": 4.4674259424209595, "l_mix": 2.767956793308258, "l_a": 2.856785923242569, "l_c": 2.808111071586609, "z_loss": 39.37191152572632, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14109539985656738, "tokens_per_second": 1420110.9841820425, "eta_seconds": 8287.9024548769, "data_wait_seconds": 3.069639205932617e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 50040, "total_steps": 72479, "tokens": 26235371520, "loss": 3.1809810996055603, "l_mix": 1.9579189866781235, "l_a": 2.0697784423828125, "l_c": 2.0070949345827103, "z_loss": 40.70412826538086, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.161681666970253, "tokens_per_second": 1066980.6675977458, "eta_seconds": 11025.97150001526, "data_wait_seconds": 1.9371509552001953e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 50050, "total_steps": 72479, "tokens": 26240614400, "loss": 3.106738716363907, "l_mix": 1.9121410548686981, "l_a": 2.0225190222263336, "l_c": 1.9594730585813522, "z_loss": 41.28327178955078, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1760508269071579, "tokens_per_second": 1476240.536150929, "eta_seconds": 7965.677180671692, "data_wait_seconds": 1.6391277313232422e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 50060, "total_steps": 72479, "tokens": 26245857280, "loss": 3.109408587217331, "l_mix": 1.9127242714166641, "l_a": 2.0298855900764465, "l_c": 1.95906201004982, "z_loss": 41.80129146575928, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13961629569530487, "tokens_per_second": 1428756.8585951745, "eta_seconds": 8226.741031050682, "data_wait_seconds": 2.6553869247436523e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 50070, "total_steps": 72479, "tokens": 26251100160, "loss": 4.641515523195267, "l_mix": 2.880129396915436, "l_a": 2.9523602426052094, "l_c": 2.9189265370368958, "z_loss": 37.564887046813965, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14929476380348206, "tokens_per_second": 1388682.3744739767, "eta_seconds": 8460.372226190566, "data_wait_seconds": 2.1070241928100586e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 50080, "total_steps": 72479, "tokens": 26256343040, "loss": 4.7392430901527405, "l_mix": 2.9407710134983063, "l_a": 3.0157379806041718, "l_c": 2.979168653488159, "z_loss": 38.05193042755127, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.10969776660203934, "tokens_per_second": 1408489.4155938819, "eta_seconds": 8337.674945926667, "data_wait_seconds": 2.536177635192871e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 50090, "total_steps": 72479, "tokens": 26261585920, "loss": 3.1776552498340607, "l_mix": 1.9561065882444382, "l_a": 2.0681773722171783, "l_c": 2.003651335835457, "z_loss": 40.52582931518555, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1865413784980774, "tokens_per_second": 1416573.9561683543, "eta_seconds": 8286.389835762977, "data_wait_seconds": 1.9609928131103516e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 50100, "total_steps": 72479, "tokens": 26266828800, "loss": 3.128234475851059, "l_mix": 1.9250094145536423, "l_a": 2.0377468168735504, "l_c": 1.973003163933754, "z_loss": 41.9534707069397, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16116835176944733, "tokens_per_second": 1414382.708586923, "eta_seconds": 8295.520781445504, "data_wait_seconds": 1.5795230865478516e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.17906661, "gate_a2b/L1_absmax": 0.71818084, "gate_a2b/L1_rms": 0.19052421, "gate_a2b/L3_absmean": 0.17353147, "gate_a2b/L3_absmax": 0.55815166, "gate_a2b/L3_rms": 0.17973733, "gate_a2b/L5_absmean": 0.26365989, "gate_a2b/L5_absmax": 0.73440075, "gate_a2b/L5_rms": 0.26971886, "gate_c2b/L1_absmean": 0.02252559, "gate_c2b/L1_absmax": 0.63569421, "gate_c2b/L1_rms": 0.06629283, "gate_c2b/L3_absmean": 0.18308114, "gate_c2b/L3_absmax": 0.40866598, "gate_c2b/L3_rms": 0.18696257, "gate_c2b/L5_absmean": 0.5213111, "gate_c2b/L5_absmax": 0.7429561, "gate_c2b/L5_rms": 0.53212917, "gate_b2a/L7_absmean": 0.05050138, "gate_b2a/L7_absmax": 0.32477865, "gate_b2a/L7_rms": 0.05601052, "gate_b2a/L11_absmean": 0.08226572, "gate_b2a/L11_absmax": 0.60428846, "gate_b2a/L11_rms": 0.09133119, "gate_b2a/L14_absmean": 0.21191406, "gate_b2a/L14_absmax": 0.80136728, "gate_b2a/L14_rms": 0.22067912, "gate_b2c/L3_absmean": 0.03585787, "gate_b2c/L3_absmax": 0.56763619, "gate_b2c/L3_rms": 0.04694874, "gate_b2c/L5_absmean": 0.03835695, "gate_b2c/L5_absmax": 0.56408387, "gate_b2c/L5_rms": 0.04830433, "gate_b2c/L6_absmean": 0.05492228, "gate_b2c/L6_absmax": 0.36637688, "gate_b2c/L6_rms": 0.06146307}, "streams": {"cos_a2b/round0": 0.358858, "cos_c2b/round0": 0.262792, "cos_b2a/round0": 0.143504, "cos_b2c/round0": 0.116282, "rms_a/round0": 6.573647, "rms_b/round0": 11.508742, "rms_c/round0": 4.3922, "cos_a2b/round1": 0.412021, "cos_c2b/round1": 0.301945, "cos_b2a/round1": 0.24094, "cos_b2c/round1": 0.121023, "rms_a/round1": 12.965931, "rms_b/round1": 27.383337, "rms_c/round1": 9.473463, "cos_a2b/round2": 0.520313, "cos_c2b/round2": 0.515887, "cos_b2a/round2": 0.090906, "cos_b2c/round2": 0.171551, "rms_a/round2": 34.738678, "rms_b/round2": 71.606728, "rms_c/round2": 18.387707}, "alpha_mean": 0.41339983809697856, "alpha_hist": [0.03406, 0.16393, 0.18057, 0.15504, 0.12119, 0.1073, 0.09882, 0.08736, 0.04854, 0.0032], "alpha_by_type": {"special": {"share": 0.00116, "mean": 0.67497, "std": 0.10329}, "whitespace": {"share": 0.03629, "mean": 0.62052, "std": 0.16879}, "digit": {"share": 0.0143, "mean": 0.39519, "std": 0.14033}, "word_start": {"share": 0.56137, "mean": 0.41596, "std": 0.21274}, "word_cont": {"share": 0.27122, "mean": 0.28608, "std": 0.14463}, "punct": {"share": 0.11566, "mean": 0.63417, "std": 0.18852}}, "alpha_p05": 0.12065, "alpha_p50": 0.38237, "alpha_p95": 0.82332}
{"step": 50110, "total_steps": 72479, "tokens": 26272071680, "loss": 4.394592463970184, "l_mix": 2.7239654064178467, "l_a": 2.809319168329239, "l_c": 2.75943723320961, "z_loss": 38.990803241729736, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14437606930732727, "tokens_per_second": 1453235.3643775666, "eta_seconds": 8070.129973077774, "data_wait_seconds": 1.7344951629638672e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 50120, "total_steps": 72479, "tokens": 26277314560, "loss": 4.432704389095306, "l_mix": 2.7471322417259216, "l_a": 2.833107203245163, "l_c": 2.785466343164444, "z_loss": 39.12409067153931, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.10784254968166351, "tokens_per_second": 930221.9727916969, "eta_seconds": 12601.890446448326, "data_wait_seconds": 0.2573981285095215, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 50130, "total_steps": 72479, "tokens": 26282557440, "loss": 4.515296697616577, "l_mix": 2.798582136631012, "l_a": 2.8861142694950104, "l_c": 2.8362674713134766, "z_loss": 38.84013032913208, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1078917607665062, "tokens_per_second": 1364377.900235311, "eta_seconds": 8588.025729513169, "data_wait_seconds": 2.372264862060547e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 50140, "total_steps": 72479, "tokens": 26287800320, "loss": 4.415664821863174, "l_mix": 2.7366731464862823, "l_a": 2.8228311836719513, "l_c": 2.773807555437088, "z_loss": 39.737778663635254, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18176034092903137, "tokens_per_second": 1297975.218764356, "eta_seconds": 9023.338398671149, "data_wait_seconds": 4.461407661437988e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 50150, "total_steps": 72479, "tokens": 26293043200, "loss": 3.0971643030643463, "l_mix": 1.9049137979745865, "l_a": 2.0195006728172302, "l_c": 1.9546674340963364, "z_loss": 42.07267475128174, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16881823539733887, "tokens_per_second": 1300350.810450003, "eta_seconds": 9002.821898460388, "data_wait_seconds": 3.36766242980957e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 50160, "total_steps": 72479, "tokens": 26298286080, "loss": 3.185240089893341, "l_mix": 1.9602866917848587, "l_a": 2.07466858625412, "l_c": 2.0085092335939407, "z_loss": 42.763251304626465, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.22108905017375946, "tokens_per_second": 1201747.8636968925, "eta_seconds": 9737.137236094475, "data_wait_seconds": 1.6570091247558594e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 50170, "total_steps": 72479, "tokens": 26303528960, "loss": 3.1331637799739838, "l_mix": 1.9280415177345276, "l_a": 2.0388741493225098, "l_c": 1.9781996309757233, "z_loss": 41.57722091674805, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15774360299110413, "tokens_per_second": 1460228.423760846, "eta_seconds": 8009.939268183708, "data_wait_seconds": 2.1964311599731445e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 50180, "total_steps": 72479, "tokens": 26308771840, "loss": 3.1529315412044525, "l_mix": 1.940551832318306, "l_a": 2.0522022396326065, "l_c": 1.989063248038292, "z_loss": 40.50544500350952, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2389007806777954, "tokens_per_second": 1429893.2543317536, "eta_seconds": 8176.203416991235, "data_wait_seconds": 1.8477439880371094e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 50190, "total_steps": 72479, "tokens": 26314014720, "loss": 3.16800194978714, "l_mix": 1.948631927371025, "l_a": 2.0664226710796356, "l_c": 1.9981438517570496, "z_loss": 42.342331886291504, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.23878958821296692, "tokens_per_second": 1222623.1118425892, "eta_seconds": 9558.01924469471, "data_wait_seconds": 2.1129846572875977e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 50200, "total_steps": 72479, "tokens": 26319257600, "loss": 3.1415484845638275, "l_mix": 1.933170735836029, "l_a": 2.0466273427009583, "l_c": 1.9812983870506287, "z_loss": 41.24040508270264, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1632768213748932, "tokens_per_second": 1238270.9728038816, "eta_seconds": 9433.001829600333, "data_wait_seconds": 1.9222497940063477e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.17938314, "gate_a2b/L1_absmax": 0.7162416, "gate_a2b/L1_rms": 0.19075075, "gate_a2b/L3_absmean": 0.17371266, "gate_a2b/L3_absmax": 0.55685455, "gate_a2b/L3_rms": 0.17986071, "gate_a2b/L5_absmean": 0.26306975, "gate_a2b/L5_absmax": 0.73395473, "gate_a2b/L5_rms": 0.26914001, "gate_c2b/L1_absmean": 0.02277677, "gate_c2b/L1_absmax": 0.63437665, "gate_c2b/L1_rms": 0.06657427, "gate_c2b/L3_absmean": 0.18307565, "gate_c2b/L3_absmax": 0.41084871, "gate_c2b/L3_rms": 0.18699411, "gate_c2b/L5_absmean": 0.5207895, "gate_c2b/L5_absmax": 0.73814052, "gate_c2b/L5_rms": 0.53158891, "gate_b2a/L7_absmean": 0.05048939, "gate_b2a/L7_absmax": 0.32492846, "gate_b2a/L7_rms": 0.05608255, "gate_b2a/L11_absmean": 0.08196272, "gate_b2a/L11_absmax": 0.60301888, "gate_b2a/L11_rms": 0.09107089, "gate_b2a/L14_absmean": 0.21172088, "gate_b2a/L14_absmax": 0.79912263, "gate_b2a/L14_rms": 0.22047339, "gate_b2c/L3_absmean": 0.03560921, "gate_b2c/L3_absmax": 0.56828064, "gate_b2c/L3_rms": 0.04682417, "gate_b2c/L5_absmean": 0.03804487, "gate_b2c/L5_absmax": 0.56488949, "gate_b2c/L5_rms": 0.04812004, "gate_b2c/L6_absmean": 0.0549056, "gate_b2c/L6_absmax": 0.36087036, "gate_b2c/L6_rms": 0.0613538}, "streams": {"cos_a2b/round0": 0.361298, "cos_c2b/round0": 0.260988, "cos_b2a/round0": 0.142124, "cos_b2c/round0": 0.117233, "rms_a/round0": 6.595015, "rms_b/round0": 11.450885, "rms_c/round0": 4.422865, "cos_a2b/round1": 0.410575, "cos_c2b/round1": 0.305281, "cos_b2a/round1": 0.241778, "cos_b2c/round1": 0.118621, "rms_a/round1": 12.894933, "rms_b/round1": 27.191917, "rms_c/round1": 9.539093, "cos_a2b/round2": 0.515997, "cos_c2b/round2": 0.518906, "cos_b2a/round2": 0.08724, "cos_b2c/round2": 0.17076, "rms_a/round2": 34.447994, "rms_b/round2": 70.738922, "rms_c/round2": 18.26627}, "alpha_mean": 0.40379538713271046, "alpha_hist": [0.03622, 0.16745, 0.19403, 0.14792, 0.12054, 0.1095, 0.10339, 0.08212, 0.03752, 0.0013], "alpha_by_type": {"special": {"share": 0.00104, "mean": 0.64, "std": 0.12714}, "whitespace": {"share": 0.03949, "mean": 0.60824, "std": 0.1662}, "digit": {"share": 0.01276, "mean": 0.40388, "std": 0.16571}, "word_start": {"share": 0.55432, "mean": 0.40308, "std": 0.2063}, "word_cont": {"share": 0.26489, "mean": 0.27415, "std": 0.14066}, "punct": {"share": 0.1275, "mean": 0.611, "std": 0.17917}}, "alpha_p05": 0.1192, "alpha_p50": 0.36252, "alpha_p95": 0.78956}
{"step": 50210, "total_steps": 72479, "tokens": 26324500480, "loss": 3.15049546957016, "l_mix": 1.9396397918462753, "l_a": 2.0501191318035126, "l_c": 1.9860662817955017, "z_loss": 40.32921648025513, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1993042677640915, "tokens_per_second": 1395734.6796426093, "eta_seconds": 8365.03501868248, "data_wait_seconds": 2.2172927856445312e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 50220, "total_steps": 72479, "tokens": 26329743360, "loss": 4.309816628694534, "l_mix": 2.6713102757930756, "l_a": 2.754467934370041, "l_c": 2.707219570875168, "z_loss": 39.40370512008667, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1216447651386261, "tokens_per_second": 1268925.9284884995, "eta_seconds": 9196.854071617126, "data_wait_seconds": 1.7464160919189453e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 50230, "total_steps": 72479, "tokens": 26334986240, "loss": 4.462376534938812, "l_mix": 2.7664073705673218, "l_a": 2.84975728392601, "l_c": 2.803473025560379, "z_loss": 39.514779567718506, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17854656279087067, "tokens_per_second": 1437980.445210778, "eta_seconds": 8111.990500879288, "data_wait_seconds": 1.8894672393798828e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 50240, "total_steps": 72479, "tokens": 26340229120, "loss": 4.397920668125153, "l_mix": 2.72577565908432, "l_a": 2.811435729265213, "l_c": 2.762380689382553, "z_loss": 38.809261322021484, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12408364564180374, "tokens_per_second": 1400833.5958209098, "eta_seconds": 8323.358939123153, "data_wait_seconds": 1.6450881958007812e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 50250, "total_steps": 72479, "tokens": 26345472000, "loss": 4.353086888790131, "l_mix": 2.696908473968506, "l_a": 2.7862879037857056, "l_c": 2.7343066930770874, "z_loss": 39.973825454711914, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17190545797348022, "tokens_per_second": 1322206.108859523, "eta_seconds": 8814.357968783379, "data_wait_seconds": 2.002716064453125e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 50260, "total_steps": 72479, "tokens": 26350714880, "loss": 3.141119569540024, "l_mix": 1.931936502456665, "l_a": 2.0465746223926544, "l_c": 1.9840355962514877, "z_loss": 41.81047439575195, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20744732022285461, "tokens_per_second": 1463190.9598723957, "eta_seconds": 7961.4728299140925, "data_wait_seconds": 2.637505531311035e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 50270, "total_steps": 72479, "tokens": 26355957760, "loss": 3.1464633643627167, "l_mix": 1.9370025396347046, "l_a": 2.048774152994156, "l_c": 1.9827619045972824, "z_loss": 40.594117641448975, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17746485769748688, "tokens_per_second": 1410368.3848085627, "eta_seconds": 8255.936759090424, "data_wait_seconds": 3.263354301452637e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 50280, "total_steps": 72479, "tokens": 26361200640, "loss": 3.126892387866974, "l_mix": 1.9231619238853455, "l_a": 2.039054796099663, "l_c": 1.9733798503875732, "z_loss": 40.711286544799805, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.22684146463871002, "tokens_per_second": 1185840.713106049, "eta_seconds": 9814.698705625535, "data_wait_seconds": 2.339482307434082e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 50290, "total_steps": 72479, "tokens": 26366443520, "loss": 3.1588414013385773, "l_mix": 1.9442336112260818, "l_a": 2.0575020909309387, "l_c": 1.9911903142929077, "z_loss": 42.30645990371704, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.162403866648674, "tokens_per_second": 1419774.3980723813, "eta_seconds": 8193.855620861053, "data_wait_seconds": 2.4646520614624023e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 50300, "total_steps": 72479, "tokens": 26371686400, "loss": 3.1244588792324066, "l_mix": 1.9240161031484604, "l_a": 2.031355708837509, "l_c": 1.9701200425624847, "z_loss": 41.78243112564087, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.21356754004955292, "tokens_per_second": 1350343.5452277712, "eta_seconds": 8611.277917456628, "data_wait_seconds": 3.120303153991699e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.17926846, "gate_a2b/L1_absmax": 0.71407145, "gate_a2b/L1_rms": 0.19064277, "gate_a2b/L3_absmean": 0.17361306, "gate_a2b/L3_absmax": 0.55583185, "gate_a2b/L3_rms": 0.17977349, "gate_a2b/L5_absmean": 0.26265869, "gate_a2b/L5_absmax": 0.73382437, "gate_a2b/L5_rms": 0.26871315, "gate_c2b/L1_absmean": 0.02254414, "gate_c2b/L1_absmax": 0.63176256, "gate_c2b/L1_rms": 0.06649321, "gate_c2b/L3_absmean": 0.18299679, "gate_c2b/L3_absmax": 0.41616419, "gate_c2b/L3_rms": 0.18686104, "gate_c2b/L5_absmean": 0.52047491, "gate_c2b/L5_absmax": 0.73594648, "gate_c2b/L5_rms": 0.5313046, "gate_b2a/L7_absmean": 0.05087916, "gate_b2a/L7_absmax": 0.323504, "gate_b2a/L7_rms": 0.05641236, "gate_b2a/L11_absmean": 0.08190776, "gate_b2a/L11_absmax": 0.60311431, "gate_b2a/L11_rms": 0.09101925, "gate_b2a/L14_absmean": 0.21206674, "gate_b2a/L14_absmax": 0.79803514, "gate_b2a/L14_rms": 0.22077471, "gate_b2c/L3_absmean": 0.03587485, "gate_b2c/L3_absmax": 0.56745732, "gate_b2c/L3_rms": 0.04698993, "gate_b2c/L5_absmean": 0.03809312, "gate_b2c/L5_absmax": 0.56574863, "gate_b2c/L5_rms": 0.04814567, "gate_b2c/L6_absmean": 0.05493959, "gate_b2c/L6_absmax": 0.36098582, "gate_b2c/L6_rms": 0.06137711}, "streams": {"cos_a2b/round0": 0.359813, "cos_c2b/round0": 0.260629, "cos_b2a/round0": 0.144338, "cos_b2c/round0": 0.116702, "rms_a/round0": 6.595783, "rms_b/round0": 11.552089, "rms_c/round0": 4.420107, "cos_a2b/round1": 0.411362, "cos_c2b/round1": 0.307173, "cos_b2a/round1": 0.240845, "cos_b2c/round1": 0.121848, "rms_a/round1": 12.980835, "rms_b/round1": 27.373791, "rms_c/round1": 9.636066, "cos_a2b/round2": 0.524369, "cos_c2b/round2": 0.510691, "cos_b2a/round2": 0.093673, "cos_b2c/round2": 0.17379, "rms_a/round2": 34.966816, "rms_b/round2": 71.039986, "rms_c/round2": 18.631725}, "alpha_mean": 0.41898516179981726, "alpha_hist": [0.02596, 0.14162, 0.1935, 0.15823, 0.13043, 0.11374, 0.10594, 0.08881, 0.04022, 0.00156], "alpha_by_type": {"special": {"share": 0.00117, "mean": 0.65775, "std": 0.10956}, "whitespace": {"share": 0.03439, "mean": 0.60804, "std": 0.16517}, "digit": {"share": 0.01306, "mean": 0.42413, "std": 0.16968}, "word_start": {"share": 0.57063, "mean": 0.42197, "std": 0.20096}, "word_cont": {"share": 0.25485, "mean": 0.28509, "std": 0.14057}, "punct": {"share": 0.12589, "mean": 0.6221, "std": 0.17967}}, "alpha_p05": 0.12918, "alpha_p50": 0.38191, "alpha_p95": 0.79881}
{"step": 50310, "total_steps": 72479, "tokens": 26376929280, "loss": 3.1422010362148285, "l_mix": 1.932764396071434, "l_a": 2.048033833503723, "l_c": 1.9834213107824326, "z_loss": 41.50688171386719, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19992130994796753, "tokens_per_second": 1415040.5544480805, "eta_seconds": 8213.85693538189, "data_wait_seconds": 2.244114875793457e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 50320, "total_steps": 72479, "tokens": 26382172160, "loss": 4.277263551950455, "l_mix": 2.6494747400283813, "l_a": 2.7373020946979523, "l_c": 2.6886604130268097, "z_loss": 40.80635452270508, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14977768063545227, "tokens_per_second": 1430050.2174167088, "eta_seconds": 8123.978899836541, "data_wait_seconds": 1.901388168334961e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 50330, "total_steps": 72479, "tokens": 26387415040, "loss": 4.3545162081718445, "l_mix": 2.6980898082256317, "l_a": 2.7825462222099304, "l_c": 2.7388751208782196, "z_loss": 39.07918882369995, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20226919651031494, "tokens_per_second": 1443099.8477719666, "eta_seconds": 8046.882500839234, "data_wait_seconds": 4.0471553802490234e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 50340, "total_steps": 72479, "tokens": 26392657920, "loss": 4.287230432033539, "l_mix": 2.657061368227005, "l_a": 2.7417984008789062, "l_c": 2.692098170518875, "z_loss": 39.688547134399414, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1511601060628891, "tokens_per_second": 1336624.0573690727, "eta_seconds": 8683.97659611702, "data_wait_seconds": 1.9311904907226562e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 50350, "total_steps": 72479, "tokens": 26397900800, "loss": 4.3112727999687195, "l_mix": 2.670701503753662, "l_a": 2.760248839855194, "l_c": 2.7083221077919006, "z_loss": 40.902894020080566, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12118390202522278, "tokens_per_second": 1479772.554534829, "eta_seconds": 7840.373249554635, "data_wait_seconds": 2.0265579223632812e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 50360, "total_steps": 72479, "tokens": 26403143680, "loss": 4.391173422336578, "l_mix": 2.721010774374008, "l_a": 2.8065707087516785, "l_c": 2.7606379687786102, "z_loss": 40.25093126296997, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1298658400774002, "tokens_per_second": 1452971.0204737242, "eta_seconds": 7981.38855392933, "data_wait_seconds": 2.8014183044433594e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 50370, "total_steps": 72479, "tokens": 26408386560, "loss": 3.1256004571914673, "l_mix": 1.9235374182462692, "l_a": 2.0377259850502014, "l_c": 1.9691505879163742, "z_loss": 42.01622676849365, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20638443529605865, "tokens_per_second": 1424390.72350705, "eta_seconds": 8137.853750872612, "data_wait_seconds": 2.3037195205688477e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 50380, "total_steps": 72479, "tokens": 26413629440, "loss": 3.148765951395035, "l_mix": 1.9370117336511612, "l_a": 2.052404761314392, "l_c": 1.9867755770683289, "z_loss": 41.73520278930664, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1698753386735916, "tokens_per_second": 1423955.743193107, "eta_seconds": 8136.657734894753, "data_wait_seconds": 2.568960189819336e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 50390, "total_steps": 72479, "tokens": 26418872320, "loss": 3.1209468841552734, "l_mix": 1.9208432734012604, "l_a": 2.0347011387348175, "l_c": 1.9656441509723663, "z_loss": 40.98284864425659, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17445439100265503, "tokens_per_second": 1435635.354011922, "eta_seconds": 8066.810001325607, "data_wait_seconds": 2.676248550415039e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 50400, "total_steps": 72479, "tokens": 26424115200, "loss": 3.0865122079849243, "l_mix": 1.8988830000162125, "l_a": 2.013257533311844, "l_c": 1.9455064535140991, "z_loss": 41.29723119735718, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15551277995109558, "tokens_per_second": 1359948.2443136335, "eta_seconds": 8511.908302688598, "data_wait_seconds": 2.5004148483276367e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.17885804, "gate_a2b/L1_absmax": 0.71230131, "gate_a2b/L1_rms": 0.19019829, "gate_a2b/L3_absmean": 0.17341173, "gate_a2b/L3_absmax": 0.55385911, "gate_a2b/L3_rms": 0.17954434, "gate_a2b/L5_absmean": 0.26257429, "gate_a2b/L5_absmax": 0.7337091, "gate_a2b/L5_rms": 0.26860616, "gate_c2b/L1_absmean": 0.02290124, "gate_c2b/L1_absmax": 0.63170731, "gate_c2b/L1_rms": 0.06639451, "gate_c2b/L3_absmean": 0.18283676, "gate_c2b/L3_absmax": 0.41488808, "gate_c2b/L3_rms": 0.18668267, "gate_c2b/L5_absmean": 0.51971245, "gate_c2b/L5_absmax": 0.73536456, "gate_c2b/L5_rms": 0.53048062, "gate_b2a/L7_absmean": 0.05060976, "gate_b2a/L7_absmax": 0.3231999, "gate_b2a/L7_rms": 0.0561009, "gate_b2a/L11_absmean": 0.08216529, "gate_b2a/L11_absmax": 0.60424984, "gate_b2a/L11_rms": 0.09124148, "gate_b2a/L14_absmean": 0.21197848, "gate_b2a/L14_absmax": 0.79576981, "gate_b2a/L14_rms": 0.22068505, "gate_b2c/L3_absmean": 0.03571505, "gate_b2c/L3_absmax": 0.56653988, "gate_b2c/L3_rms": 0.04682276, "gate_b2c/L5_absmean": 0.03816424, "gate_b2c/L5_absmax": 0.56802887, "gate_b2c/L5_rms": 0.04825457, "gate_b2c/L6_absmean": 0.05506074, "gate_b2c/L6_absmax": 0.36069715, "gate_b2c/L6_rms": 0.06150652}, "streams": {"cos_a2b/round0": 0.358631, "cos_c2b/round0": 0.256954, "cos_b2a/round0": 0.139647, "cos_b2c/round0": 0.115427, "rms_a/round0": 6.637298, "rms_b/round0": 11.567411, "rms_c/round0": 4.420165, "cos_a2b/round1": 0.407831, "cos_c2b/round1": 0.300288, "cos_b2a/round1": 0.238148, "cos_b2c/round1": 0.117986, "rms_a/round1": 13.017352, "rms_b/round1": 27.494509, "rms_c/round1": 9.528534, "cos_a2b/round2": 0.520129, "cos_c2b/round2": 0.507378, "cos_b2a/round2": 0.090893, "cos_b2c/round2": 0.169765, "rms_a/round2": 34.971432, "rms_b/round2": 71.363297, "rms_c/round2": 18.433342}, "alpha_mean": 0.41952572545679345, "alpha_hist": [0.03242, 0.15021, 0.18741, 0.14691, 0.12167, 0.11481, 0.10818, 0.08821, 0.04773, 0.00244], "alpha_by_type": {"special": {"share": 0.00119, "mean": 0.65369, "std": 0.13372}, "whitespace": {"share": 0.03865, "mean": 0.61807, "std": 0.15205}, "digit": {"share": 0.02278, "mean": 0.48008, "std": 0.15594}, "word_start": {"share": 0.5625, "mean": 0.42204, "std": 0.20956}, "word_cont": {"share": 0.25354, "mean": 0.27726, "std": 0.14296}, "punct": {"share": 0.12134, "mean": 0.62821, "std": 0.18293}}, "alpha_p05": 0.11517, "alpha_p50": 0.38353, "alpha_p95": 0.80532}
{"step": 50410, "total_steps": 72479, "tokens": 26429358080, "loss": 3.1258103251457214, "l_mix": 1.9227433502674103, "l_a": 2.038912534713745, "l_c": 1.9713104367256165, "z_loss": 42.4661979675293, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14645375311374664, "tokens_per_second": 1437464.8640642618, "eta_seconds": 8049.248479914665, "data_wait_seconds": 2.8073787689208984e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 50420, "total_steps": 72479, "tokens": 26434600960, "loss": 3.1499318182468414, "l_mix": 1.9382241368293762, "l_a": 2.052485004067421, "l_c": 1.9865404069423676, "z_loss": 42.647090911865234, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17348644137382507, "tokens_per_second": 1456664.2050595859, "eta_seconds": 7939.557347416878, "data_wait_seconds": 2.6911497116088867e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 50430, "total_steps": 72479, "tokens": 26439843840, "loss": 4.334369599819183, "l_mix": 2.685833305120468, "l_a": 2.7706949412822723, "l_c": 2.724425733089447, "z_loss": 39.19247531890869, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1423162966966629, "tokens_per_second": 1284769.3261979048, "eta_seconds": 8997.744479322433, "data_wait_seconds": 2.7418136596679688e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 50440, "total_steps": 72479, "tokens": 26445086720, "loss": 4.296393811702728, "l_mix": 2.66144797205925, "l_a": 2.7537706196308136, "l_c": 2.6960487961769104, "z_loss": 40.51189613342285, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18496419489383698, "tokens_per_second": 1463912.3496257043, "eta_seconds": 7893.08405995369, "data_wait_seconds": 1.800060272216797e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 50450, "total_steps": 72479, "tokens": 26450329600, "loss": 4.3888800740242, "l_mix": 2.7204163670539856, "l_a": 2.8033778071403503, "l_c": 2.7581677734851837, "z_loss": 39.26962375640869, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13203351199626923, "tokens_per_second": 1350889.4616452197, "eta_seconds": 8549.582093811036, "data_wait_seconds": 1.615285873413086e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 50460, "total_steps": 72479, "tokens": 26455572480, "loss": 4.346597969532013, "l_mix": 2.6925769448280334, "l_a": 2.7816306352615356, "l_c": 2.731772482395172, "z_loss": 40.36397075653076, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1237466111779213, "tokens_per_second": 1282560.0768944668, "eta_seconds": 9000.979899477958, "data_wait_seconds": 1.8417835235595703e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 50470, "total_steps": 72479, "tokens": 26460815360, "loss": 4.554252445697784, "l_mix": 2.824352204799652, "l_a": 2.905822664499283, "l_c": 2.860510915517807, "z_loss": 40.66544961929321, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2745007872581482, "tokens_per_second": 1440818.470292852, "eta_seconds": 8008.680364608764, "data_wait_seconds": 2.9414892196655273e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 50480, "total_steps": 72479, "tokens": 26466058240, "loss": 3.1762428283691406, "l_mix": 1.9558015018701553, "l_a": 2.067594587802887, "l_c": 2.0005428344011307, "z_loss": 42.153581619262695, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.26890185475349426, "tokens_per_second": 1435037.258728021, "eta_seconds": 8037.2907684803, "data_wait_seconds": 1.862645149230957e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 50490, "total_steps": 72479, "tokens": 26471301120, "loss": 3.0953187942504883, "l_mix": 1.9045754671096802, "l_a": 2.014388456940651, "l_c": 1.9547559022903442, "z_loss": 41.27442407608032, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19771495461463928, "tokens_per_second": 1439002.83037019, "eta_seconds": 8011.4983714342125, "data_wait_seconds": 1.6242265701293945e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 50500, "total_steps": 72479, "tokens": 26476544000, "loss": 3.120794415473938, "l_mix": 1.9202521443367004, "l_a": 2.0341897308826447, "l_c": 1.9676176309585571, "z_loss": 42.22057628631592, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12681405246257782, "tokens_per_second": 1198333.7808475045, "eta_seconds": 9616.123767995834, "data_wait_seconds": 2.9265880584716797e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.17907268, "gate_a2b/L1_absmax": 0.71208471, "gate_a2b/L1_rms": 0.19038889, "gate_a2b/L3_absmean": 0.17278637, "gate_a2b/L3_absmax": 0.55336648, "gate_a2b/L3_rms": 0.17896722, "gate_a2b/L5_absmean": 0.26222882, "gate_a2b/L5_absmax": 0.73449016, "gate_a2b/L5_rms": 0.26824787, "gate_c2b/L1_absmean": 0.02278079, "gate_c2b/L1_absmax": 0.63102126, "gate_c2b/L1_rms": 0.06633431, "gate_c2b/L3_absmean": 0.18219864, "gate_c2b/L3_absmax": 0.41264459, "gate_c2b/L3_rms": 0.18609887, "gate_c2b/L5_absmean": 0.51900101, "gate_c2b/L5_absmax": 0.7349878, "gate_c2b/L5_rms": 0.52980787, "gate_b2a/L7_absmean": 0.05075663, "gate_b2a/L7_absmax": 0.32244807, "gate_b2a/L7_rms": 0.05628647, "gate_b2a/L11_absmean": 0.08178815, "gate_b2a/L11_absmax": 0.60246634, "gate_b2a/L11_rms": 0.09089855, "gate_b2a/L14_absmean": 0.2115939, "gate_b2a/L14_absmax": 0.79711062, "gate_b2a/L14_rms": 0.22035296, "gate_b2c/L3_absmean": 0.03564668, "gate_b2c/L3_absmax": 0.56723887, "gate_b2c/L3_rms": 0.04676553, "gate_b2c/L5_absmean": 0.0383696, "gate_b2c/L5_absmax": 0.56699735, "gate_b2c/L5_rms": 0.04839114, "gate_b2c/L6_absmean": 0.05486031, "gate_b2c/L6_absmax": 0.35981709, "gate_b2c/L6_rms": 0.0613157}, "streams": {"cos_a2b/round0": 0.359176, "cos_c2b/round0": 0.256746, "cos_b2a/round0": 0.141626, "cos_b2c/round0": 0.114925, "rms_a/round0": 6.648527, "rms_b/round0": 11.680987, "rms_c/round0": 4.448313, "cos_a2b/round1": 0.410263, "cos_c2b/round1": 0.30346, "cos_b2a/round1": 0.240794, "cos_b2c/round1": 0.122816, "rms_a/round1": 13.137384, "rms_b/round1": 27.604231, "rms_c/round1": 9.694818, "cos_a2b/round2": 0.519488, "cos_c2b/round2": 0.510968, "cos_b2a/round2": 0.090574, "cos_b2c/round2": 0.172143, "rms_a/round2": 35.211582, "rms_b/round2": 71.285652, "rms_c/round2": 18.747585}, "alpha_mean": 0.4148041883101854, "alpha_hist": [0.03368, 0.15788, 0.18787, 0.15471, 0.12086, 0.10539, 0.09891, 0.08537, 0.05148, 0.00385], "alpha_by_type": {"special": {"share": 0.00116, "mean": 0.66005, "std": 0.12676}, "whitespace": {"share": 0.03496, "mean": 0.62364, "std": 0.17177}, "digit": {"share": 0.01221, "mean": 0.39595, "std": 0.17}, "word_start": {"share": 0.57553, "mean": 0.41639, "std": 0.21129}, "word_cont": {"share": 0.25348, "mean": 0.2782, "std": 0.14207}, "punct": {"share": 0.12267, "mean": 0.62969, "std": 0.18731}}, "alpha_p05": 0.12421, "alpha_p50": 0.36297, "alpha_p95": 0.81493}
{"step": 50510, "total_steps": 72479, "tokens": 26481786880, "loss": 3.1830860376358032, "l_mix": 1.9590806812047958, "l_a": 2.073737293481827, "l_c": 2.0062803626060486, "z_loss": 41.79786205291748, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20481501519680023, "tokens_per_second": 1323305.158616214, "eta_seconds": 8704.026427316667, "data_wait_seconds": 1.677870750427246e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 50520, "total_steps": 72479, "tokens": 26487029760, "loss": 3.0743818283081055, "l_mix": 1.8915025293827057, "l_a": 2.0032653510570526, "l_c": 1.9396654218435287, "z_loss": 42.26219606399536, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14877301454544067, "tokens_per_second": 1409779.4936835943, "eta_seconds": 8166.412012362481, "data_wait_seconds": 2.1338462829589844e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 50530, "total_steps": 72479, "tokens": 26492272640, "loss": 3.133044719696045, "l_mix": 1.9281658232212067, "l_a": 2.042009651660919, "l_c": 1.974253073334694, "z_loss": 41.560723304748535, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15305158495903015, "tokens_per_second": 1478066.5746354226, "eta_seconds": 7785.574418282508, "data_wait_seconds": 1.722574234008789e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 50540, "total_steps": 72479, "tokens": 26497515520, "loss": 4.4021249413490295, "l_mix": 2.728467643260956, "l_a": 2.81233873963356, "l_c": 2.7665190398693085, "z_loss": 39.306565284729004, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16031430661678314, "tokens_per_second": 1345994.930416042, "eta_seconds": 8545.614973783493, "data_wait_seconds": 2.8967857360839844e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 50550, "total_steps": 72479, "tokens": 26502758400, "loss": 4.307437360286713, "l_mix": 2.669682800769806, "l_a": 2.7541632652282715, "l_c": 2.705018162727356, "z_loss": 38.99037742614746, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13037367165088654, "tokens_per_second": 1378029.1681335096, "eta_seconds": 8343.15544102192, "data_wait_seconds": 1.7344951629638672e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 50560, "total_steps": 72479, "tokens": 26508001280, "loss": 4.3612529039382935, "l_mix": 2.7026029527187347, "l_a": 2.787736564874649, "l_c": 2.741096407175064, "z_loss": 39.47145891189575, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2586260735988617, "tokens_per_second": 1474565.6250214577, "eta_seconds": 7793.392492675782, "data_wait_seconds": 2.2590160369873047e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 50570, "total_steps": 72479, "tokens": 26513244160, "loss": 4.336904168128967, "l_mix": 2.687359780073166, "l_a": 2.77202370762825, "l_c": 2.726457327604294, "z_loss": 40.01551055908203, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16418813169002533, "tokens_per_second": 1366208.4225218599, "eta_seconds": 8407.667236304284, "data_wait_seconds": 1.9669532775878906e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 50580, "total_steps": 72479, "tokens": 26518487040, "loss": 4.380790531635284, "l_mix": 2.714546114206314, "l_a": 2.7989467084407806, "l_c": 2.7552011609077454, "z_loss": 40.88173723220825, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17202571034431458, "tokens_per_second": 1411730.875166305, "eta_seconds": 8132.841120052338, "data_wait_seconds": 2.384185791015625e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 50590, "total_steps": 72479, "tokens": 26523729920, "loss": 3.122130423784256, "l_mix": 1.9210135787725449, "l_a": 2.0343143194913864, "l_c": 1.9694082885980606, "z_loss": 42.92392110824585, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18302059173583984, "tokens_per_second": 1435044.3759789555, "eta_seconds": 7997.0628254413605, "data_wait_seconds": 2.4199485778808594e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 50600, "total_steps": 72479, "tokens": 26528972800, "loss": 4.676382839679718, "l_mix": 2.9016561210155487, "l_a": 2.975214511156082, "l_c": 2.9405410289764404, "z_loss": 37.15792512893677, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19619937241077423, "tokens_per_second": 1458924.3532515424, "eta_seconds": 7862.571576404572, "data_wait_seconds": 2.6494264602661133e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.17895579, "gate_a2b/L1_absmax": 0.71220845, "gate_a2b/L1_rms": 0.19031022, "gate_a2b/L3_absmean": 0.17289758, "gate_a2b/L3_absmax": 0.55322504, "gate_a2b/L3_rms": 0.17904198, "gate_a2b/L5_absmean": 0.26135373, "gate_a2b/L5_absmax": 0.73510665, "gate_a2b/L5_rms": 0.26738396, "gate_c2b/L1_absmean": 0.02333088, "gate_c2b/L1_absmax": 0.63066041, "gate_c2b/L1_rms": 0.06602331, "gate_c2b/L3_absmean": 0.18204013, "gate_c2b/L3_absmax": 0.41000348, "gate_c2b/L3_rms": 0.185873, "gate_c2b/L5_absmean": 0.51847053, "gate_c2b/L5_absmax": 0.73659372, "gate_c2b/L5_rms": 0.52920032, "gate_b2a/L7_absmean": 0.05024488, "gate_b2a/L7_absmax": 0.32203647, "gate_b2a/L7_rms": 0.05584518, "gate_b2a/L11_absmean": 0.08170836, "gate_b2a/L11_absmax": 0.60334283, "gate_b2a/L11_rms": 0.09083079, "gate_b2a/L14_absmean": 0.21184549, "gate_b2a/L14_absmax": 0.79679859, "gate_b2a/L14_rms": 0.22057451, "gate_b2c/L3_absmean": 0.03567116, "gate_b2c/L3_absmax": 0.56761533, "gate_b2c/L3_rms": 0.0467765, "gate_b2c/L5_absmean": 0.03811029, "gate_b2c/L5_absmax": 0.56728059, "gate_b2c/L5_rms": 0.04820085, "gate_b2c/L6_absmean": 0.05483139, "gate_b2c/L6_absmax": 0.35882092, "gate_b2c/L6_rms": 0.06122255}, "streams": {"cos_a2b/round0": 0.363385, "cos_c2b/round0": 0.296755, "cos_b2a/round0": 0.158525, "cos_b2c/round0": 0.138671, "rms_a/round0": 6.363341, "rms_b/round0": 11.392003, "rms_c/round0": 4.306262, "cos_a2b/round1": 0.416285, "cos_c2b/round1": 0.308308, "cos_b2a/round1": 0.253788, "cos_b2c/round1": 0.116525, "rms_a/round1": 12.255154, "rms_b/round1": 25.852028, "rms_c/round1": 9.064889, "cos_a2b/round2": 0.553444, "cos_c2b/round2": 0.520976, "cos_b2a/round2": 0.104674, "cos_b2c/round2": 0.185138, "rms_a/round2": 34.152695, "rms_b/round2": 69.05117, "rms_c/round2": 17.525072}, "alpha_mean": 0.43712697679472967, "alpha_hist": [0.00467, 0.05617, 0.16942, 0.20439, 0.20557, 0.18227, 0.11848, 0.0545, 0.00449, 5e-05], "alpha_by_type": {"special": {"share": 0.00093, "mean": 0.54822, "std": 0.14023}, "whitespace": {"share": 0.05396, "mean": 0.51734, "std": 0.11278}, "digit": {"share": 0.02667, "mean": 0.44092, "std": 0.12339}, "word_start": {"share": 0.57867, "mean": 0.45174, "std": 0.15403}, "word_cont": {"share": 0.2363, "mean": 0.32629, "std": 0.12008}, "punct": {"share": 0.10347, "mean": 0.5647, "std": 0.15422}}, "alpha_p05": 0.18447, "alpha_p50": 0.43182, "alpha_p95": 0.71503}
{"step": 50610, "total_steps": 72479, "tokens": 26534215680, "loss": 4.7717480063438416, "l_mix": 2.9607571363449097, "l_a": 3.0400466322898865, "l_c": 2.9965895414352417, "z_loss": 37.011085510253906, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12401464581489563, "tokens_per_second": 1383632.9813288285, "eta_seconds": 8286.629783129692, "data_wait_seconds": 1.7881393432617188e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 50620, "total_steps": 72479, "tokens": 26539458560, "loss": 4.754368424415588, "l_mix": 2.9498470723629, "l_a": 3.028381258249283, "l_c": 2.9866896271705627, "z_loss": 38.25608444213867, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15947484970092773, "tokens_per_second": 1373017.3754803792, "eta_seconds": 8346.880088090897, "data_wait_seconds": 1.704692840576172e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 50630, "total_steps": 72479, "tokens": 26544701440, "loss": 3.109476089477539, "l_mix": 1.9127900302410126, "l_a": 2.028741091489792, "l_c": 1.9602121710777283, "z_loss": 43.207555294036865, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1958398073911667, "tokens_per_second": 1394494.3962096535, "eta_seconds": 8214.567619013786, "data_wait_seconds": 1.659989356994629e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 50640, "total_steps": 72479, "tokens": 26549944320, "loss": 3.1312232613563538, "l_mix": 1.9274476319551468, "l_a": 2.0393776148557663, "l_c": 1.9732076525688171, "z_loss": 42.676053524017334, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2425728142261505, "tokens_per_second": 1354637.814737897, "eta_seconds": 8452.388902354242, "data_wait_seconds": 4.3898820877075195e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 50650, "total_steps": 72479, "tokens": 26555187200, "loss": 4.311827719211578, "l_mix": 2.672310620546341, "l_a": 2.7573404908180237, "l_c": 2.7077163755893707, "z_loss": 38.84091520309448, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13635310530662537, "tokens_per_second": 1444570.9443373745, "eta_seconds": 7922.54807343483, "data_wait_seconds": 1.65402889251709e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 50660, "total_steps": 72479, "tokens": 26560430080, "loss": 4.315160751342773, "l_mix": 2.673604667186737, "l_a": 2.76150444149971, "l_c": 2.7103488743305206, "z_loss": 39.974205017089844, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15526717901229858, "tokens_per_second": 1467599.0430688045, "eta_seconds": 7794.662940144539, "data_wait_seconds": 1.5735626220703125e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 50670, "total_steps": 72479, "tokens": 26565672960, "loss": 4.387004911899567, "l_mix": 2.719557136297226, "l_a": 2.80085951089859, "l_c": 2.757299453020096, "z_loss": 40.37395668029785, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1693364977836609, "tokens_per_second": 1358890.2029266935, "eta_seconds": 8414.364138746261, "data_wait_seconds": 1.6570091247558594e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 50680, "total_steps": 72479, "tokens": 26570915840, "loss": 4.42925089597702, "l_mix": 2.744086742401123, "l_a": 2.8340204656124115, "l_c": 2.7831932306289673, "z_loss": 39.14578294754028, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18342918157577515, "tokens_per_second": 1300279.456932409, "eta_seconds": 8789.613687324523, "data_wait_seconds": 1.9222497940063477e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 50690, "total_steps": 72479, "tokens": 26576158720, "loss": 4.425304114818573, "l_mix": 2.7431302070617676, "l_a": 2.8278000950813293, "l_c": 2.779446005821228, "z_loss": 39.60470199584961, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13419084250926971, "tokens_per_second": 1477054.0207497363, "eta_seconds": 7734.118773937225, "data_wait_seconds": 1.6868114471435547e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 50700, "total_steps": 72479, "tokens": 26581401600, "loss": 3.1223766803741455, "l_mix": 1.9213810861110687, "l_a": 2.0349283814430237, "l_c": 1.9683901816606522, "z_loss": 41.887216091156006, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1612139344215393, "tokens_per_second": 1414023.6453581322, "eta_seconds": 8075.16082880497, "data_wait_seconds": 2.199411392211914e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.1784219, "gate_a2b/L1_absmax": 0.71133798, "gate_a2b/L1_rms": 0.18964648, "gate_a2b/L3_absmean": 0.17280638, "gate_a2b/L3_absmax": 0.55302638, "gate_a2b/L3_rms": 0.17892018, "gate_a2b/L5_absmean": 0.26118198, "gate_a2b/L5_absmax": 0.73509187, "gate_a2b/L5_rms": 0.26718655, "gate_c2b/L1_absmean": 0.02336513, "gate_c2b/L1_absmax": 0.62979078, "gate_c2b/L1_rms": 0.06637213, "gate_c2b/L3_absmean": 0.18220995, "gate_c2b/L3_absmax": 0.41261956, "gate_c2b/L3_rms": 0.18608275, "gate_c2b/L5_absmean": 0.51786202, "gate_c2b/L5_absmax": 0.72714829, "gate_c2b/L5_rms": 0.52851033, "gate_b2a/L7_absmean": 0.05070666, "gate_b2a/L7_absmax": 0.32264268, "gate_b2a/L7_rms": 0.05621657, "gate_b2a/L11_absmean": 0.08194342, "gate_b2a/L11_absmax": 0.60379797, "gate_b2a/L11_rms": 0.09106626, "gate_b2a/L14_absmean": 0.21215177, "gate_b2a/L14_absmax": 0.79540402, "gate_b2a/L14_rms": 0.22083247, "gate_b2c/L3_absmean": 0.03570694, "gate_b2c/L3_absmax": 0.56592542, "gate_b2c/L3_rms": 0.04681153, "gate_b2c/L5_absmean": 0.03808885, "gate_b2c/L5_absmax": 0.5682472, "gate_b2c/L5_rms": 0.04822008, "gate_b2c/L6_absmean": 0.05509989, "gate_b2c/L6_absmax": 0.3579199, "gate_b2c/L6_rms": 0.06141339}, "streams": {"cos_a2b/round0": 0.357696, "cos_c2b/round0": 0.254473, "cos_b2a/round0": 0.137771, "cos_b2c/round0": 0.113656, "rms_a/round0": 6.648925, "rms_b/round0": 11.697047, "rms_c/round0": 4.45563, "cos_a2b/round1": 0.410429, "cos_c2b/round1": 0.300819, "cos_b2a/round1": 0.237996, "cos_b2c/round1": 0.119457, "rms_a/round1": 13.194055, "rms_b/round1": 27.698221, "rms_c/round1": 9.744326, "cos_a2b/round2": 0.520838, "cos_c2b/round2": 0.510604, "cos_b2a/round2": 0.088757, "cos_b2c/round2": 0.171779, "rms_a/round2": 35.365841, "rms_b/round2": 71.684975, "rms_c/round2": 18.80409}, "alpha_mean": 0.40689252633950446, "alpha_hist": [0.03455, 0.16107, 0.19963, 0.15126, 0.11807, 0.10808, 0.09682, 0.08401, 0.04387, 0.00264], "alpha_by_type": {"special": {"share": 0.00119, "mean": 0.61824, "std": 0.11622}, "whitespace": {"share": 0.03656, "mean": 0.59169, "std": 0.18041}, "digit": {"share": 0.01129, "mean": 0.40834, "std": 0.17756}, "word_start": {"share": 0.56625, "mean": 0.41444, "std": 0.2062}, "word_cont": {"share": 0.26601, "mean": 0.27047, "std": 0.14284}, "punct": {"share": 0.1187, "mean": 0.61743, "std": 0.19098}}, "alpha_p05": 0.11596, "alpha_p50": 0.36932, "alpha_p95": 0.79149}
{"step": 50710, "total_steps": 72479, "tokens": 26586644480, "loss": 2.945341259241104, "l_mix": 1.8231621384620667, "l_a": 1.8905668258666992, "l_c": 1.850029855966568, "z_loss": 48.353732109069824, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14815929532051086, "tokens_per_second": 1423191.29802605, "eta_seconds": 8019.4598490238195, "data_wait_seconds": 2.6106834411621094e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 50720, "total_steps": 72479, "tokens": 26591887360, "loss": 4.682569921016693, "l_mix": 2.904876708984375, "l_a": 2.9818322360515594, "l_c": 2.9438118040561676, "z_loss": 37.8407187461853, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12299404293298721, "tokens_per_second": 1424781.842259075, "eta_seconds": 8006.827609419823, "data_wait_seconds": 3.212690353393555e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 50730, "total_steps": 72479, "tokens": 26597130240, "loss": 4.753836810588837, "l_mix": 2.949086844921112, "l_a": 3.025966316461563, "l_c": 2.989866465330124, "z_loss": 38.22765827178955, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15155702829360962, "tokens_per_second": 1437289.8291754331, "eta_seconds": 7933.500592947006, "data_wait_seconds": 1.913309097290039e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 50740, "total_steps": 72479, "tokens": 26602373120, "loss": 4.809708058834076, "l_mix": 2.9847473800182343, "l_a": 3.062863826751709, "l_c": 3.0203381776809692, "z_loss": 37.70586681365967, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14499147236347198, "tokens_per_second": 1482563.3819832362, "eta_seconds": 7687.696169018745, "data_wait_seconds": 1.7315149307250977e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 50750, "total_steps": 72479, "tokens": 26607616000, "loss": 4.679140001535416, "l_mix": 2.903433322906494, "l_a": 2.977726936340332, "l_c": 2.941295176744461, "z_loss": 37.8971061706543, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14006704092025757, "tokens_per_second": 1379338.4285844574, "eta_seconds": 8259.21595158577, "data_wait_seconds": 2.1070241928100586e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 50760, "total_steps": 72479, "tokens": 26612858880, "loss": 4.7656930685043335, "l_mix": 2.9563397765159607, "l_a": 3.0379622876644135, "l_c": 2.993214964866638, "z_loss": 37.536723613739014, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2132406383752823, "tokens_per_second": 1425250.766332835, "eta_seconds": 7989.47900326252, "data_wait_seconds": 1.8864870071411133e-05, "mix": {"fineweb_hq": 0.50955, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 50770, "total_steps": 72479, "tokens": 26618101760, "loss": 4.342697083950043, "l_mix": 2.6917311549186707, "l_a": 2.7758976817131042, "l_c": 2.7273218035697937, "z_loss": 39.67180061340332, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15067942440509796, "tokens_per_second": 1403885.789354368, "eta_seconds": 8107.332005429268, "data_wait_seconds": 2.238154411315918e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 50780, "total_steps": 72479, "tokens": 26623344640, "loss": 4.454380929470062, "l_mix": 2.7607820332050323, "l_a": 2.843805342912674, "l_c": 2.8015241026878357, "z_loss": 38.99166488647461, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.21241600811481476, "tokens_per_second": 1463187.5523459958, "eta_seconds": 7775.165455555916, "data_wait_seconds": 2.4557113647460938e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 50790, "total_steps": 72479, "tokens": 26628587520, "loss": 4.373659729957581, "l_mix": 2.711131513118744, "l_a": 2.795450448989868, "l_c": 2.7463099658489227, "z_loss": 40.91985750198364, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.28597432374954224, "tokens_per_second": 1315890.7873371835, "eta_seconds": 8641.509266138077, "data_wait_seconds": 1.767277717590332e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 50800, "total_steps": 72479, "tokens": 26633830400, "loss": 3.1683942079544067, "l_mix": 1.9502342194318771, "l_a": 2.063416540622711, "l_c": 1.997116431593895, "z_loss": 41.23830842971802, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20487774908542633, "tokens_per_second": 1377178.7520612841, "eta_seconds": 8253.133106350899, "data_wait_seconds": 2.6464462280273438e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.17838283, "gate_a2b/L1_absmax": 0.71022189, "gate_a2b/L1_rms": 0.18954399, "gate_a2b/L3_absmean": 0.17257349, "gate_a2b/L3_absmax": 0.5528003, "gate_a2b/L3_rms": 0.17866507, "gate_a2b/L5_absmean": 0.26077974, "gate_a2b/L5_absmax": 0.73547518, "gate_a2b/L5_rms": 0.266835, "gate_c2b/L1_absmean": 0.02279241, "gate_c2b/L1_absmax": 0.62906641, "gate_c2b/L1_rms": 0.06594143, "gate_c2b/L3_absmean": 0.18182404, "gate_c2b/L3_absmax": 0.41501224, "gate_c2b/L3_rms": 0.18572649, "gate_c2b/L5_absmean": 0.51747096, "gate_c2b/L5_absmax": 0.72925621, "gate_c2b/L5_rms": 0.52816182, "gate_b2a/L7_absmean": 0.05068734, "gate_b2a/L7_absmax": 0.32214028, "gate_b2a/L7_rms": 0.05617186, "gate_b2a/L11_absmean": 0.08183283, "gate_b2a/L11_absmax": 0.60467631, "gate_b2a/L11_rms": 0.09094638, "gate_b2a/L14_absmean": 0.2118897, "gate_b2a/L14_absmax": 0.79694152, "gate_b2a/L14_rms": 0.22062717, "gate_b2c/L3_absmean": 0.03571822, "gate_b2c/L3_absmax": 0.56522769, "gate_b2c/L3_rms": 0.04673903, "gate_b2c/L5_absmean": 0.0380884, "gate_b2c/L5_absmax": 0.5664801, "gate_b2c/L5_rms": 0.04820143, "gate_b2c/L6_absmean": 0.05475001, "gate_b2c/L6_absmax": 0.35754034, "gate_b2c/L6_rms": 0.06113562}, "streams": {"cos_a2b/round0": 0.358489, "cos_c2b/round0": 0.255225, "cos_b2a/round0": 0.141256, "cos_b2c/round0": 0.113721, "rms_a/round0": 6.687104, "rms_b/round0": 11.66409, "rms_c/round0": 4.459615, "cos_a2b/round1": 0.413104, "cos_c2b/round1": 0.305089, "cos_b2a/round1": 0.238336, "cos_b2c/round1": 0.119016, "rms_a/round1": 13.236633, "rms_b/round1": 27.723019, "rms_c/round1": 9.689127, "cos_a2b/round2": 0.519603, "cos_c2b/round2": 0.512374, "cos_b2a/round2": 0.093144, "cos_b2c/round2": 0.170457, "rms_a/round2": 35.486034, "rms_b/round2": 72.276917, "rms_c/round2": 18.683887}, "alpha_mean": 0.41411255955048887, "alpha_hist": [0.03377, 0.16411, 0.1898, 0.14517, 0.11734, 0.10585, 0.10185, 0.09004, 0.04829, 0.00377], "alpha_by_type": {"special": {"share": 0.0011, "mean": 0.67434, "std": 0.0962}, "whitespace": {"share": 0.0313, "mean": 0.63272, "std": 0.1784}, "digit": {"share": 0.01106, "mean": 0.38449, "std": 0.18192}, "word_start": {"share": 0.58089, "mean": 0.42236, "std": 0.21263}, "word_cont": {"share": 0.25702, "mean": 0.27339, "std": 0.14295}, "punct": {"share": 0.11864, "mean": 0.62131, "std": 0.19103}}, "alpha_p05": 0.1294, "alpha_p50": 0.36932, "alpha_p95": 0.80716}
{"step": 50810, "total_steps": 72479, "tokens": 26639073280, "loss": 3.1699192225933075, "l_mix": 1.950087532401085, "l_a": 2.0645137429237366, "l_c": 2.0015919655561447, "z_loss": 41.328367710113525, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1746915876865387, "tokens_per_second": 1476441.2459152392, "eta_seconds": 7694.716402316093, "data_wait_seconds": 1.5735626220703125e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 50820, "total_steps": 72479, "tokens": 26644316160, "loss": 2.996529370546341, "l_mix": 1.8546229898929596, "l_a": 1.9240964353084564, "l_c": 1.8822579234838486, "z_loss": 48.60123872756958, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1462773233652115, "tokens_per_second": 1409698.6081838326, "eta_seconds": 8055.3060959815975, "data_wait_seconds": 2.3037195205688477e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 50830, "total_steps": 72479, "tokens": 26649559040, "loss": 4.3052361607551575, "l_mix": 2.667402923107147, "l_a": 2.752836912870407, "l_c": 2.7066069543361664, "z_loss": 40.479588985443115, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1319562941789627, "tokens_per_second": 1430593.810214579, "eta_seconds": 7933.985755395889, "data_wait_seconds": 3.314018249511719e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 50840, "total_steps": 72479, "tokens": 26654801920, "loss": 4.391104340553284, "l_mix": 2.7224140763282776, "l_a": 2.8052937388420105, "l_c": 2.7570067942142487, "z_loss": 38.480165004730225, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1466713398694992, "tokens_per_second": 1457113.02879971, "eta_seconds": 7785.990384936333, "data_wait_seconds": 1.5527009963989258e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 50850, "total_steps": 72479, "tokens": 26660044800, "loss": 4.450983643531799, "l_mix": 2.75803604722023, "l_a": 2.8450819849967957, "l_c": 2.798076331615448, "z_loss": 38.93922138214111, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.21356847882270813, "tokens_per_second": 1410875.8421550102, "eta_seconds": 8037.4366143226625, "data_wait_seconds": 1.9758939743041992e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 50860, "total_steps": 72479, "tokens": 26665287680, "loss": 4.517220079898834, "l_mix": 2.8002032041549683, "l_a": 2.8853501975536346, "l_c": 2.838039308786392, "z_loss": 40.10940742492676, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1699380725622177, "tokens_per_second": 1418632.9732439232, "eta_seconds": 7989.791923475265, "data_wait_seconds": 1.5914440155029297e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 50870, "total_steps": 72479, "tokens": 26670530560, "loss": 4.25832462310791, "l_mix": 2.6381210684776306, "l_a": 2.7258787751197815, "l_c": 2.6747993528842926, "z_loss": 39.907772064208984, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13607972860336304, "tokens_per_second": 1423567.9346442276, "eta_seconds": 7958.411478853226, "data_wait_seconds": 1.9490718841552734e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 50880, "total_steps": 72479, "tokens": 26675773440, "loss": 4.3863900899887085, "l_mix": 2.718356490135193, "l_a": 2.8045988976955414, "l_c": 2.7555131018161774, "z_loss": 40.23999738693237, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12939924001693726, "tokens_per_second": 1455320.3544167248, "eta_seconds": 7781.170982480049, "data_wait_seconds": 1.963973045349121e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 50890, "total_steps": 72479, "tokens": 26681016320, "loss": 4.433928668498993, "l_mix": 2.7475330233573914, "l_a": 2.834635317325592, "l_c": 2.786683142185211, "z_loss": 41.11387014389038, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16184663772583008, "tokens_per_second": 1391801.6695063594, "eta_seconds": 8132.519079399109, "data_wait_seconds": 2.339482307434082e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 50900, "total_steps": 72479, "tokens": 26686259200, "loss": 4.456915020942688, "l_mix": 2.7631017863750458, "l_a": 2.8460602164268494, "l_c": 2.799983710050583, "z_loss": 39.992310523986816, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19923986494541168, "tokens_per_second": 1425544.3009960554, "eta_seconds": 7936.344555616379, "data_wait_seconds": 1.6301870346069336e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.17862955, "gate_a2b/L1_absmax": 0.7110287, "gate_a2b/L1_rms": 0.18982106, "gate_a2b/L3_absmean": 0.17231491, "gate_a2b/L3_absmax": 0.55205196, "gate_a2b/L3_rms": 0.17846712, "gate_a2b/L5_absmean": 0.26028487, "gate_a2b/L5_absmax": 0.73636115, "gate_a2b/L5_rms": 0.26635137, "gate_c2b/L1_absmean": 0.0227358, "gate_c2b/L1_absmax": 0.62920606, "gate_c2b/L1_rms": 0.06600032, "gate_c2b/L3_absmean": 0.1815608, "gate_c2b/L3_absmax": 0.40618372, "gate_c2b/L3_rms": 0.18538679, "gate_c2b/L5_absmean": 0.51707679, "gate_c2b/L5_absmax": 0.72738063, "gate_c2b/L5_rms": 0.5276956, "gate_b2a/L7_absmean": 0.05048493, "gate_b2a/L7_absmax": 0.32274663, "gate_b2a/L7_rms": 0.05603201, "gate_b2a/L11_absmean": 0.08189349, "gate_b2a/L11_absmax": 0.60598695, "gate_b2a/L11_rms": 0.09106663, "gate_b2a/L14_absmean": 0.21200657, "gate_b2a/L14_absmax": 0.79797876, "gate_b2a/L14_rms": 0.22072107, "gate_b2c/L3_absmean": 0.0358162, "gate_b2c/L3_absmax": 0.5647282, "gate_b2c/L3_rms": 0.04682463, "gate_b2c/L5_absmean": 0.03831328, "gate_b2c/L5_absmax": 0.56790805, "gate_b2c/L5_rms": 0.04839744, "gate_b2c/L6_absmean": 0.05486028, "gate_b2c/L6_absmax": 0.35681579, "gate_b2c/L6_rms": 0.06123878}, "streams": {"cos_a2b/round0": 0.360939, "cos_c2b/round0": 0.289156, "cos_b2a/round0": 0.148106, "cos_b2c/round0": 0.134821, "rms_a/round0": 6.394656, "rms_b/round0": 11.446129, "rms_c/round0": 4.335581, "cos_a2b/round1": 0.406022, "cos_c2b/round1": 0.292358, "cos_b2a/round1": 0.243768, "cos_b2c/round1": 0.108551, "rms_a/round1": 12.396867, "rms_b/round1": 26.42234, "rms_c/round1": 9.074028, "cos_a2b/round2": 0.533295, "cos_c2b/round2": 0.524794, "cos_b2a/round2": 0.099282, "cos_b2c/round2": 0.181217, "rms_a/round2": 34.529308, "rms_b/round2": 71.100784, "rms_c/round2": 17.329931}, "alpha_mean": 0.4228645030078866, "alpha_hist": [0.0137, 0.07425, 0.19534, 0.21521, 0.16522, 0.14439, 0.11719, 0.06578, 0.00874, 0.00017], "alpha_by_type": {"special": {"share": 0.00073, "mean": 0.55723, "std": 0.1597}, "whitespace": {"share": 0.03055, "mean": 0.50894, "std": 0.13442}, "digit": {"share": 0.05765, "mean": 0.44339, "std": 0.12208}, "word_start": {"share": 0.51872, "mean": 0.44141, "std": 0.17192}, "word_cont": {"share": 0.27226, "mean": 0.3089, "std": 0.12394}, "punct": {"share": 0.12009, "mean": 0.56859, "std": 0.16408}}, "alpha_p05": 0.15766, "alpha_p50": 0.39164, "alpha_p95": 0.71978}
{"step": 50910, "total_steps": 72479, "tokens": 26691502080, "loss": 3.1455336809158325, "l_mix": 1.9354448169469833, "l_a": 2.051529958844185, "l_c": 1.9820993393659592, "z_loss": 42.32541084289551, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.22636941075325012, "tokens_per_second": 1422956.6458697615, "eta_seconds": 7947.092348051072, "data_wait_seconds": 2.276897430419922e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 50920, "total_steps": 72479, "tokens": 26696744960, "loss": 3.177261531352997, "l_mix": 1.9545844197273254, "l_a": 2.070968419313431, "l_c": 2.0046221017837524, "z_loss": 41.3041296005249, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17714716494083405, "tokens_per_second": 1345877.951491375, "eta_seconds": 8398.328377008438, "data_wait_seconds": 1.773238182067871e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 50930, "total_steps": 72479, "tokens": 26701987840, "loss": 3.03574138879776, "l_mix": 1.880109280347824, "l_a": 1.945228487253189, "l_c": 1.9068783223628998, "z_loss": 47.027414321899414, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13674606382846832, "tokens_per_second": 1443741.650023655, "eta_seconds": 7825.418149995804, "data_wait_seconds": 3.3855438232421875e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 50940, "total_steps": 72479, "tokens": 26707230720, "loss": 4.416640639305115, "l_mix": 2.7371810972690582, "l_a": 2.8213265240192413, "l_c": 2.7768715620040894, "z_loss": 40.66391611099243, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11087510734796524, "tokens_per_second": 1469140.070803919, "eta_seconds": 7686.5640359401705, "data_wait_seconds": 1.6361474990844727e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 50950, "total_steps": 72479, "tokens": 26712473600, "loss": 4.495814621448517, "l_mix": 2.787453681230545, "l_a": 2.8724466264247894, "l_c": 2.822089582681656, "z_loss": 39.43148946762085, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.25210312008857727, "tokens_per_second": 1124947.2475159597, "eta_seconds": 10033.711693525314, "data_wait_seconds": 4.1604042053222656e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 50960, "total_steps": 72479, "tokens": 26717716480, "loss": 4.386023581027985, "l_mix": 2.7181805670261383, "l_a": 2.8020637929439545, "l_c": 2.7574125826358795, "z_loss": 41.470438957214355, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19214047491550446, "tokens_per_second": 1426543.2423901632, "eta_seconds": 7908.735702323913, "data_wait_seconds": 2.0056962966918945e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 50970, "total_steps": 72479, "tokens": 26722959360, "loss": 4.317744970321655, "l_mix": 2.6743633449077606, "l_a": 2.762872725725174, "l_c": 2.7150661647319794, "z_loss": 39.98774480819702, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1723102182149887, "tokens_per_second": 1375818.9245939704, "eta_seconds": 8196.507832837104, "data_wait_seconds": 1.683831214904785e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 50980, "total_steps": 72479, "tokens": 26728202240, "loss": 4.4119380712509155, "l_mix": 2.7354145646095276, "l_a": 2.8165182769298553, "l_c": 2.771893233060837, "z_loss": 39.21881818771362, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17603780329227448, "tokens_per_second": 1358124.2985890233, "eta_seconds": 8299.437484264374, "data_wait_seconds": 2.3484230041503906e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 50990, "total_steps": 72479, "tokens": 26733445120, "loss": 4.6312108635902405, "l_mix": 2.872864842414856, "l_a": 2.9489101469516754, "l_c": 2.9122430086135864, "z_loss": 39.350321769714355, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1664736270904541, "tokens_per_second": 1437250.092892171, "eta_seconds": 7838.875702786446, "data_wait_seconds": 2.0951032638549805e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 51000, "total_steps": 72479, "tokens": 26738688000, "loss": 4.636889696121216, "l_mix": 2.8760456144809723, "l_a": 2.954644560813904, "l_c": 2.914835512638092, "z_loss": 40.16017961502075, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20301084220409393, "tokens_per_second": 1372382.511071465, "eta_seconds": 8205.570867562294, "data_wait_seconds": 1.8388032913208008e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.17809367, "gate_a2b/L1_absmax": 0.71371436, "gate_a2b/L1_rms": 0.18920307, "gate_a2b/L3_absmean": 0.17174615, "gate_a2b/L3_absmax": 0.55171746, "gate_a2b/L3_rms": 0.17791434, "gate_a2b/L5_absmean": 0.26002058, "gate_a2b/L5_absmax": 0.74002379, "gate_a2b/L5_rms": 0.26606449, "gate_c2b/L1_absmean": 0.02331313, "gate_c2b/L1_absmax": 0.63381952, "gate_c2b/L1_rms": 0.06652644, "gate_c2b/L3_absmean": 0.18147163, "gate_c2b/L3_absmax": 0.41006225, "gate_c2b/L3_rms": 0.18536018, "gate_c2b/L5_absmean": 0.51669353, "gate_c2b/L5_absmax": 0.72977966, "gate_c2b/L5_rms": 0.52731043, "gate_b2a/L7_absmean": 0.05087618, "gate_b2a/L7_absmax": 0.32177004, "gate_b2a/L7_rms": 0.05636968, "gate_b2a/L11_absmean": 0.08219057, "gate_b2a/L11_absmax": 0.60518175, "gate_b2a/L11_rms": 0.09127226, "gate_b2a/L14_absmean": 0.21191761, "gate_b2a/L14_absmax": 0.80435592, "gate_b2a/L14_rms": 0.22077923, "gate_b2c/L3_absmean": 0.0357186, "gate_b2c/L3_absmax": 0.56272471, "gate_b2c/L3_rms": 0.04664956, "gate_b2c/L5_absmean": 0.03804994, "gate_b2c/L5_absmax": 0.56925219, "gate_b2c/L5_rms": 0.048238, "gate_b2c/L6_absmean": 0.05491996, "gate_b2c/L6_absmax": 0.36199966, "gate_b2c/L6_rms": 0.0613462}, "streams": {"cos_a2b/round0": 0.359402, "cos_c2b/round0": 0.290576, "cos_b2a/round0": 0.155841, "cos_b2c/round0": 0.137844, "rms_a/round0": 6.42225, "rms_b/round0": 11.515613, "rms_c/round0": 4.338513, "cos_a2b/round1": 0.4137, "cos_c2b/round1": 0.306587, "cos_b2a/round1": 0.252453, "cos_b2c/round1": 0.115798, "rms_a/round1": 12.404075, "rms_b/round1": 26.088505, "rms_c/round1": 9.080256, "cos_a2b/round2": 0.555814, "cos_c2b/round2": 0.520613, "cos_b2a/round2": 0.103639, "cos_b2c/round2": 0.183179, "rms_a/round2": 34.684246, "rms_b/round2": 69.659531, "rms_c/round2": 17.538088}, "alpha_mean": 0.46648097349712714, "alpha_hist": [0.00381, 0.04778, 0.13731, 0.17937, 0.19286, 0.19946, 0.15024, 0.07866, 0.01025, 0.00026], "alpha_by_type": {"special": {"share": 0.00076, "mean": 0.61389, "std": 0.10989}, "whitespace": {"share": 0.04495, "mean": 0.57016, "std": 0.11389}, "digit": {"share": 0.03188, "mean": 0.43811, "std": 0.1343}, "word_start": {"share": 0.59277, "mean": 0.48093, "std": 0.15874}, "word_cont": {"share": 0.22841, "mean": 0.35149, "std": 0.13023}, "punct": {"share": 0.10123, "mean": 0.60313, "std": 0.14973}}, "alpha_p05": 0.20307, "alpha_p50": 0.47207, "alpha_p95": 0.74317}
{"step": 51010, "total_steps": 72479, "tokens": 26743930880, "loss": 4.805167019367218, "l_mix": 2.982116788625717, "l_a": 3.0571916103363037, "l_c": 3.0196423530578613, "z_loss": 37.28213691711426, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.10654596984386444, "tokens_per_second": 1461703.28411941, "eta_seconds": 7700.563578319549, "data_wait_seconds": 1.6242265701293945e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 51020, "total_steps": 72479, "tokens": 26749173760, "loss": 4.763778924942017, "l_mix": 2.9545750617980957, "l_a": 3.0364083349704742, "l_c": 2.994271159172058, "z_loss": 37.69339418411255, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13539168238639832, "tokens_per_second": 1393014.663146771, "eta_seconds": 8076.5095225811, "data_wait_seconds": 2.4527311325073242e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 51030, "total_steps": 72479, "tokens": 26754416640, "loss": 4.671487510204315, "l_mix": 2.897789478302002, "l_a": 2.9770919382572174, "l_c": 2.9352345168590546, "z_loss": 38.64560317993164, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17174872756004333, "tokens_per_second": 1398608.3615873826, "eta_seconds": 8040.459088373184, "data_wait_seconds": 1.6748905181884766e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 51040, "total_steps": 72479, "tokens": 26759659520, "loss": 3.182623118162155, "l_mix": 1.9584335684776306, "l_a": 2.0738853216171265, "l_c": 2.0067465603351593, "z_loss": 41.20325517654419, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17834874987602234, "tokens_per_second": 1449698.4708855632, "eta_seconds": 7753.481608581544, "data_wait_seconds": 1.58846378326416e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 51050, "total_steps": 72479, "tokens": 26764902400, "loss": 4.472588658332825, "l_mix": 2.7723312079906464, "l_a": 2.8573050796985626, "l_c": 2.8102197647094727, "z_loss": 39.636125564575195, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11625836044549942, "tokens_per_second": 1472413.237378626, "eta_seconds": 7630.308711433411, "data_wait_seconds": 1.5616416931152344e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 51060, "total_steps": 72479, "tokens": 26770145280, "loss": 4.4433106780052185, "l_mix": 2.7540451288223267, "l_a": 2.8411557972431183, "l_c": 2.7897289395332336, "z_loss": 40.800353050231934, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.28603464365005493, "tokens_per_second": 1472400.3222989289, "eta_seconds": 7626.814869523048, "data_wait_seconds": 1.7374753952026367e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 51070, "total_steps": 72479, "tokens": 26775388160, "loss": 4.470362544059753, "l_mix": 2.770587533712387, "l_a": 2.8552307784557343, "l_c": 2.810685932636261, "z_loss": 39.90932273864746, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13258494436740875, "tokens_per_second": 1182143.2646538117, "eta_seconds": 9495.02664153576, "data_wait_seconds": 1.6033649444580078e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 51080, "total_steps": 72479, "tokens": 26780631040, "loss": 4.234742105007172, "l_mix": 2.623928427696228, "l_a": 2.7106068432331085, "l_c": 2.6587716937065125, "z_loss": 39.56586694717407, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17619585990905762, "tokens_per_second": 1456883.2737139366, "eta_seconds": 7700.849556326865, "data_wait_seconds": 1.5914440155029297e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 51090, "total_steps": 72479, "tokens": 26785873920, "loss": 4.389610946178436, "l_mix": 2.719781756401062, "l_a": 2.806472361087799, "l_c": 2.759624481201172, "z_loss": 39.625178813934326, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14775824546813965, "tokens_per_second": 1430119.132120019, "eta_seconds": 7841.3020147323605, "data_wait_seconds": 2.5600194931030273e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 51100, "total_steps": 72479, "tokens": 26791116800, "loss": 2.865473747253418, "l_mix": 1.773003563284874, "l_a": 1.8408843874931335, "l_c": 1.8006826043128967, "z_loss": 48.6889386177063, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1530390828847885, "tokens_per_second": 1291091.3091051858, "eta_seconds": 8681.61149637699, "data_wait_seconds": 7.635354995727539e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.17704444, "gate_a2b/L1_absmax": 0.71309185, "gate_a2b/L1_rms": 0.18837778, "gate_a2b/L3_absmean": 0.1714872, "gate_a2b/L3_absmax": 0.55079257, "gate_a2b/L3_rms": 0.17762925, "gate_a2b/L5_absmean": 0.25907531, "gate_a2b/L5_absmax": 0.73852348, "gate_a2b/L5_rms": 0.26513192, "gate_c2b/L1_absmean": 0.0231808, "gate_c2b/L1_absmax": 0.63396293, "gate_c2b/L1_rms": 0.06644371, "gate_c2b/L3_absmean": 0.18158713, "gate_c2b/L3_absmax": 0.41034898, "gate_c2b/L3_rms": 0.18543795, "gate_c2b/L5_absmean": 0.51605058, "gate_c2b/L5_absmax": 0.72653425, "gate_c2b/L5_rms": 0.52678829, "gate_b2a/L7_absmean": 0.05066935, "gate_b2a/L7_absmax": 0.32264102, "gate_b2a/L7_rms": 0.05619727, "gate_b2a/L11_absmean": 0.08161207, "gate_b2a/L11_absmax": 0.60574698, "gate_b2a/L11_rms": 0.09080384, "gate_b2a/L14_absmean": 0.21158527, "gate_b2a/L14_absmax": 0.8037408, "gate_b2a/L14_rms": 0.22047544, "gate_b2c/L3_absmean": 0.0355259, "gate_b2c/L3_absmax": 0.56424063, "gate_b2c/L3_rms": 0.04653955, "gate_b2c/L5_absmean": 0.03785099, "gate_b2c/L5_absmax": 0.56774807, "gate_b2c/L5_rms": 0.04804005, "gate_b2c/L6_absmean": 0.05456012, "gate_b2c/L6_absmax": 0.36027005, "gate_b2c/L6_rms": 0.0610371}, "streams": {"cos_a2b/round0": 0.369494, "cos_c2b/round0": 0.286291, "cos_b2a/round0": 0.138506, "cos_b2c/round0": 0.132005, "rms_a/round0": 6.625703, "rms_b/round0": 11.220337, "rms_c/round0": 4.321811, "cos_a2b/round1": 0.371866, "cos_c2b/round1": 0.272905, "cos_b2a/round1": 0.220158, "cos_b2c/round1": 0.094546, "rms_a/round1": 12.499551, "rms_b/round1": 26.109177, "rms_c/round1": 9.112839, "cos_a2b/round2": 0.485422, "cos_c2b/round2": 0.550881, "cos_b2a/round2": 0.082611, "cos_b2c/round2": 0.164384, "rms_a/round2": 33.094383, "rms_b/round2": 71.603531, "rms_c/round2": 17.387054}, "alpha_mean": 0.41030519623306105, "alpha_hist": [0.04277, 0.07803, 0.15878, 0.2095, 0.2016, 0.15024, 0.10289, 0.04639, 0.00865, 0.00114], "alpha_by_type": {"special": {"share": 0.00064, "mean": 0.54236, "std": 0.16386}, "whitespace": {"share": 0.10005, "mean": 0.50036, "std": 0.14802}, "digit": {"share": 0.09827, "mean": 0.37456, "std": 0.15503}, "word_start": {"share": 0.30481, "mean": 0.41234, "std": 0.16906}, "word_cont": {"share": 0.2374, "mean": 0.2997, "std": 0.14006}, "punct": {"share": 0.25883, "mean": 0.48779, "std": 0.17639}}, "alpha_p05": 0.11398, "alpha_p50": 0.40357, "alpha_p95": 0.72076}
{"step": 51110, "total_steps": 72479, "tokens": 26796359680, "loss": 4.655424475669861, "l_mix": 2.8878441751003265, "l_a": 2.9658361077308655, "l_c": 2.9260977804660797, "z_loss": 38.9844913482666, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13899019360542297, "tokens_per_second": 1393851.800291736, "eta_seconds": 8037.805934357643, "data_wait_seconds": 2.3812055587768555e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 51120, "total_steps": 72479, "tokens": 26801602560, "loss": 4.8504496812820435, "l_mix": 3.0096917152404785, "l_a": 3.08586049079895, "l_c": 3.0499991476535797, "z_loss": 38.58940076828003, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1878272145986557, "tokens_per_second": 1373320.3190191237, "eta_seconds": 8154.155470442772, "data_wait_seconds": 2.1845102310180664e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 51130, "total_steps": 72479, "tokens": 26806845440, "loss": 4.7115156054496765, "l_mix": 2.924004077911377, "l_a": 2.9980523586273193, "l_c": 2.960319072008133, "z_loss": 37.61733531951904, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15251927077770233, "tokens_per_second": 1434032.936337567, "eta_seconds": 7805.277151155471, "data_wait_seconds": 3.501772880554199e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 51140, "total_steps": 72479, "tokens": 26812088320, "loss": 4.65595155954361, "l_mix": 2.888299137353897, "l_a": 2.965176433324814, "l_c": 2.926998108625412, "z_loss": 38.18624448776245, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17353980243206024, "tokens_per_second": 1424433.0735688906, "eta_seconds": 7854.199568653107, "data_wait_seconds": 2.014636993408203e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 51150, "total_steps": 72479, "tokens": 26817331200, "loss": 4.7010298371315, "l_mix": 2.9161019325256348, "l_a": 2.9958419501781464, "l_c": 2.9539173245429993, "z_loss": 37.65355825424194, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17892037332057953, "tokens_per_second": 1289119.6669404919, "eta_seconds": 8674.55445663929, "data_wait_seconds": 1.65402889251709e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 51160, "total_steps": 72479, "tokens": 26822574080, "loss": 4.337077081203461, "l_mix": 2.688220828771591, "l_a": 2.7725881040096283, "l_c": 2.7235989570617676, "z_loss": 39.62794542312622, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15062600374221802, "tokens_per_second": 1349580.4503475947, "eta_seconds": 8282.052299380302, "data_wait_seconds": 1.8537044525146484e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 51170, "total_steps": 72479, "tokens": 26827816960, "loss": 4.482867419719696, "l_mix": 2.7785093784332275, "l_a": 2.8638364374637604, "l_c": 2.8173567354679108, "z_loss": 39.402814865112305, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13726790249347687, "tokens_per_second": 1435608.9238648298, "eta_seconds": 7782.100547218323, "data_wait_seconds": 2.41696834564209e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 51180, "total_steps": 72479, "tokens": 26833059840, "loss": 4.41557252407074, "l_mix": 2.7369504570961, "l_a": 2.8220841884613037, "l_c": 2.773322343826294, "z_loss": 39.73086929321289, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13220205903053284, "tokens_per_second": 1315938.4283480898, "eta_seconds": 8485.81504380703, "data_wait_seconds": 1.8924474716186523e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 51190, "total_steps": 72479, "tokens": 26838302720, "loss": 4.319380223751068, "l_mix": 2.6761354804039, "l_a": 2.762761265039444, "l_c": 2.7147211134433746, "z_loss": 40.45168685913086, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13328923285007477, "tokens_per_second": 1394722.5850969981, "eta_seconds": 8002.714913535118, "data_wait_seconds": 1.6927719116210938e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 51200, "total_steps": 72479, "tokens": 26843545600, "loss": 4.44374418258667, "l_mix": 2.7543559670448303, "l_a": 2.8406211137771606, "l_c": 2.790672868490219, "z_loss": 40.144500732421875, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1857897937297821, "tokens_per_second": 1454537.6501349418, "eta_seconds": 7670.0141457080845, "data_wait_seconds": 1.4662742614746094e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.17660625, "gate_a2b/L1_absmax": 0.71029097, "gate_a2b/L1_rms": 0.18786041, "gate_a2b/L3_absmean": 0.17117904, "gate_a2b/L3_absmax": 0.54832417, "gate_a2b/L3_rms": 0.17735235, "gate_a2b/L5_absmean": 0.25907359, "gate_a2b/L5_absmax": 0.73967624, "gate_a2b/L5_rms": 0.26510859, "gate_c2b/L1_absmean": 0.02434425, "gate_c2b/L1_absmax": 0.62935728, "gate_c2b/L1_rms": 0.06629401, "gate_c2b/L3_absmean": 0.18167786, "gate_c2b/L3_absmax": 0.41018087, "gate_c2b/L3_rms": 0.18549134, "gate_c2b/L5_absmean": 0.51566482, "gate_c2b/L5_absmax": 0.73017126, "gate_c2b/L5_rms": 0.52630615, "gate_b2a/L7_absmean": 0.05066745, "gate_b2a/L7_absmax": 0.32191294, "gate_b2a/L7_rms": 0.05615931, "gate_b2a/L11_absmean": 0.0819506, "gate_b2a/L11_absmax": 0.60556906, "gate_b2a/L11_rms": 0.09110183, "gate_b2a/L14_absmean": 0.21162337, "gate_b2a/L14_absmax": 0.80223268, "gate_b2a/L14_rms": 0.22048835, "gate_b2c/L3_absmean": 0.03590686, "gate_b2c/L3_absmax": 0.56381279, "gate_b2c/L3_rms": 0.04678834, "gate_b2c/L5_absmean": 0.03782263, "gate_b2c/L5_absmax": 0.56510115, "gate_b2c/L5_rms": 0.04795848, "gate_b2c/L6_absmean": 0.0547962, "gate_b2c/L6_absmax": 0.35792172, "gate_b2c/L6_rms": 0.06120827}, "streams": {"cos_a2b/round0": 0.364787, "cos_c2b/round0": 0.293432, "cos_b2a/round0": 0.153126, "cos_b2c/round0": 0.141029, "rms_a/round0": 6.486773, "rms_b/round0": 11.474557, "rms_c/round0": 4.367511, "cos_a2b/round1": 0.413994, "cos_c2b/round1": 0.30169, "cos_b2a/round1": 0.254312, "cos_b2c/round1": 0.110415, "rms_a/round1": 12.449884, "rms_b/round1": 26.334829, "rms_c/round1": 9.069393, "cos_a2b/round2": 0.551961, "cos_c2b/round2": 0.512494, "cos_b2a/round2": 0.099547, "cos_b2c/round2": 0.180515, "rms_a/round2": 34.926029, "rms_b/round2": 70.475746, "rms_c/round2": 17.056578}, "alpha_mean": 0.44426673933615035, "alpha_hist": [0.00803, 0.06808, 0.17755, 0.1913, 0.16408, 0.16541, 0.13094, 0.08009, 0.01448, 5e-05], "alpha_by_type": {"special": {"share": 0.0007, "mean": 0.60786, "std": 0.11489}, "whitespace": {"share": 0.0239, "mean": 0.54825, "std": 0.12346}, "digit": {"share": 0.04413, "mean": 0.4764, "std": 0.11483}, "word_start": {"share": 0.57002, "mean": 0.46083, "std": 0.17081}, "word_cont": {"share": 0.24292, "mean": 0.31454, "std": 0.1255}, "punct": {"share": 0.11833, "mean": 0.59683, "std": 0.16418}}, "alpha_p05": 0.18447, "alpha_p50": 0.43446, "alpha_p95": 0.73048}
{"step": 51210, "total_steps": 72479, "tokens": 26848788480, "loss": 2.7992445528507233, "l_mix": 1.732643336057663, "l_a": 1.7964366525411606, "l_c": 1.7589005082845688, "z_loss": 49.94178056716919, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13061390817165375, "tokens_per_second": 1444815.7226035711, "eta_seconds": 7717.995656847954, "data_wait_seconds": 2.1964311599731445e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 51220, "total_steps": 72479, "tokens": 26854031360, "loss": 2.9040696024894714, "l_mix": 1.7979370653629303, "l_a": 1.862857162952423, "l_c": 1.8242511451244354, "z_loss": 46.76964807510376, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.21941392123699188, "tokens_per_second": 1431157.562241842, "eta_seconds": 7787.988468956947, "data_wait_seconds": 1.7940998077392578e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 51230, "total_steps": 72479, "tokens": 26859274240, "loss": 4.686551868915558, "l_mix": 2.908063054084778, "l_a": 2.9831821024417877, "l_c": 2.945113718509674, "z_loss": 38.27451467514038, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12012336403131485, "tokens_per_second": 1448602.0515861562, "eta_seconds": 7690.5839666605, "data_wait_seconds": 1.913309097290039e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 51240, "total_steps": 72479, "tokens": 26864517120, "loss": 4.633633464574814, "l_mix": 2.87473726272583, "l_a": 2.9509869515895844, "l_c": 2.9120001792907715, "z_loss": 38.53679943084717, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14931979775428772, "tokens_per_second": 1444629.6873529889, "eta_seconds": 7708.101895928383, "data_wait_seconds": 2.1219253540039062e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 51250, "total_steps": 72479, "tokens": 26869760000, "loss": 4.745571196079254, "l_mix": 2.9458512365818024, "l_a": 3.0154629349708557, "l_c": 2.9836032390594482, "z_loss": 37.5800142288208, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.22014476358890533, "tokens_per_second": 1415681.1491990478, "eta_seconds": 7862.017487692832, "data_wait_seconds": 2.232193946838379e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 51260, "total_steps": 72479, "tokens": 26875002880, "loss": 4.696536660194397, "l_mix": 2.9127652049064636, "l_a": 2.9933072924613953, "l_c": 2.952597200870514, "z_loss": 39.02222728729248, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15458813309669495, "tokens_per_second": 1412600.374859962, "eta_seconds": 7875.452442169189, "data_wait_seconds": 1.7404556274414062e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 51270, "total_steps": 72479, "tokens": 26880245760, "loss": 4.396981120109558, "l_mix": 2.7253819406032562, "l_a": 2.8109143376350403, "l_c": 2.7610825300216675, "z_loss": 39.35627746582031, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13036467134952545, "tokens_per_second": 1450966.0834834431, "eta_seconds": 7663.600354671478, "data_wait_seconds": 1.8596649169921875e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 51280, "total_steps": 72479, "tokens": 26885488640, "loss": 4.361750364303589, "l_mix": 2.703822076320648, "l_a": 2.7870684564113617, "l_c": 2.7393589317798615, "z_loss": 38.92793607711792, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12294172495603561, "tokens_per_second": 1394227.739812085, "eta_seconds": 7971.711503529548, "data_wait_seconds": 2.256035804748535e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 51290, "total_steps": 72479, "tokens": 26890731520, "loss": 4.4182000160217285, "l_mix": 2.738455206155777, "l_a": 2.824884980916977, "l_c": 2.7742641270160675, "z_loss": 39.18722438812256, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1172507181763649, "tokens_per_second": 1382909.554050659, "eta_seconds": 8033.163412213325, "data_wait_seconds": 1.7374753952026367e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 51300, "total_steps": 72479, "tokens": 26895974400, "loss": 4.393250942230225, "l_mix": 2.7225215137004852, "l_a": 2.808781087398529, "l_c": 2.760316550731659, "z_loss": 39.533244609832764, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.10399482399225235, "tokens_per_second": 1393987.8715792587, "eta_seconds": 7965.561091589928, "data_wait_seconds": 1.621246337890625e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.17670839, "gate_a2b/L1_absmax": 0.70903677, "gate_a2b/L1_rms": 0.18788499, "gate_a2b/L3_absmean": 0.17105027, "gate_a2b/L3_absmax": 0.54758304, "gate_a2b/L3_rms": 0.17715736, "gate_a2b/L5_absmean": 0.25827807, "gate_a2b/L5_absmax": 0.73818439, "gate_a2b/L5_rms": 0.26430449, "gate_c2b/L1_absmean": 0.02370587, "gate_c2b/L1_absmax": 0.62675041, "gate_c2b/L1_rms": 0.06589057, "gate_c2b/L3_absmean": 0.18121958, "gate_c2b/L3_absmax": 0.41096419, "gate_c2b/L3_rms": 0.18507628, "gate_c2b/L5_absmean": 0.51522744, "gate_c2b/L5_absmax": 0.73318219, "gate_c2b/L5_rms": 0.52591723, "gate_b2a/L7_absmean": 0.05082902, "gate_b2a/L7_absmax": 0.32133576, "gate_b2a/L7_rms": 0.05632203, "gate_b2a/L11_absmean": 0.08171703, "gate_b2a/L11_absmax": 0.60364795, "gate_b2a/L11_rms": 0.09084441, "gate_b2a/L14_absmean": 0.21149909, "gate_b2a/L14_absmax": 0.80218256, "gate_b2a/L14_rms": 0.22034724, "gate_b2c/L3_absmean": 0.0354022, "gate_b2c/L3_absmax": 0.56310302, "gate_b2c/L3_rms": 0.04644507, "gate_b2c/L5_absmean": 0.03817631, "gate_b2c/L5_absmax": 0.56784272, "gate_b2c/L5_rms": 0.04827086, "gate_b2c/L6_absmean": 0.05462014, "gate_b2c/L6_absmax": 0.35729593, "gate_b2c/L6_rms": 0.06102364}, "streams": {"cos_a2b/round0": 0.362558, "cos_c2b/round0": 0.291453, "cos_b2a/round0": 0.149113, "cos_b2c/round0": 0.136176, "rms_a/round0": 6.507534, "rms_b/round0": 11.557784, "rms_c/round0": 4.410303, "cos_a2b/round1": 0.409201, "cos_c2b/round1": 0.298088, "cos_b2a/round1": 0.247692, "cos_b2c/round1": 0.109226, "rms_a/round1": 12.623584, "rms_b/round1": 26.776228, "rms_c/round1": 9.232531, "cos_a2b/round2": 0.534005, "cos_c2b/round2": 0.524856, "cos_b2a/round2": 0.098139, "cos_b2c/round2": 0.177001, "rms_a/round2": 35.006588, "rms_b/round2": 71.698143, "rms_c/round2": 17.813053}, "alpha_mean": 0.4280676368717451, "alpha_hist": [0.00851, 0.0672, 0.18604, 0.22118, 0.18159, 0.14748, 0.11241, 0.06528, 0.01007, 0.00024], "alpha_by_type": {"special": {"share": 0.00121, "mean": 0.54488, "std": 0.11343}, "whitespace": {"share": 0.02795, "mean": 0.53157, "std": 0.13124}, "digit": {"share": 0.04944, "mean": 0.47874, "std": 0.1209}, "word_start": {"share": 0.54147, "mean": 0.4383, "std": 0.16637}, "word_cont": {"share": 0.26862, "mean": 0.32492, "std": 0.12478}, "punct": {"share": 0.11131, "mean": 0.57744, "std": 0.16368}}, "alpha_p05": 0.18243, "alpha_p50": 0.41822, "alpha_p95": 0.73602}
{"step": 51310, "total_steps": 72479, "tokens": 26901217280, "loss": 4.428084313869476, "l_mix": 2.7450831830501556, "l_a": 2.830289512872696, "l_c": 2.7797144055366516, "z_loss": 38.69846963882446, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1207706555724144, "tokens_per_second": 1437551.4107479595, "eta_seconds": 7720.525741910934, "data_wait_seconds": 2.0802021026611328e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 51320, "total_steps": 72479, "tokens": 26906460160, "loss": 4.430424630641937, "l_mix": 2.7459044754505157, "l_a": 2.8331316113471985, "l_c": 2.781935393810272, "z_loss": 39.34390163421631, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13074466586112976, "tokens_per_second": 1361813.4429147032, "eta_seconds": 8146.056899142266, "data_wait_seconds": 4.06801700592041e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 51330, "total_steps": 72479, "tokens": 26911703040, "loss": 2.8657190799713135, "l_mix": 1.7743526250123978, "l_a": 1.8380153626203537, "l_c": 1.7998725473880768, "z_loss": 47.10059881210327, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1746576875448227, "tokens_per_second": 1435013.7535719164, "eta_seconds": 7726.871526074409, "data_wait_seconds": 3.7729740142822266e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 51340, "total_steps": 72479, "tokens": 26916945920, "loss": 2.940992623567581, "l_mix": 1.8198124915361404, "l_a": 1.8891292959451675, "l_c": 1.8481376469135284, "z_loss": 50.29172325134277, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19778716564178467, "tokens_per_second": 1352783.1361167808, "eta_seconds": 8192.683465743065, "data_wait_seconds": 3.167986869812012e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 51350, "total_steps": 72479, "tokens": 26922188800, "loss": 4.398024916648865, "l_mix": 2.725579261779785, "l_a": 2.812644898891449, "l_c": 2.7621740102767944, "z_loss": 38.89228057861328, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12310555577278137, "tokens_per_second": 1359163.4215424568, "eta_seconds": 8150.367333626747, "data_wait_seconds": 2.6017427444458008e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 51360, "total_steps": 72479, "tokens": 26927431680, "loss": 4.360259115695953, "l_mix": 2.7028132379055023, "l_a": 2.7869967222213745, "l_c": 2.7378225922584534, "z_loss": 39.31608772277832, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1486966907978058, "tokens_per_second": 1444809.3624443682, "eta_seconds": 7663.598091077804, "data_wait_seconds": 3.2961368560791016e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 51370, "total_steps": 72479, "tokens": 26932674560, "loss": 4.29252415895462, "l_mix": 2.660228282213211, "l_a": 2.744349926710129, "l_c": 2.696636348962784, "z_loss": 39.88159942626953, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12744198739528656, "tokens_per_second": 1347588.3605091048, "eta_seconds": 8212.593486499787, "data_wait_seconds": 1.8894672393798828e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 51380, "total_steps": 72479, "tokens": 26937917440, "loss": 4.357136487960815, "l_mix": 2.7002339363098145, "l_a": 2.7858462929725647, "l_c": 2.7371619641780853, "z_loss": 39.30742645263672, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16062642633914948, "tokens_per_second": 1436531.5554447628, "eta_seconds": 7700.459116315842, "data_wait_seconds": 1.7136335372924805e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 51390, "total_steps": 72479, "tokens": 26943160320, "loss": 4.3818541169166565, "l_mix": 2.716456800699234, "l_a": 2.798468917608261, "l_c": 2.7528553903102875, "z_loss": 39.2090859413147, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11339376121759415, "tokens_per_second": 1402024.3101380141, "eta_seconds": 7886.246730566025, "data_wait_seconds": 1.7136335372924805e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 51400, "total_steps": 72479, "tokens": 26948403200, "loss": 4.380976438522339, "l_mix": 2.7156824469566345, "l_a": 2.7991358637809753, "l_c": 2.7518440186977386, "z_loss": 39.022517681121826, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.10426752269268036, "tokens_per_second": 1428377.0077461514, "eta_seconds": 7737.079700994492, "data_wait_seconds": 1.6242265701293945e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.17667185, "gate_a2b/L1_absmax": 0.70979321, "gate_a2b/L1_rms": 0.18779527, "gate_a2b/L3_absmean": 0.17131259, "gate_a2b/L3_absmax": 0.54758281, "gate_a2b/L3_rms": 0.17737582, "gate_a2b/L5_absmean": 0.25786063, "gate_a2b/L5_absmax": 0.7380541, "gate_a2b/L5_rms": 0.26391324, "gate_c2b/L1_absmean": 0.02363111, "gate_c2b/L1_absmax": 0.62566191, "gate_c2b/L1_rms": 0.06589247, "gate_c2b/L3_absmean": 0.18138848, "gate_c2b/L3_absmax": 0.41450974, "gate_c2b/L3_rms": 0.18525046, "gate_c2b/L5_absmean": 0.51562291, "gate_c2b/L5_absmax": 0.72768104, "gate_c2b/L5_rms": 0.52630419, "gate_b2a/L7_absmean": 0.05077605, "gate_b2a/L7_absmax": 0.32081717, "gate_b2a/L7_rms": 0.05624263, "gate_b2a/L11_absmean": 0.0819518, "gate_b2a/L11_absmax": 0.6046592, "gate_b2a/L11_rms": 0.09110133, "gate_b2a/L14_absmean": 0.21154702, "gate_b2a/L14_absmax": 0.80261952, "gate_b2a/L14_rms": 0.22039486, "gate_b2c/L3_absmean": 0.0356866, "gate_b2c/L3_absmax": 0.5636524, "gate_b2c/L3_rms": 0.04667213, "gate_b2c/L5_absmean": 0.03787492, "gate_b2c/L5_absmax": 0.5677954, "gate_b2c/L5_rms": 0.04801911, "gate_b2c/L6_absmean": 0.05460181, "gate_b2c/L6_absmax": 0.3576659, "gate_b2c/L6_rms": 0.06100381}, "streams": {"cos_a2b/round0": 0.360702, "cos_c2b/round0": 0.289267, "cos_b2a/round0": 0.151023, "cos_b2c/round0": 0.137288, "rms_a/round0": 6.542374, "rms_b/round0": 11.618639, "rms_c/round0": 4.386436, "cos_a2b/round1": 0.41181, "cos_c2b/round1": 0.299602, "cos_b2a/round1": 0.249118, "cos_b2c/round1": 0.109983, "rms_a/round1": 12.73622, "rms_b/round1": 26.875925, "rms_c/round1": 9.232865, "cos_a2b/round2": 0.539023, "cos_c2b/round2": 0.522663, "cos_b2a/round2": 0.095551, "cos_b2c/round2": 0.179451, "rms_a/round2": 35.290249, "rms_b/round2": 71.785904, "rms_c/round2": 17.76568}, "alpha_mean": 0.42439711974864736, "alpha_hist": [0.01218, 0.07321, 0.18628, 0.21408, 0.18326, 0.14421, 0.1144, 0.06322, 0.00905, 0.00012], "alpha_by_type": {"special": {"share": 0.00108, "mean": 0.57129, "std": 0.12262}, "whitespace": {"share": 0.02974, "mean": 0.52754, "std": 0.12577}, "digit": {"share": 0.04468, "mean": 0.44267, "std": 0.12902}, "word_start": {"share": 0.55035, "mean": 0.43743, "std": 0.16372}, "word_cont": {"share": 0.26141, "mean": 0.3132, "std": 0.12841}, "punct": {"share": 0.11273, "mean": 0.58277, "std": 0.16135}}, "alpha_p05": 0.17301, "alpha_p50": 0.40663, "alpha_p95": 0.72798}
{"step": 51410, "total_steps": 72479, "tokens": 26953646080, "loss": 4.410223007202148, "l_mix": 2.7331273555755615, "l_a": 2.8197088539600372, "l_c": 2.7706097960472107, "z_loss": 39.29211235046387, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13364072144031525, "tokens_per_second": 1415399.8610695777, "eta_seconds": 7804.313237428666, "data_wait_seconds": 1.5914440155029297e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 51420, "total_steps": 72479, "tokens": 26958888960, "loss": 4.527661919593811, "l_mix": 2.8066026270389557, "l_a": 2.8927853107452393, "l_c": 2.844078630208969, "z_loss": 40.44740438461304, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19975680112838745, "tokens_per_second": 1446785.6011103617, "eta_seconds": 7631.3871132850645, "data_wait_seconds": 2.2411346435546875e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 51430, "total_steps": 72479, "tokens": 26964131840, "loss": 4.394727110862732, "l_mix": 2.723041594028473, "l_a": 2.8090759813785553, "l_c": 2.763208717107773, "z_loss": 38.855239391326904, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.22113607823848724, "tokens_per_second": 1487231.4553038245, "eta_seconds": 7420.323227190971, "data_wait_seconds": 2.002716064453125e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 51440, "total_steps": 72479, "tokens": 26969374720, "loss": 2.997149705886841, "l_mix": 1.8553226739168167, "l_a": 1.9239144772291183, "l_c": 1.8821755349636078, "z_loss": 48.35341119766235, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1342577040195465, "tokens_per_second": 1438260.715486261, "eta_seconds": 7669.329429101944, "data_wait_seconds": 2.6494264602661133e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 51450, "total_steps": 72479, "tokens": 26974617600, "loss": 2.648389458656311, "l_mix": 1.6394141167402267, "l_a": 1.7007084041833878, "l_c": 1.6625425517559052, "z_loss": 48.577237129211426, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14565005898475647, "tokens_per_second": 1447746.391980926, "eta_seconds": 7615.4583517313, "data_wait_seconds": 1.710653305053711e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 51460, "total_steps": 72479, "tokens": 26979860480, "loss": 4.357992887496948, "l_mix": 2.7013796865940094, "l_a": 2.783362478017807, "l_c": 2.7386810779571533, "z_loss": 38.49775409698486, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.194276362657547, "tokens_per_second": 1410887.157329834, "eta_seconds": 7810.695146489144, "data_wait_seconds": 2.09808349609375e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 51470, "total_steps": 72479, "tokens": 26985103360, "loss": 4.310305535793304, "l_mix": 2.671319752931595, "l_a": 2.756238728761673, "l_c": 2.7070468962192535, "z_loss": 38.18107891082764, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20724201202392578, "tokens_per_second": 1209004.1185710966, "eta_seconds": 9110.611306285859, "data_wait_seconds": 2.3931264877319336e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 51480, "total_steps": 72479, "tokens": 26990346240, "loss": 4.327291488647461, "l_mix": 2.6821583211421967, "l_a": 2.7665383219718933, "l_c": 2.7172387838363647, "z_loss": 39.946410179138184, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1336739957332611, "tokens_per_second": 1404999.3780448518, "eta_seconds": 7835.963406133652, "data_wait_seconds": 2.5957822799682617e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 51490, "total_steps": 72479, "tokens": 26995589120, "loss": 4.354510486125946, "l_mix": 2.697944939136505, "l_a": 2.7869808673858643, "l_c": 2.7349043488502502, "z_loss": 41.302165031433105, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2098739892244339, "tokens_per_second": 1476277.601401207, "eta_seconds": 7454.072880029677, "data_wait_seconds": 1.6957521438598633e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 51500, "total_steps": 72479, "tokens": 27000832000, "loss": 4.334222853183746, "l_mix": 2.6861831843852997, "l_a": 2.7724708914756775, "l_c": 2.7209945023059845, "z_loss": 40.27470874786377, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16885511577129364, "tokens_per_second": 1429146.7567758323, "eta_seconds": 7696.227066850662, "data_wait_seconds": 2.0802021026611328e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.17625797, "gate_a2b/L1_absmax": 0.7093066, "gate_a2b/L1_rms": 0.1873564, "gate_a2b/L3_absmean": 0.17059012, "gate_a2b/L3_absmax": 0.54563183, "gate_a2b/L3_rms": 0.17671099, "gate_a2b/L5_absmean": 0.2573061, "gate_a2b/L5_absmax": 0.73938602, "gate_a2b/L5_rms": 0.26335481, "gate_c2b/L1_absmean": 0.02342832, "gate_c2b/L1_absmax": 0.62835145, "gate_c2b/L1_rms": 0.06598713, "gate_c2b/L3_absmean": 0.18103625, "gate_c2b/L3_absmax": 0.40853536, "gate_c2b/L3_rms": 0.18489541, "gate_c2b/L5_absmean": 0.51566303, "gate_c2b/L5_absmax": 0.72713935, "gate_c2b/L5_rms": 0.52624422, "gate_b2a/L7_absmean": 0.05088559, "gate_b2a/L7_absmax": 0.32249999, "gate_b2a/L7_rms": 0.05635493, "gate_b2a/L11_absmean": 0.08160339, "gate_b2a/L11_absmax": 0.60620749, "gate_b2a/L11_rms": 0.09081979, "gate_b2a/L14_absmean": 0.21174753, "gate_b2a/L14_absmax": 0.8044768, "gate_b2a/L14_rms": 0.22060019, "gate_b2c/L3_absmean": 0.03544389, "gate_b2c/L3_absmax": 0.56242859, "gate_b2c/L3_rms": 0.04643247, "gate_b2c/L5_absmean": 0.03789547, "gate_b2c/L5_absmax": 0.56840318, "gate_b2c/L5_rms": 0.04806099, "gate_b2c/L6_absmean": 0.05445887, "gate_b2c/L6_absmax": 0.35778853, "gate_b2c/L6_rms": 0.060894}, "streams": {"cos_a2b/round0": 0.363089, "cos_c2b/round0": 0.289632, "cos_b2a/round0": 0.144395, "cos_b2c/round0": 0.136551, "rms_a/round0": 6.541981, "rms_b/round0": 11.446312, "rms_c/round0": 4.361067, "cos_a2b/round1": 0.40133, "cos_c2b/round1": 0.293762, "cos_b2a/round1": 0.244463, "cos_b2c/round1": 0.109712, "rms_a/round1": 12.509783, "rms_b/round1": 26.493143, "rms_c/round1": 9.101833, "cos_a2b/round2": 0.533986, "cos_c2b/round2": 0.524889, "cos_b2a/round2": 0.095425, "cos_b2c/round2": 0.180554, "rms_a/round2": 35.22015, "rms_b/round2": 72.229218, "rms_c/round2": 17.357525}, "alpha_mean": 0.4223984152419291, "alpha_hist": [0.01012, 0.07602, 0.18799, 0.21211, 0.18253, 0.15292, 0.11217, 0.05847, 0.00735, 0.00032], "alpha_by_type": {"special": {"share": 0.00069, "mean": 0.59456, "std": 0.10855}, "whitespace": {"share": 0.03314, "mean": 0.5224, "std": 0.12696}, "digit": {"share": 0.09483, "mean": 0.44702, "std": 0.12651}, "word_start": {"share": 0.48286, "mean": 0.43594, "std": 0.16676}, "word_cont": {"share": 0.25713, "mean": 0.30602, "std": 0.12172}, "punct": {"share": 0.13135, "mean": 0.55654, "std": 0.15967}}, "alpha_p05": 0.19012, "alpha_p50": 0.40521, "alpha_p95": 0.71642}
{"step": 51510, "total_steps": 72479, "tokens": 27006074880, "loss": 4.643208563327789, "l_mix": 2.881639540195465, "l_a": 2.953270733356476, "l_c": 2.9186257421970367, "z_loss": 38.33622741699219, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15391327440738678, "tokens_per_second": 1446727.2537124215, "eta_seconds": 7599.079262375831, "data_wait_seconds": 2.7298927307128906e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 51520, "total_steps": 72479, "tokens": 27011317760, "loss": 4.550664007663727, "l_mix": 2.823322743177414, "l_a": 2.8984784185886383, "l_c": 2.8593256771564484, "z_loss": 38.6839280128479, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13957080245018005, "tokens_per_second": 1449369.0174133354, "eta_seconds": 7581.6110734939575, "data_wait_seconds": 2.002716064453125e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 51530, "total_steps": 72479, "tokens": 27016560640, "loss": 4.68645054101944, "l_mix": 2.90825155377388, "l_a": 2.9805164635181427, "l_c": 2.946813404560089, "z_loss": 39.44040536880493, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20594754815101624, "tokens_per_second": 1377477.8400373638, "eta_seconds": 7973.492562103272, "data_wait_seconds": 2.059340476989746e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 51540, "total_steps": 72479, "tokens": 27021803520, "loss": 4.680805861949921, "l_mix": 2.902923196554184, "l_a": 2.979688882827759, "l_c": 2.9465865194797516, "z_loss": 39.72578001022339, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19791728258132935, "tokens_per_second": 1371470.3600586152, "eta_seconds": 8004.5961996078495, "data_wait_seconds": 1.722574234008789e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 51550, "total_steps": 72479, "tokens": 27027046400, "loss": 4.605793237686157, "l_mix": 2.857988327741623, "l_a": 2.933601200580597, "l_c": 2.8924147486686707, "z_loss": 38.96724033355713, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13779370486736298, "tokens_per_second": 1417308.1957586776, "eta_seconds": 7742.016581034661, "data_wait_seconds": 2.6792287826538086e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 51560, "total_steps": 72479, "tokens": 27032289280, "loss": 2.918287843465805, "l_mix": 1.8073879927396774, "l_a": 1.8721413910388947, "l_c": 1.83085797727108, "z_loss": 47.578418254852295, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19036723673343658, "tokens_per_second": 1425839.6207086265, "eta_seconds": 7692.0156465768805, "data_wait_seconds": 1.6033649444580078e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 51570, "total_steps": 72479, "tokens": 27037532160, "loss": 4.406726539134979, "l_mix": 2.7308495938777924, "l_a": 2.81940034031868, "l_c": 2.7668558061122894, "z_loss": 39.34834384918213, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13824217021465302, "tokens_per_second": 1260171.3812386706, "eta_seconds": 8699.084866714478, "data_wait_seconds": 2.294778823852539e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 51580, "total_steps": 72479, "tokens": 27042775040, "loss": 4.2803069949150085, "l_mix": 2.652433305978775, "l_a": 2.7385683059692383, "l_c": 2.687676876783371, "z_loss": 40.14419412612915, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12730403244495392, "tokens_per_second": 1332142.710140867, "eta_seconds": 8225.165989041328, "data_wait_seconds": 2.4199485778808594e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 51590, "total_steps": 72479, "tokens": 27048017920, "loss": 4.356117904186249, "l_mix": 2.699043869972229, "l_a": 2.789016216993332, "l_c": 2.7345633804798126, "z_loss": 39.39087772369385, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14991092681884766, "tokens_per_second": 1420276.631020619, "eta_seconds": 7711.069655585289, "data_wait_seconds": 1.683831214904785e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 51600, "total_steps": 72479, "tokens": 27053260800, "loss": 4.432995140552521, "l_mix": 2.747839391231537, "l_a": 2.8350390195846558, "l_c": 2.7821463644504547, "z_loss": 39.03952169418335, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14146575331687927, "tokens_per_second": 1411836.104929837, "eta_seconds": 7753.4560235261915, "data_wait_seconds": 1.704692840576172e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.17626363, "gate_a2b/L1_absmax": 0.70787376, "gate_a2b/L1_rms": 0.18732372, "gate_a2b/L3_absmean": 0.17053933, "gate_a2b/L3_absmax": 0.54678208, "gate_a2b/L3_rms": 0.17659441, "gate_a2b/L5_absmean": 0.25728986, "gate_a2b/L5_absmax": 0.73795235, "gate_a2b/L5_rms": 0.26331168, "gate_c2b/L1_absmean": 0.02383044, "gate_c2b/L1_absmax": 0.6263476, "gate_c2b/L1_rms": 0.06600066, "gate_c2b/L3_absmean": 0.18097457, "gate_c2b/L3_absmax": 0.40523025, "gate_c2b/L3_rms": 0.18478511, "gate_c2b/L5_absmean": 0.51481003, "gate_c2b/L5_absmax": 0.73421723, "gate_c2b/L5_rms": 0.52549541, "gate_b2a/L7_absmean": 0.05035983, "gate_b2a/L7_absmax": 0.32065299, "gate_b2a/L7_rms": 0.05586572, "gate_b2a/L11_absmean": 0.08190432, "gate_b2a/L11_absmax": 0.60464346, "gate_b2a/L11_rms": 0.091075, "gate_b2a/L14_absmean": 0.21161793, "gate_b2a/L14_absmax": 0.80291408, "gate_b2a/L14_rms": 0.22047751, "gate_b2c/L3_absmean": 0.03579458, "gate_b2c/L3_absmax": 0.56189138, "gate_b2c/L3_rms": 0.04670908, "gate_b2c/L5_absmean": 0.03776954, "gate_b2c/L5_absmax": 0.56697303, "gate_b2c/L5_rms": 0.04793164, "gate_b2c/L6_absmean": 0.05447298, "gate_b2c/L6_absmax": 0.35725054, "gate_b2c/L6_rms": 0.06083896}, "streams": {"cos_a2b/round0": 0.3611, "cos_c2b/round0": 0.289063, "cos_b2a/round0": 0.154368, "cos_b2c/round0": 0.13745, "rms_a/round0": 6.554914, "rms_b/round0": 11.572081, "rms_c/round0": 4.386595, "cos_a2b/round1": 0.411863, "cos_c2b/round1": 0.300116, "cos_b2a/round1": 0.248864, "cos_b2c/round1": 0.112936, "rms_a/round1": 12.735697, "rms_b/round1": 26.730228, "rms_c/round1": 9.189777, "cos_a2b/round2": 0.543215, "cos_c2b/round2": 0.521154, "cos_b2a/round2": 0.098515, "cos_b2c/round2": 0.18206, "rms_a/round2": 35.529236, "rms_b/round2": 71.843597, "rms_c/round2": 17.827564}, "alpha_mean": 0.43187362243619987, "alpha_hist": [0.00995, 0.07224, 0.18784, 0.19949, 0.17812, 0.14571, 0.12477, 0.07066, 0.0112, 3e-05], "alpha_by_type": {"special": {"share": 0.00102, "mean": 0.58035, "std": 0.10598}, "whitespace": {"share": 0.02797, "mean": 0.5182, "std": 0.12023}, "digit": {"share": 0.05083, "mean": 0.41285, "std": 0.10782}, "word_start": {"share": 0.56055, "mean": 0.45261, "std": 0.17174}, "word_cont": {"share": 0.24472, "mean": 0.31141, "std": 0.12859}, "punct": {"share": 0.11491, "mean": 0.57333, "std": 0.16225}}, "alpha_p05": 0.16969, "alpha_p50": 0.41253, "alpha_p95": 0.71622}
{"step": 51610, "total_steps": 72479, "tokens": 27058503680, "loss": 4.377769470214844, "l_mix": 2.7140497863292694, "l_a": 2.7952453196048737, "l_c": 2.7504864633083344, "z_loss": 38.63670325279236, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14966851472854614, "tokens_per_second": 1370532.8552652835, "eta_seconds": 7983.293672943115, "data_wait_seconds": 1.8298625946044922e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 51620, "total_steps": 72479, "tokens": 27063746560, "loss": 4.607810974121094, "l_mix": 2.8583017885684967, "l_a": 2.9346407651901245, "l_c": 2.897056221961975, "z_loss": 39.0910758972168, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15650835633277893, "tokens_per_second": 1439914.74508311, "eta_seconds": 7594.979792618751, "data_wait_seconds": 3.0159950256347656e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 51630, "total_steps": 72479, "tokens": 27068989440, "loss": 4.662572026252747, "l_mix": 2.892282783985138, "l_a": 2.970828056335449, "l_c": 2.9301359355449677, "z_loss": 39.0840744972229, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.23933546245098114, "tokens_per_second": 1383267.953626102, "eta_seconds": 7902.21481192112, "data_wait_seconds": 2.0205974578857422e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 51640, "total_steps": 72479, "tokens": 27074232320, "loss": 4.819587528705597, "l_mix": 2.9908412992954254, "l_a": 3.0641340911388397, "l_c": 3.0316863358020782, "z_loss": 36.90844392776489, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.23356188833713531, "tokens_per_second": 1271513.7419473093, "eta_seconds": 8592.622534513474, "data_wait_seconds": 2.396106719970703e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 51650, "total_steps": 72479, "tokens": 27079475200, "loss": 4.676722764968872, "l_mix": 2.9015347063541412, "l_a": 2.980121999979019, "l_c": 2.937171459197998, "z_loss": 39.39704656600952, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19796709716320038, "tokens_per_second": 1422368.6978394324, "eta_seconds": 7677.611837625504, "data_wait_seconds": 2.6702880859375e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 51660, "total_steps": 72479, "tokens": 27084718080, "loss": 4.639098346233368, "l_mix": 2.8778611719608307, "l_a": 2.953655779361725, "l_c": 2.917134314775467, "z_loss": 39.35685634613037, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17426183819770813, "tokens_per_second": 1343544.9339555977, "eta_seconds": 8124.143522214889, "data_wait_seconds": 2.2917985916137695e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 51670, "total_steps": 72479, "tokens": 27089960960, "loss": 2.969849020242691, "l_mix": 1.839751973748207, "l_a": 1.901351973414421, "l_c": 1.8656380474567413, "z_loss": 46.20427751541138, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.21694065630435944, "tokens_per_second": 1261069.0067853283, "eta_seconds": 8651.317995524407, "data_wait_seconds": 1.8268823623657227e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 51680, "total_steps": 72479, "tokens": 27095203840, "loss": 4.354742586612701, "l_mix": 2.698432594537735, "l_a": 2.7866721153259277, "l_c": 2.734361231327057, "z_loss": 40.05902910232544, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13495029509067535, "tokens_per_second": 1467181.0323134032, "eta_seconds": 7432.393052959442, "data_wait_seconds": 1.6570091247558594e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 51690, "total_steps": 72479, "tokens": 27100446720, "loss": 4.396810591220856, "l_mix": 2.724892795085907, "l_a": 2.8135147094726562, "l_c": 2.7595444917678833, "z_loss": 39.55701208114624, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12292654067277908, "tokens_per_second": 1428457.7311887944, "eta_seconds": 7630.203536319732, "data_wait_seconds": 1.6808509826660156e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 51700, "total_steps": 72479, "tokens": 27105689600, "loss": 4.351610004901886, "l_mix": 2.697440028190613, "l_a": 2.7797687649726868, "l_c": 2.734130620956421, "z_loss": 39.47810459136963, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11086249351501465, "tokens_per_second": 1325673.2787978374, "eta_seconds": 8217.847131896018, "data_wait_seconds": 1.96993350982666e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.17631543, "gate_a2b/L1_absmax": 0.70823783, "gate_a2b/L1_rms": 0.18747173, "gate_a2b/L3_absmean": 0.17006803, "gate_a2b/L3_absmax": 0.54487383, "gate_a2b/L3_rms": 0.17618027, "gate_a2b/L5_absmean": 0.25638983, "gate_a2b/L5_absmax": 0.73979712, "gate_a2b/L5_rms": 0.26246667, "gate_c2b/L1_absmean": 0.02443904, "gate_c2b/L1_absmax": 0.62931782, "gate_c2b/L1_rms": 0.06612405, "gate_c2b/L3_absmean": 0.18064614, "gate_c2b/L3_absmax": 0.40872589, "gate_c2b/L3_rms": 0.18450366, "gate_c2b/L5_absmean": 0.51363862, "gate_c2b/L5_absmax": 0.73029327, "gate_c2b/L5_rms": 0.52421522, "gate_b2a/L7_absmean": 0.05050105, "gate_b2a/L7_absmax": 0.32240954, "gate_b2a/L7_rms": 0.05598392, "gate_b2a/L11_absmean": 0.08200583, "gate_b2a/L11_absmax": 0.60474819, "gate_b2a/L11_rms": 0.09111439, "gate_b2a/L14_absmean": 0.21111484, "gate_b2a/L14_absmax": 0.80570424, "gate_b2a/L14_rms": 0.2200041, "gate_b2c/L3_absmean": 0.03547054, "gate_b2c/L3_absmax": 0.56344622, "gate_b2c/L3_rms": 0.04656511, "gate_b2c/L5_absmean": 0.0376596, "gate_b2c/L5_absmax": 0.56333888, "gate_b2c/L5_rms": 0.04781369, "gate_b2c/L6_absmean": 0.05416469, "gate_b2c/L6_absmax": 0.35601047, "gate_b2c/L6_rms": 0.06061957}, "streams": {"cos_a2b/round0": 0.363227, "cos_c2b/round0": 0.294264, "cos_b2a/round0": 0.161947, "cos_b2c/round0": 0.139346, "rms_a/round0": 6.551356, "rms_b/round0": 11.51158, "rms_c/round0": 4.39792, "cos_a2b/round1": 0.417734, "cos_c2b/round1": 0.3036, "cos_b2a/round1": 0.253843, "cos_b2c/round1": 0.115828, "rms_a/round1": 12.657366, "rms_b/round1": 26.55081, "rms_c/round1": 9.153322, "cos_a2b/round2": 0.545201, "cos_c2b/round2": 0.52469, "cos_b2a/round2": 0.101416, "cos_b2c/round2": 0.182658, "rms_a/round2": 35.22393, "rms_b/round2": 71.600433, "rms_c/round2": 17.693195}, "alpha_mean": 0.43722062798131844, "alpha_hist": [0.0076, 0.06821, 0.17848, 0.19751, 0.18028, 0.16142, 0.12726, 0.06981, 0.00929, 0.00014], "alpha_by_type": {"special": {"share": 0.00113, "mean": 0.55789, "std": 0.10905}, "whitespace": {"share": 0.02315, "mean": 0.54628, "std": 0.11972}, "digit": {"share": 0.02567, "mean": 0.45494, "std": 0.121}, "word_start": {"share": 0.60474, "mean": 0.45266, "std": 0.16694}, "word_cont": {"share": 0.23901, "mean": 0.32257, "std": 0.12666}, "punct": {"share": 0.10631, "mean": 0.57789, "std": 0.16348}}, "alpha_p05": 0.17867, "alpha_p50": 0.42298, "alpha_p95": 0.74651}
{"step": 51710, "total_steps": 72479, "tokens": 27110932480, "loss": 4.595912873744965, "l_mix": 2.8497683703899384, "l_a": 2.934437096118927, "l_c": 2.886044353246689, "z_loss": 38.58480882644653, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.141763374209404, "tokens_per_second": 1253893.2177536173, "eta_seconds": 8684.10269536972, "data_wait_seconds": 1.6361474990844727e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 51720, "total_steps": 72479, "tokens": 27116175360, "loss": 4.412296116352081, "l_mix": 2.7345095574855804, "l_a": 2.822269469499588, "l_c": 2.7703519761562347, "z_loss": 39.06856107711792, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12995074689388275, "tokens_per_second": 1367281.8907213216, "eta_seconds": 7960.095621728897, "data_wait_seconds": 1.8388032913208008e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 51730, "total_steps": 72479, "tokens": 27121418240, "loss": 2.9354555308818817, "l_mix": 1.8163058459758759, "l_a": 1.8868629187345505, "l_c": 1.8436359018087387, "z_loss": 48.28088569641113, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17180196940898895, "tokens_per_second": 1439132.2258951664, "eta_seconds": 7559.0355884313585, "data_wait_seconds": 2.86102294921875e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 51740, "total_steps": 72479, "tokens": 27126661120, "loss": 4.7044870257377625, "l_mix": 2.919064164161682, "l_a": 2.9954479038715363, "l_c": 2.955961227416992, "z_loss": 38.59223413467407, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12014325708150864, "tokens_per_second": 1470362.679976774, "eta_seconds": 7394.9162203788765, "data_wait_seconds": 1.52587890625e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 51750, "total_steps": 72479, "tokens": 27131904000, "loss": 4.647992193698883, "l_mix": 2.884171098470688, "l_a": 2.959655225276947, "l_c": 2.919748157262802, "z_loss": 38.504310607910156, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14410950243473053, "tokens_per_second": 1300122.6297412885, "eta_seconds": 8359.185282516479, "data_wait_seconds": 2.193450927734375e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 51760, "total_steps": 72479, "tokens": 27137146880, "loss": 4.6954386830329895, "l_mix": 2.912689298391342, "l_a": 2.98969966173172, "l_c": 2.9527982473373413, "z_loss": 38.761398792266846, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18260015547275543, "tokens_per_second": 1409237.9637848258, "eta_seconds": 7708.224835801125, "data_wait_seconds": 3.1113624572753906e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 51770, "total_steps": 72479, "tokens": 27142389760, "loss": 4.742949545383453, "l_mix": 2.943143606185913, "l_a": 3.019623726606369, "l_c": 2.9797289669513702, "z_loss": 38.03755331039429, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15889108180999756, "tokens_per_second": 1480461.2524179264, "eta_seconds": 7333.849619007111, "data_wait_seconds": 2.2619962692260742e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 51780, "total_steps": 72479, "tokens": 27147632640, "loss": 4.679374814033508, "l_mix": 2.9023416936397552, "l_a": 2.980820506811142, "l_c": 2.9426232874393463, "z_loss": 39.359853744506836, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18598678708076477, "tokens_per_second": 1456177.9537598528, "eta_seconds": 7452.54883441925, "data_wait_seconds": 2.2739171981811523e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 51790, "total_steps": 72479, "tokens": 27152875520, "loss": 4.382286965847015, "l_mix": 2.7161173224449158, "l_a": 2.804227113723755, "l_c": 2.749671757221222, "z_loss": 39.05784845352173, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12845133244991302, "tokens_per_second": 1393905.0769885122, "eta_seconds": 7781.7310597896585, "data_wait_seconds": 1.9818544387817383e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 51800, "total_steps": 72479, "tokens": 27158118400, "loss": 4.389321982860565, "l_mix": 2.720156103372574, "l_a": 2.8059673607349396, "l_c": 2.7579187154769897, "z_loss": 40.48362112045288, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11131331324577332, "tokens_per_second": 1478766.6124471426, "eta_seconds": 7331.617755460739, "data_wait_seconds": 1.5079975128173828e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.17617933, "gate_a2b/L1_absmax": 0.70811725, "gate_a2b/L1_rms": 0.18720651, "gate_a2b/L3_absmean": 0.16988553, "gate_a2b/L3_absmax": 0.54677612, "gate_a2b/L3_rms": 0.1759645, "gate_a2b/L5_absmean": 0.25610718, "gate_a2b/L5_absmax": 0.74201256, "gate_a2b/L5_rms": 0.26212302, "gate_c2b/L1_absmean": 0.0243435, "gate_c2b/L1_absmax": 0.63078499, "gate_c2b/L1_rms": 0.06624682, "gate_c2b/L3_absmean": 0.1812319, "gate_c2b/L3_absmax": 0.40361071, "gate_c2b/L3_rms": 0.18505229, "gate_c2b/L5_absmean": 0.51377994, "gate_c2b/L5_absmax": 0.73540574, "gate_c2b/L5_rms": 0.52440608, "gate_b2a/L7_absmean": 0.05032814, "gate_b2a/L7_absmax": 0.32183746, "gate_b2a/L7_rms": 0.05589939, "gate_b2a/L11_absmean": 0.08193419, "gate_b2a/L11_absmax": 0.60732138, "gate_b2a/L11_rms": 0.09113561, "gate_b2a/L14_absmean": 0.21160719, "gate_b2a/L14_absmax": 0.80946648, "gate_b2a/L14_rms": 0.22054981, "gate_b2c/L3_absmean": 0.03534295, "gate_b2c/L3_absmax": 0.56184059, "gate_b2c/L3_rms": 0.046349, "gate_b2c/L5_absmean": 0.03786158, "gate_b2c/L5_absmax": 0.56238973, "gate_b2c/L5_rms": 0.04784992, "gate_b2c/L6_absmean": 0.05442529, "gate_b2c/L6_absmax": 0.35631436, "gate_b2c/L6_rms": 0.06079479}, "streams": {"cos_a2b/round0": 0.361715, "cos_c2b/round0": 0.289537, "cos_b2a/round0": 0.149676, "cos_b2c/round0": 0.139403, "rms_a/round0": 6.573757, "rms_b/round0": 11.698973, "rms_c/round0": 4.404728, "cos_a2b/round1": 0.412152, "cos_c2b/round1": 0.301406, "cos_b2a/round1": 0.249696, "cos_b2c/round1": 0.112099, "rms_a/round1": 12.714145, "rms_b/round1": 26.931562, "rms_c/round1": 9.212168, "cos_a2b/round2": 0.541469, "cos_c2b/round2": 0.519056, "cos_b2a/round2": 0.103519, "cos_b2c/round2": 0.182346, "rms_a/round2": 35.531506, "rms_b/round2": 71.681084, "rms_c/round2": 17.81126}, "alpha_mean": 0.4331418706974546, "alpha_hist": [0.01079, 0.07362, 0.16966, 0.19618, 0.19107, 0.16707, 0.11874, 0.06416, 0.00867, 3e-05], "alpha_by_type": {"special": {"share": 0.00107, "mean": 0.55618, "std": 0.10338}, "whitespace": {"share": 0.03047, "mean": 0.51012, "std": 0.11606}, "digit": {"share": 0.0475, "mean": 0.46046, "std": 0.08972}, "word_start": {"share": 0.56076, "mean": 0.45019, "std": 0.16708}, "word_cont": {"share": 0.25113, "mean": 0.31675, "std": 0.12982}, "punct": {"share": 0.10907, "mean": 0.57887, "std": 0.15817}}, "alpha_p05": 0.17134, "alpha_p50": 0.44927, "alpha_p95": 0.72447}
{"step": 51810, "total_steps": 72479, "tokens": 27163361280, "loss": 4.346581876277924, "l_mix": 2.693568915128708, "l_a": 2.781848669052124, "l_c": 2.728194296360016, "z_loss": 39.875903606414795, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19502268731594086, "tokens_per_second": 1442520.499155357, "eta_seconds": 7512.204282951356, "data_wait_seconds": 1.5348196029663086e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 51820, "total_steps": 72479, "tokens": 27168604160, "loss": 4.354639768600464, "l_mix": 2.698216289281845, "l_a": 2.7841637134552, "l_c": 2.7372476160526276, "z_loss": 40.18739175796509, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20429781079292297, "tokens_per_second": 1442950.6115526168, "eta_seconds": 7506.331613349915, "data_wait_seconds": 2.2977590560913086e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 51830, "total_steps": 72479, "tokens": 27173847040, "loss": 4.361134350299835, "l_mix": 2.702625036239624, "l_a": 2.789395719766617, "l_c": 2.738968789577484, "z_loss": 39.707595348358154, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11622365564107895, "tokens_per_second": 1403568.5852069776, "eta_seconds": 7713.2126111269, "data_wait_seconds": 1.6808509826660156e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 51840, "total_steps": 72479, "tokens": 27179089920, "loss": 2.97496098279953, "l_mix": 1.8414093554019928, "l_a": 1.9099093526601791, "l_c": 1.868595838546753, "z_loss": 48.33206129074097, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1199192926287651, "tokens_per_second": 1395490.396127241, "eta_seconds": 7754.105697917938, "data_wait_seconds": 3.3408403396606445e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 51850, "total_steps": 72479, "tokens": 27184332800, "loss": 2.9679945707321167, "l_mix": 1.8374140858650208, "l_a": 1.903973549604416, "l_c": 1.8646278232336044, "z_loss": 48.24436712265015, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13904795050621033, "tokens_per_second": 1308474.4007646313, "eta_seconds": 8265.761367344856, "data_wait_seconds": 2.9534101486206055e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 51860, "total_steps": 72479, "tokens": 27189575680, "loss": 4.353305637836456, "l_mix": 2.6978891491889954, "l_a": 2.782706171274185, "l_c": 2.7353487610816956, "z_loss": 40.20016527175903, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11825552582740784, "tokens_per_second": 1450104.1871707956, "eta_seconds": 7454.839705753326, "data_wait_seconds": 2.2530555725097656e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 51870, "total_steps": 72479, "tokens": 27194818560, "loss": 4.442223310470581, "l_mix": 2.7537398636341095, "l_a": 2.838180363178253, "l_c": 2.7900978326797485, "z_loss": 39.76734972000122, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1246967762708664, "tokens_per_second": 1440768.1549131083, "eta_seconds": 7499.507367062569, "data_wait_seconds": 1.901388168334961e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 51880, "total_steps": 72479, "tokens": 27200061440, "loss": 4.347578585147858, "l_mix": 2.6946957111358643, "l_a": 2.7785570323467255, "l_c": 2.7310521602630615, "z_loss": 39.9948525428772, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1288934051990509, "tokens_per_second": 1412771.7167198448, "eta_seconds": 7644.411608886719, "data_wait_seconds": 2.4646520614624023e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 51890, "total_steps": 72479, "tokens": 27205304320, "loss": 4.395513474941254, "l_mix": 2.723738819360733, "l_a": 2.811971962451935, "l_c": 2.7606100142002106, "z_loss": 40.35967445373535, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15589168667793274, "tokens_per_second": 1458517.3631717581, "eta_seconds": 7401.053908967972, "data_wait_seconds": 1.9311904907226562e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 51900, "total_steps": 72479, "tokens": 27210547200, "loss": 4.38604348897934, "l_mix": 2.7190284729003906, "l_a": 2.803234815597534, "l_c": 2.7534817457199097, "z_loss": 38.99871253967285, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16554632782936096, "tokens_per_second": 1397329.4930066909, "eta_seconds": 7721.387694168091, "data_wait_seconds": 1.6868114471435547e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.1761273, "gate_a2b/L1_absmax": 0.70699459, "gate_a2b/L1_rms": 0.18707314, "gate_a2b/L3_absmean": 0.16980055, "gate_a2b/L3_absmax": 0.54568851, "gate_a2b/L3_rms": 0.17590795, "gate_a2b/L5_absmean": 0.25556955, "gate_a2b/L5_absmax": 0.74086583, "gate_a2b/L5_rms": 0.2616564, "gate_c2b/L1_absmean": 0.02474417, "gate_c2b/L1_absmax": 0.62941718, "gate_c2b/L1_rms": 0.06640123, "gate_c2b/L3_absmean": 0.1804177, "gate_c2b/L3_absmax": 0.4050301, "gate_c2b/L3_rms": 0.18425532, "gate_c2b/L5_absmean": 0.51329976, "gate_c2b/L5_absmax": 0.72724146, "gate_c2b/L5_rms": 0.52384168, "gate_b2a/L7_absmean": 0.05036753, "gate_b2a/L7_absmax": 0.32196355, "gate_b2a/L7_rms": 0.05596476, "gate_b2a/L11_absmean": 0.0817125, "gate_b2a/L11_absmax": 0.60533959, "gate_b2a/L11_rms": 0.09091078, "gate_b2a/L14_absmean": 0.21149638, "gate_b2a/L14_absmax": 0.80903071, "gate_b2a/L14_rms": 0.22044486, "gate_b2c/L3_absmean": 0.03542382, "gate_b2c/L3_absmax": 0.56180334, "gate_b2c/L3_rms": 0.04639061, "gate_b2c/L5_absmean": 0.03802757, "gate_b2c/L5_absmax": 0.56192505, "gate_b2c/L5_rms": 0.04795607, "gate_b2c/L6_absmean": 0.05431942, "gate_b2c/L6_absmax": 0.35618541, "gate_b2c/L6_rms": 0.0607014}, "streams": {"cos_a2b/round0": 0.359727, "cos_c2b/round0": 0.287871, "cos_b2a/round0": 0.159073, "cos_b2c/round0": 0.137473, "rms_a/round0": 6.598424, "rms_b/round0": 11.641701, "rms_c/round0": 4.457733, "cos_a2b/round1": 0.414963, "cos_c2b/round1": 0.303229, "cos_b2a/round1": 0.256949, "cos_b2c/round1": 0.114149, "rms_a/round1": 12.758126, "rms_b/round1": 27.005018, "rms_c/round1": 9.316725, "cos_a2b/round2": 0.534262, "cos_c2b/round2": 0.531648, "cos_b2a/round2": 0.09761, "cos_b2c/round2": 0.18336, "rms_a/round2": 35.627583, "rms_b/round2": 73.014313, "rms_c/round2": 18.152201}, "alpha_mean": 0.41407274250292403, "alpha_hist": [0.00775, 0.08221, 0.20564, 0.21803, 0.18048, 0.13931, 0.10016, 0.05937, 0.00699, 5e-05], "alpha_by_type": {"special": {"share": 0.00131, "mean": 0.55249, "std": 0.10821}, "whitespace": {"share": 0.02979, "mean": 0.5119, "std": 0.12169}, "digit": {"share": 0.03041, "mean": 0.45678, "std": 0.12679}, "word_start": {"share": 0.55682, "mean": 0.42344, "std": 0.16593}, "word_cont": {"share": 0.27211, "mean": 0.31731, "std": 0.12077}, "punct": {"share": 0.10956, "mean": 0.56671, "std": 0.16244}}, "alpha_p05": 0.16291, "alpha_p50": 0.40239, "alpha_p95": 0.71642}
{"step": 51910, "total_steps": 72479, "tokens": 27215790080, "loss": 4.446255564689636, "l_mix": 2.7559778690338135, "l_a": 2.8406157791614532, "l_c": 2.7936431765556335, "z_loss": 39.16316795349121, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13415715098381042, "tokens_per_second": 1407709.2201238964, "eta_seconds": 7660.729728722572, "data_wait_seconds": 1.659989356994629e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 51920, "total_steps": 72479, "tokens": 27221032960, "loss": 4.38455867767334, "l_mix": 2.7172136902809143, "l_a": 2.8050346672534943, "l_c": 2.752781867980957, "z_loss": 39.99173307418823, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13870884478092194, "tokens_per_second": 1247670.9993614194, "eta_seconds": 8639.166092276573, "data_wait_seconds": 2.250075340270996e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 51930, "total_steps": 72479, "tokens": 27226275840, "loss": 4.26173210144043, "l_mix": 2.6403237879276276, "l_a": 2.728925794363022, "l_c": 2.6757681369781494, "z_loss": 39.26770639419556, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19246640801429749, "tokens_per_second": 1319615.5905934428, "eta_seconds": 8164.191290855407, "data_wait_seconds": 1.6182661056518555e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 51940, "total_steps": 72479, "tokens": 27231518720, "loss": 4.436702191829681, "l_mix": 2.749758720397949, "l_a": 2.837716579437256, "l_c": 2.78542822599411, "z_loss": 39.583558082580566, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18351882696151733, "tokens_per_second": 1429216.8848795847, "eta_seconds": 7534.441655373574, "data_wait_seconds": 2.2679567337036133e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 51950, "total_steps": 72479, "tokens": 27236761600, "loss": 2.7492038309574127, "l_mix": 1.7016917318105698, "l_a": 1.765169769525528, "l_c": 1.7265370786190033, "z_loss": 51.00799036026001, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19950246810913086, "tokens_per_second": 1467797.409756778, "eta_seconds": 7332.82964015007, "data_wait_seconds": 1.6957521438598633e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 51960, "total_steps": 72479, "tokens": 27242004480, "loss": 2.8498722910881042, "l_mix": 1.7636163830757141, "l_a": 1.8308872282505035, "l_c": 1.7899658232927322, "z_loss": 45.845149993896484, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.22337806224822998, "tokens_per_second": 1411606.8130215916, "eta_seconds": 7621.007048678399, "data_wait_seconds": 1.8090009689331055e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 51970, "total_steps": 72479, "tokens": 27247247360, "loss": 4.3852972984313965, "l_mix": 2.718083441257477, "l_a": 2.804166406393051, "l_c": 2.7532128393650055, "z_loss": 39.21731996536255, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11679947376251221, "tokens_per_second": 1398122.489209012, "eta_seconds": 7690.7586245059965, "data_wait_seconds": 2.911686897277832e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 51980, "total_steps": 72479, "tokens": 27252490240, "loss": 4.4402695298194885, "l_mix": 2.7518267035484314, "l_a": 2.839389204978943, "l_c": 2.788753241300583, "z_loss": 39.07846260070801, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15623340010643005, "tokens_per_second": 1434703.3891733037, "eta_seconds": 7491.011586856843, "data_wait_seconds": 1.5527009963989258e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 51990, "total_steps": 72479, "tokens": 27257733120, "loss": 4.316604316234589, "l_mix": 2.6754588186740875, "l_a": 2.760544240474701, "l_c": 2.7099402248859406, "z_loss": 38.72740459442139, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14835114777088165, "tokens_per_second": 1440667.817847006, "eta_seconds": 7456.3592654228205, "data_wait_seconds": 2.2619962692260742e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 52000, "total_steps": 72479, "tokens": 27262976000, "loss": 4.364652633666992, "l_mix": 2.7049390375614166, "l_a": 2.790630728006363, "l_c": 2.741747945547104, "z_loss": 40.21654176712036, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13079287111759186, "tokens_per_second": 1419681.1797471032, "eta_seconds": 7562.890954089165, "data_wait_seconds": 2.1338462829589844e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.1753953, "gate_a2b/L1_absmax": 0.70499283, "gate_a2b/L1_rms": 0.18647303, "gate_a2b/L3_absmean": 0.16983959, "gate_a2b/L3_absmax": 0.54500955, "gate_a2b/L3_rms": 0.17585111, "gate_a2b/L5_absmean": 0.25519761, "gate_a2b/L5_absmax": 0.73939115, "gate_a2b/L5_rms": 0.26123053, "gate_c2b/L1_absmean": 0.0250386, "gate_c2b/L1_absmax": 0.62782007, "gate_c2b/L1_rms": 0.06625738, "gate_c2b/L3_absmean": 0.18022466, "gate_c2b/L3_absmax": 0.40695295, "gate_c2b/L3_rms": 0.18401109, "gate_c2b/L5_absmean": 0.51265568, "gate_c2b/L5_absmax": 0.72343218, "gate_c2b/L5_rms": 0.5231356, "gate_b2a/L7_absmean": 0.05068251, "gate_b2a/L7_absmax": 0.32107788, "gate_b2a/L7_rms": 0.05615157, "gate_b2a/L11_absmean": 0.08200968, "gate_b2a/L11_absmax": 0.60571408, "gate_b2a/L11_rms": 0.09114934, "gate_b2a/L14_absmean": 0.21155308, "gate_b2a/L14_absmax": 0.80648845, "gate_b2a/L14_rms": 0.22048958, "gate_b2c/L3_absmean": 0.03531175, "gate_b2c/L3_absmax": 0.56133878, "gate_b2c/L3_rms": 0.04628683, "gate_b2c/L5_absmean": 0.03763401, "gate_b2c/L5_absmax": 0.56229943, "gate_b2c/L5_rms": 0.04767863, "gate_b2c/L6_absmean": 0.05460087, "gate_b2c/L6_absmax": 0.35593092, "gate_b2c/L6_rms": 0.0608936}, "streams": {"cos_a2b/round0": 0.362613, "cos_c2b/round0": 0.291267, "cos_b2a/round0": 0.151313, "cos_b2c/round0": 0.140649, "rms_a/round0": 6.591103, "rms_b/round0": 11.624902, "rms_c/round0": 4.382588, "cos_a2b/round1": 0.410549, "cos_c2b/round1": 0.29639, "cos_b2a/round1": 0.246325, "cos_b2c/round1": 0.109125, "rms_a/round1": 12.803162, "rms_b/round1": 26.849628, "rms_c/round1": 9.204543, "cos_a2b/round2": 0.54072, "cos_c2b/round2": 0.515371, "cos_b2a/round2": 0.102578, "cos_b2c/round2": 0.178779, "rms_a/round2": 35.736763, "rms_b/round2": 72.247482, "rms_c/round2": 17.703142}, "alpha_mean": 0.4354180904397609, "alpha_hist": [0.01636, 0.05795, 0.16838, 0.20988, 0.18649, 0.16617, 0.12019, 0.06444, 0.00987, 0.00026], "alpha_by_type": {"special": {"share": 0.00108, "mean": 0.53164, "std": 0.12143}, "whitespace": {"share": 0.03676, "mean": 0.52566, "std": 0.11658}, "digit": {"share": 0.04973, "mean": 0.4432, "std": 0.12094}, "word_start": {"share": 0.53993, "mean": 0.45361, "std": 0.16568}, "word_cont": {"share": 0.25883, "mean": 0.31823, "std": 0.12899}, "punct": {"share": 0.11366, "mean": 0.58236, "std": 0.1551}}, "alpha_p05": 0.17955, "alpha_p50": 0.42274, "alpha_p95": 0.72565}
{"step": 52010, "total_steps": 72479, "tokens": 27268218880, "loss": 4.360487401485443, "l_mix": 2.7029087245464325, "l_a": 2.7885990142822266, "l_c": 2.7366630136966705, "z_loss": 39.73223638534546, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.21359661221504211, "tokens_per_second": 1433537.3764692054, "eta_seconds": 7486.132728838921, "data_wait_seconds": 2.16066837310791e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 52020, "total_steps": 72479, "tokens": 27273461760, "loss": 4.675557076931, "l_mix": 2.90008482336998, "l_a": 2.978283315896988, "l_c": 2.939957320690155, "z_loss": 39.28368854522705, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12388565391302109, "tokens_per_second": 1355158.1466710544, "eta_seconds": 7915.244592189789, "data_wait_seconds": 2.6792287826538086e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 52030, "total_steps": 72479, "tokens": 27278704640, "loss": 4.627979874610901, "l_mix": 2.8708722591400146, "l_a": 2.9499164521694183, "l_c": 2.9071086943149567, "z_loss": 38.26639413833618, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1654365360736847, "tokens_per_second": 1393852.1536892417, "eta_seconds": 7691.752158665657, "data_wait_seconds": 2.2619962692260742e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 52040, "total_steps": 72479, "tokens": 27283947520, "loss": 4.5949002504348755, "l_mix": 2.851663291454315, "l_a": 2.924798607826233, "l_c": 2.885991334915161, "z_loss": 38.638686656951904, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15677152574062347, "tokens_per_second": 1469370.370130415, "eta_seconds": 7292.866829109192, "data_wait_seconds": 2.0444393157958984e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 52050, "total_steps": 72479, "tokens": 27289190400, "loss": 4.920113980770111, "l_mix": 3.052807778120041, "l_a": 3.1284618973731995, "l_c": 3.0958920419216156, "z_loss": 38.70281791687012, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18137700855731964, "tokens_per_second": 1433229.3317037665, "eta_seconds": 7473.109372711181, "data_wait_seconds": 1.7255544662475586e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 52060, "total_steps": 72479, "tokens": 27294433280, "loss": 5.135861456394196, "l_mix": 3.1859572529792786, "l_a": 3.271979033946991, "l_c": 3.227701097726822, "z_loss": 38.54978895187378, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.9723479747772217, "tokens_per_second": 1240389.4835668465, "eta_seconds": 8630.70576930046, "data_wait_seconds": 1.9103288650512695e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 52070, "total_steps": 72479, "tokens": 27299676160, "loss": 2.94848296046257, "l_mix": 1.825265347957611, "l_a": 1.8943880647420883, "l_c": 1.8496706038713455, "z_loss": 49.72317409515381, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2027127742767334, "tokens_per_second": 1453762.1303743096, "eta_seconds": 7360.347039198876, "data_wait_seconds": 1.990795135498047e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 52080, "total_steps": 72479, "tokens": 27304919040, "loss": 4.35424417257309, "l_mix": 2.6977698802948, "l_a": 2.7863695323467255, "l_c": 2.7352113723754883, "z_loss": 39.614062786102295, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1370447427034378, "tokens_per_second": 1452397.246162416, "eta_seconds": 7363.654083108902, "data_wait_seconds": 3.311038017272949e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 52090, "total_steps": 72479, "tokens": 27310161920, "loss": 4.313563346862793, "l_mix": 2.6731038987636566, "l_a": 2.759636163711548, "l_c": 2.7085615694522858, "z_loss": 39.597872257232666, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14835326373577118, "tokens_per_second": 1326729.1931526535, "eta_seconds": 8057.189128851891, "data_wait_seconds": 2.2411346435546875e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 52100, "total_steps": 72479, "tokens": 27315404800, "loss": 4.477541208267212, "l_mix": 2.7761062681674957, "l_a": 2.859471023082733, "l_c": 2.811978369951248, "z_loss": 40.09431791305542, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14338737726211548, "tokens_per_second": 1119838.3169950193, "eta_seconds": 9541.078376984595, "data_wait_seconds": 1.531839370727539e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.17498665, "gate_a2b/L1_absmax": 0.71100283, "gate_a2b/L1_rms": 0.18609428, "gate_a2b/L3_absmean": 0.16881834, "gate_a2b/L3_absmax": 0.54415131, "gate_a2b/L3_rms": 0.17497125, "gate_a2b/L5_absmean": 0.25397328, "gate_a2b/L5_absmax": 0.7380299, "gate_a2b/L5_rms": 0.26002225, "gate_c2b/L1_absmean": 0.02455143, "gate_c2b/L1_absmax": 0.63126409, "gate_c2b/L1_rms": 0.06655043, "gate_c2b/L3_absmean": 0.18011419, "gate_c2b/L3_absmax": 0.40620178, "gate_c2b/L3_rms": 0.18397583, "gate_c2b/L5_absmean": 0.51277113, "gate_c2b/L5_absmax": 0.72232586, "gate_c2b/L5_rms": 0.52327645, "gate_b2a/L7_absmean": 0.05067866, "gate_b2a/L7_absmax": 0.32127228, "gate_b2a/L7_rms": 0.05616687, "gate_b2a/L11_absmean": 0.0823385, "gate_b2a/L11_absmax": 0.60775805, "gate_b2a/L11_rms": 0.09147304, "gate_b2a/L14_absmean": 0.21169952, "gate_b2a/L14_absmax": 0.8090772, "gate_b2a/L14_rms": 0.2206144, "gate_b2c/L3_absmean": 0.03548637, "gate_b2c/L3_absmax": 0.56302208, "gate_b2c/L3_rms": 0.04648066, "gate_b2c/L5_absmean": 0.0376896, "gate_b2c/L5_absmax": 0.56197852, "gate_b2c/L5_rms": 0.04768499, "gate_b2c/L6_absmean": 0.05425143, "gate_b2c/L6_absmax": 0.35600507, "gate_b2c/L6_rms": 0.06058009}, "streams": {"cos_a2b/round0": 0.365341, "cos_c2b/round0": 0.291176, "cos_b2a/round0": 0.150949, "cos_b2c/round0": 0.139332, "rms_a/round0": 6.613487, "rms_b/round0": 11.593898, "rms_c/round0": 4.448758, "cos_a2b/round1": 0.406376, "cos_c2b/round1": 0.291856, "cos_b2a/round1": 0.247752, "cos_b2c/round1": 0.107833, "rms_a/round1": 12.689481, "rms_b/round1": 27.029259, "rms_c/round1": 9.291352, "cos_a2b/round2": 0.534722, "cos_c2b/round2": 0.518441, "cos_b2a/round2": 0.096878, "cos_b2c/round2": 0.173721, "rms_a/round2": 35.730034, "rms_b/round2": 72.709946, "rms_c/round2": 17.806374}, "alpha_mean": 0.42087191571458504, "alpha_hist": [0.00908, 0.07401, 0.20236, 0.215, 0.17355, 0.1447, 0.10922, 0.06345, 0.00851, 0.00012], "alpha_by_type": {"special": {"share": 0.00078, "mean": 0.55335, "std": 0.11842}, "whitespace": {"share": 0.03088, "mean": 0.54282, "std": 0.12327}, "digit": {"share": 0.0584, "mean": 0.46612, "std": 0.13452}, "word_start": {"share": 0.50722, "mean": 0.42964, "std": 0.16861}, "word_cont": {"share": 0.2646, "mean": 0.31251, "std": 0.11828}, "punct": {"share": 0.13812, "mean": 0.5491, "std": 0.16814}}, "alpha_p05": 0.17189, "alpha_p50": 0.40733, "alpha_p95": 0.7068}
{"step": 52110, "total_steps": 72479, "tokens": 27320647680, "loss": 4.380768597126007, "l_mix": 2.7148426175117493, "l_a": 2.8012419939041138, "l_c": 2.7518441677093506, "z_loss": 39.219436168670654, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15729042887687683, "tokens_per_second": 1445735.4021564042, "eta_seconds": 7386.705932545662, "data_wait_seconds": 1.5944242477416992e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 52120, "total_steps": 72479, "tokens": 27325890560, "loss": 4.386331140995026, "l_mix": 2.718510717153549, "l_a": 2.8070945739746094, "l_c": 2.7523064613342285, "z_loss": 39.34252405166626, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1499343365430832, "tokens_per_second": 1478550.755577167, "eta_seconds": 7219.217434191703, "data_wait_seconds": 1.5676021575927734e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 52130, "total_steps": 72479, "tokens": 27331133440, "loss": 3.1033803820610046, "l_mix": 1.9089232087135315, "l_a": 2.02492792904377, "l_c": 1.9565958231687546, "z_loss": 43.84420585632324, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1664944738149643, "tokens_per_second": 1420701.3807417469, "eta_seconds": 7509.485565805435, "data_wait_seconds": 1.8894672393798828e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 52140, "total_steps": 72479, "tokens": 27336376320, "loss": 3.1170102953910828, "l_mix": 1.9180019944906235, "l_a": 2.031388521194458, "l_c": 1.9653055965900421, "z_loss": 43.07432556152344, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14693708717823029, "tokens_per_second": 1441133.3752225328, "eta_seconds": 7399.380109667778, "data_wait_seconds": 2.2679567337036133e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 52150, "total_steps": 72479, "tokens": 27341619200, "loss": 4.726098895072937, "l_mix": 2.932501405477524, "l_a": 3.0076270699501038, "l_c": 2.971030980348587, "z_loss": 38.691564083099365, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11464540660381317, "tokens_per_second": 949223.0419846538, "eta_seconds": 11228.394466400146, "data_wait_seconds": 3.993511199951172e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 52160, "total_steps": 72479, "tokens": 27346862080, "loss": 4.7026606798172, "l_mix": 2.9179888665676117, "l_a": 2.9923951029777527, "l_c": 2.956510603427887, "z_loss": 38.71709156036377, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12552349269390106, "tokens_per_second": 1452672.3210531743, "eta_seconds": 7333.386695408821, "data_wait_seconds": 2.8580427169799805e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 52170, "total_steps": 72479, "tokens": 27352104960, "loss": 4.628611862659454, "l_mix": 2.8713479042053223, "l_a": 2.9482383728027344, "l_c": 2.909308135509491, "z_loss": 39.48322677612305, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.10649233311414719, "tokens_per_second": 1441257.4865141606, "eta_seconds": 7387.8297886610035, "data_wait_seconds": 2.7745962142944336e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 52180, "total_steps": 72479, "tokens": 27357347840, "loss": 3.1457068622112274, "l_mix": 1.9500157982110977, "l_a": 2.0077570229768753, "l_c": 1.977879673242569, "z_loss": 46.807459115982056, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.41603711247444153, "tokens_per_second": 1446604.5777163152, "eta_seconds": 7356.897853040695, "data_wait_seconds": 1.919269561767578e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 52190, "total_steps": 72479, "tokens": 27362590720, "loss": 4.352022588253021, "l_mix": 2.697590798139572, "l_a": 2.7828610837459564, "l_c": 2.7319113314151764, "z_loss": 39.09346008300781, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14011096954345703, "tokens_per_second": 1452592.7717739968, "eta_seconds": 7322.96032218933, "data_wait_seconds": 1.5556812286376953e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 52200, "total_steps": 72479, "tokens": 27367833600, "loss": 4.440674841403961, "l_mix": 2.752597689628601, "l_a": 2.8375734984874725, "l_c": 2.789349853992462, "z_loss": 39.59596109390259, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15322263538837433, "tokens_per_second": 1411397.1622244283, "eta_seconds": 7532.986912941933, "data_wait_seconds": 1.71661376953125e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.17511982, "gate_a2b/L1_absmax": 0.71319544, "gate_a2b/L1_rms": 0.18637441, "gate_a2b/L3_absmean": 0.16837338, "gate_a2b/L3_absmax": 0.53887486, "gate_a2b/L3_rms": 0.17446326, "gate_a2b/L5_absmean": 0.25490713, "gate_a2b/L5_absmax": 0.7343837, "gate_a2b/L5_rms": 0.26090261, "gate_c2b/L1_absmean": 0.02511907, "gate_c2b/L1_absmax": 0.62966919, "gate_c2b/L1_rms": 0.06655107, "gate_c2b/L3_absmean": 0.18013267, "gate_c2b/L3_absmax": 0.41341922, "gate_c2b/L3_rms": 0.18409251, "gate_c2b/L5_absmean": 0.51071191, "gate_c2b/L5_absmax": 0.71905845, "gate_c2b/L5_rms": 0.52107638, "gate_b2a/L7_absmean": 0.05073671, "gate_b2a/L7_absmax": 0.31887132, "gate_b2a/L7_rms": 0.056483, "gate_b2a/L11_absmean": 0.08213827, "gate_b2a/L11_absmax": 0.60295516, "gate_b2a/L11_rms": 0.09129636, "gate_b2a/L14_absmean": 0.21094258, "gate_b2a/L14_absmax": 0.80499125, "gate_b2a/L14_rms": 0.21999398, "gate_b2c/L3_absmean": 0.0352602, "gate_b2c/L3_absmax": 0.56378579, "gate_b2c/L3_rms": 0.04644113, "gate_b2c/L5_absmean": 0.03780865, "gate_b2c/L5_absmax": 0.56862187, "gate_b2c/L5_rms": 0.0480768, "gate_b2c/L6_absmean": 0.05435845, "gate_b2c/L6_absmax": 0.35938942, "gate_b2c/L6_rms": 0.06081692}, "streams": {"cos_a2b/round0": 0.361507, "cos_c2b/round0": 0.28873, "cos_b2a/round0": 0.158483, "cos_b2c/round0": 0.13987, "rms_a/round0": 6.644409, "rms_b/round0": 11.671831, "rms_c/round0": 4.471969, "cos_a2b/round1": 0.41334, "cos_c2b/round1": 0.305193, "cos_b2a/round1": 0.255653, "cos_b2c/round1": 0.113526, "rms_a/round1": 12.783493, "rms_b/round1": 26.887817, "rms_c/round1": 9.273293, "cos_a2b/round2": 0.539278, "cos_c2b/round2": 0.524385, "cos_b2a/round2": 0.098737, "cos_b2c/round2": 0.18447, "rms_a/round2": 35.818996, "rms_b/round2": 72.861076, "rms_c/round2": 17.814844}, "alpha_mean": 0.4449601725557475, "alpha_hist": [0.00845, 0.06554, 0.17104, 0.19295, 0.18407, 0.15822, 0.12056, 0.08192, 0.01707, 0.00018], "alpha_by_type": {"special": {"share": 0.00111, "mean": 0.54435, "std": 0.11721}, "whitespace": {"share": 0.02992, "mean": 0.53395, "std": 0.12347}, "digit": {"share": 0.0442, "mean": 0.45883, "std": 0.11728}, "word_start": {"share": 0.5795, "mean": 0.46049, "std": 0.17271}, "word_cont": {"share": 0.23943, "mean": 0.33137, "std": 0.13289}, "punct": {"share": 0.10583, "mean": 0.58489, "std": 0.18243}}, "alpha_p05": 0.18536, "alpha_p50": 0.44179, "alpha_p95": 0.74614}
{"step": 52210, "total_steps": 72479, "tokens": 27373076480, "loss": 4.4096872210502625, "l_mix": 2.7318198680877686, "l_a": 2.821716845035553, "l_c": 2.771173983812332, "z_loss": 40.6751275062561, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18425774574279785, "tokens_per_second": 1478810.5670771697, "eta_seconds": 7186.041071510314, "data_wait_seconds": 1.6123056411743164e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 52220, "total_steps": 72479, "tokens": 27378319360, "loss": 4.324094355106354, "l_mix": 2.6802835166454315, "l_a": 2.762724280357361, "l_c": 2.7166448831558228, "z_loss": 39.70520639419556, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11619497090578079, "tokens_per_second": 1345747.6510262946, "eta_seconds": 7892.676300716401, "data_wait_seconds": 1.9043684005737305e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 52230, "total_steps": 72479, "tokens": 27383562240, "loss": 4.250699520111084, "l_mix": 2.6329663395881653, "l_a": 2.723383754491806, "l_c": 2.669059604406357, "z_loss": 40.22709131240845, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13980187475681305, "tokens_per_second": 1297672.6675979046, "eta_seconds": 8181.036695218087, "data_wait_seconds": 1.946091651916504e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 52240, "total_steps": 72479, "tokens": 27388805120, "loss": 3.1236080825328827, "l_mix": 1.921809360384941, "l_a": 2.035668522119522, "l_c": 1.9703269600868225, "z_loss": 43.274155139923096, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17234691977500916, "tokens_per_second": 1315281.2869164571, "eta_seconds": 8067.5251275539395, "data_wait_seconds": 1.9878149032592773e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 52250, "total_steps": 72479, "tokens": 27394048000, "loss": 3.158418118953705, "l_mix": 1.943529173731804, "l_a": 2.0569924414157867, "l_c": 1.9926371425390244, "z_loss": 43.15407752990723, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1874416321516037, "tokens_per_second": 1451886.0391455763, "eta_seconds": 7304.858415913583, "data_wait_seconds": 1.9222497940063477e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 52260, "total_steps": 72479, "tokens": 27399290880, "loss": 3.1622008085250854, "l_mix": 1.9449071735143661, "l_a": 2.0616772770881653, "l_c": 1.9959680140018463, "z_loss": 43.162282943725586, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17746266722679138, "tokens_per_second": 1456483.8847637367, "eta_seconds": 7278.198669338227, "data_wait_seconds": 2.41696834564209e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 52270, "total_steps": 72479, "tokens": 27404533760, "loss": 3.1466748416423798, "l_mix": 1.9364100396633148, "l_a": 2.0491218268871307, "l_c": 1.9850940704345703, "z_loss": 41.59704303741455, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1808517575263977, "tokens_per_second": 1394980.4045400391, "eta_seconds": 7595.3297677278515, "data_wait_seconds": 2.60770320892334e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 52280, "total_steps": 72479, "tokens": 27409776640, "loss": 3.0959765315055847, "l_mix": 1.90421824157238, "l_a": 2.0205514430999756, "l_c": 1.9519763588905334, "z_loss": 42.70655822753906, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15582843124866486, "tokens_per_second": 1445779.8864834816, "eta_seconds": 7324.830986380577, "data_wait_seconds": 1.6391277313232422e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 52290, "total_steps": 72479, "tokens": 27415019520, "loss": 4.72305691242218, "l_mix": 2.9302115738391876, "l_a": 3.0086854100227356, "l_c": 2.967465579509735, "z_loss": 38.78494071960449, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1276969462633133, "tokens_per_second": 1156896.2257446302, "eta_seconds": 9149.35168466568, "data_wait_seconds": 1.722574234008789e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 52300, "total_steps": 72479, "tokens": 27420262400, "loss": 4.414725959300995, "l_mix": 2.7369294464588165, "l_a": 2.8215811252593994, "l_c": 2.7710735201835632, "z_loss": 38.70468282699585, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1303696632385254, "tokens_per_second": 1296549.102461436, "eta_seconds": 8159.82019648552, "data_wait_seconds": 2.1576881408691406e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.17535166, "gate_a2b/L1_absmax": 0.7077921, "gate_a2b/L1_rms": 0.18647094, "gate_a2b/L3_absmean": 0.16838197, "gate_a2b/L3_absmax": 0.53350586, "gate_a2b/L3_rms": 0.17436799, "gate_a2b/L5_absmean": 0.25380278, "gate_a2b/L5_absmax": 0.7321952, "gate_a2b/L5_rms": 0.25976747, "gate_c2b/L1_absmean": 0.02450657, "gate_c2b/L1_absmax": 0.62892091, "gate_c2b/L1_rms": 0.06637557, "gate_c2b/L3_absmean": 0.17982693, "gate_c2b/L3_absmax": 0.41010666, "gate_c2b/L3_rms": 0.18374373, "gate_c2b/L5_absmean": 0.51110858, "gate_c2b/L5_absmax": 0.72442007, "gate_c2b/L5_rms": 0.52153063, "gate_b2a/L7_absmean": 0.05041359, "gate_b2a/L7_absmax": 0.32278681, "gate_b2a/L7_rms": 0.05611288, "gate_b2a/L11_absmean": 0.08190058, "gate_b2a/L11_absmax": 0.606947, "gate_b2a/L11_rms": 0.09113258, "gate_b2a/L14_absmean": 0.21139941, "gate_b2a/L14_absmax": 0.80534369, "gate_b2a/L14_rms": 0.2204144, "gate_b2c/L3_absmean": 0.03546025, "gate_b2c/L3_absmax": 0.56591392, "gate_b2c/L3_rms": 0.04647535, "gate_b2c/L5_absmean": 0.03778782, "gate_b2c/L5_absmax": 0.5635221, "gate_b2c/L5_rms": 0.04782274, "gate_b2c/L6_absmean": 0.05422351, "gate_b2c/L6_absmax": 0.35513821, "gate_b2c/L6_rms": 0.06055385}, "streams": {"cos_a2b/round0": 0.357027, "cos_c2b/round0": 0.285085, "cos_b2a/round0": 0.156195, "cos_b2c/round0": 0.138456, "rms_a/round0": 6.590512, "rms_b/round0": 11.648938, "rms_c/round0": 4.450963, "cos_a2b/round1": 0.413393, "cos_c2b/round1": 0.303683, "cos_b2a/round1": 0.250417, "cos_b2c/round1": 0.114098, "rms_a/round1": 12.718412, "rms_b/round1": 26.890142, "rms_c/round1": 9.131419, "cos_a2b/round2": 0.544345, "cos_c2b/round2": 0.520605, "cos_b2a/round2": 0.097965, "cos_b2c/round2": 0.183727, "rms_a/round2": 35.547852, "rms_b/round2": 72.205048, "rms_c/round2": 17.24379}, "alpha_mean": 0.43643496376657964, "alpha_hist": [0.01537, 0.06584, 0.17133, 0.19725, 0.1833, 0.16095, 0.12582, 0.07057, 0.00938, 0.00018], "alpha_by_type": {"special": {"share": 0.00089, "mean": 0.52483, "std": 0.1317}, "whitespace": {"share": 0.03259, "mean": 0.51279, "std": 0.12871}, "digit": {"share": 0.03267, "mean": 0.43816, "std": 0.11601}, "word_start": {"share": 0.56265, "mean": 0.46097, "std": 0.16555}, "word_cont": {"share": 0.27028, "mean": 0.31755, "std": 0.13472}, "punct": {"share": 0.10092, "mean": 0.59204, "std": 0.15925}}, "alpha_p05": 0.16886, "alpha_p50": 0.43758, "alpha_p95": 0.74017}
{"step": 52310, "total_steps": 72479, "tokens": 27425505280, "loss": 4.427136301994324, "l_mix": 2.743841767311096, "l_a": 2.8298639059066772, "l_c": 2.781117618083954, "z_loss": 39.37877178192139, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14449989795684814, "tokens_per_second": 1428886.5534243272, "eta_seconds": 7400.42283039093, "data_wait_seconds": 2.5272369384765625e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 52320, "total_steps": 72479, "tokens": 27430748160, "loss": 4.447764456272125, "l_mix": 2.7565755546092987, "l_a": 2.844656825065613, "l_c": 2.792639344930649, "z_loss": 40.06359386444092, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12680001556873322, "tokens_per_second": 1426714.0032814725, "eta_seconds": 7408.017141270637, "data_wait_seconds": 2.0951032638549805e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 52330, "total_steps": 72479, "tokens": 27435991040, "loss": 4.329699724912643, "l_mix": 2.682477593421936, "l_a": 2.772080898284912, "l_c": 2.7186594903469086, "z_loss": 39.68808078765869, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14028094708919525, "tokens_per_second": 1364284.5349728067, "eta_seconds": 7743.164011025429, "data_wait_seconds": 1.7076730728149414e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 52340, "total_steps": 72479, "tokens": 27441233920, "loss": 4.368435263633728, "l_mix": 2.7080948054790497, "l_a": 2.7912087440490723, "l_c": 2.743259012699127, "z_loss": 39.84660339355469, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1091540977358818, "tokens_per_second": 1283562.5620408277, "eta_seconds": 8226.039263105393, "data_wait_seconds": 0.038232892751693726, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 52350, "total_steps": 72479, "tokens": 27446476800, "loss": 3.1264836192131042, "l_mix": 1.9238836169242859, "l_a": 2.0381942093372345, "l_c": 1.9704723507165909, "z_loss": 43.31338310241699, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19944240152835846, "tokens_per_second": 1202003.7204065057, "eta_seconds": 8779.834016180039, "data_wait_seconds": 2.187490463256836e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 52360, "total_steps": 72479, "tokens": 27451719680, "loss": 3.093336045742035, "l_mix": 1.9025982171297073, "l_a": 2.0180881172418594, "l_c": 1.9510378986597061, "z_loss": 43.64190149307251, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16899055242538452, "tokens_per_second": 1384802.295933844, "eta_seconds": 7617.080288624764, "data_wait_seconds": 2.1189451217651367e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 52370, "total_steps": 72479, "tokens": 27456962560, "loss": 3.1603678464889526, "l_mix": 1.9459822177886963, "l_a": 2.05531707406044, "l_c": 1.9926350861787796, "z_loss": 41.36975431442261, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20988360047340393, "tokens_per_second": 1419409.019298143, "eta_seconds": 7427.673946452141, "data_wait_seconds": 2.1904706954956055e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 52380, "total_steps": 72479, "tokens": 27462205440, "loss": 3.1355701684951782, "l_mix": 1.9288573116064072, "l_a": 2.04396516084671, "l_c": 1.9784110337495804, "z_loss": 41.28536939620972, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.21818594634532928, "tokens_per_second": 1445888.352267542, "eta_seconds": 7288.02088727951, "data_wait_seconds": 1.6063451766967773e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 52390, "total_steps": 72479, "tokens": 27467448320, "loss": 3.1493270993232727, "l_mix": 1.9381888210773468, "l_a": 2.0517921447753906, "l_c": 1.9853351414203644, "z_loss": 41.52331829071045, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14379660785198212, "tokens_per_second": 1474639.4903689087, "eta_seconds": 7142.370525670051, "data_wait_seconds": 2.8699636459350586e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 52400, "total_steps": 72479, "tokens": 27472691200, "loss": 3.117443025112152, "l_mix": 1.9177696853876114, "l_a": 2.0306118428707123, "l_c": 1.9682991951704025, "z_loss": 42.99238061904907, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14888961613178253, "tokens_per_second": 1425958.2451885603, "eta_seconds": 7382.529458713532, "data_wait_seconds": 1.6987323760986328e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.1747209, "gate_a2b/L1_absmax": 0.70791155, "gate_a2b/L1_rms": 0.18581511, "gate_a2b/L3_absmean": 0.16799128, "gate_a2b/L3_absmax": 0.53790498, "gate_a2b/L3_rms": 0.17401555, "gate_a2b/L5_absmean": 0.25345427, "gate_a2b/L5_absmax": 0.73469079, "gate_a2b/L5_rms": 0.25949085, "gate_c2b/L1_absmean": 0.02493286, "gate_c2b/L1_absmax": 0.6301139, "gate_c2b/L1_rms": 0.06650917, "gate_c2b/L3_absmean": 0.17960301, "gate_c2b/L3_absmax": 0.40924746, "gate_c2b/L3_rms": 0.1834666, "gate_c2b/L5_absmean": 0.5100444, "gate_c2b/L5_absmax": 0.71577036, "gate_c2b/L5_rms": 0.52053446, "gate_b2a/L7_absmean": 0.05084534, "gate_b2a/L7_absmax": 0.32242352, "gate_b2a/L7_rms": 0.05654794, "gate_b2a/L11_absmean": 0.08189917, "gate_b2a/L11_absmax": 0.60475779, "gate_b2a/L11_rms": 0.09111221, "gate_b2a/L14_absmean": 0.21131077, "gate_b2a/L14_absmax": 0.80618453, "gate_b2a/L14_rms": 0.22027214, "gate_b2c/L3_absmean": 0.03525004, "gate_b2c/L3_absmax": 0.56275803, "gate_b2c/L3_rms": 0.04623966, "gate_b2c/L5_absmean": 0.03762513, "gate_b2c/L5_absmax": 0.55846512, "gate_b2c/L5_rms": 0.04757242, "gate_b2c/L6_absmean": 0.05445518, "gate_b2c/L6_absmax": 0.35581481, "gate_b2c/L6_rms": 0.0607684}, "streams": {"cos_a2b/round0": 0.354743, "cos_c2b/round0": 0.24673, "cos_b2a/round0": 0.140661, "cos_b2c/round0": 0.114059, "rms_a/round0": 6.893591, "rms_b/round0": 11.844896, "rms_c/round0": 4.580634, "cos_a2b/round1": 0.408359, "cos_c2b/round1": 0.305744, "cos_b2a/round1": 0.241421, "cos_b2c/round1": 0.121964, "rms_a/round1": 13.473786, "rms_b/round1": 28.48167, "rms_c/round1": 9.834839, "cos_a2b/round2": 0.517968, "cos_c2b/round2": 0.508545, "cos_b2a/round2": 0.089926, "cos_b2c/round2": 0.171971, "rms_a/round2": 36.419327, "rms_b/round2": 74.138741, "rms_c/round2": 18.878349}, "alpha_mean": 0.3986791463191892, "alpha_hist": [0.04095, 0.1776, 0.19429, 0.14737, 0.11337, 0.10492, 0.09581, 0.08298, 0.04071, 0.002], "alpha_by_type": {"special": {"share": 0.00105, "mean": 0.57494, "std": 0.13747}, "whitespace": {"share": 0.03268, "mean": 0.62041, "std": 0.17123}, "digit": {"share": 0.01048, "mean": 0.34642, "std": 0.16917}, "word_start": {"share": 0.57658, "mean": 0.40408, "std": 0.21066}, "word_cont": {"share": 0.26521, "mean": 0.26641, "std": 0.13859}, "punct": {"share": 0.11398, "mean": 0.61874, "std": 0.18297}}, "alpha_p05": 0.10088, "alpha_p50": 0.35622, "alpha_p95": 0.79597}
{"step": 52410, "total_steps": 72479, "tokens": 27477934080, "loss": 4.4283058643341064, "l_mix": 2.7450684010982513, "l_a": 2.8305851221084595, "l_c": 2.7802064418792725, "z_loss": 39.806697368621826, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13538198173046112, "tokens_per_second": 1402779.333567574, "eta_seconds": 7500.777649211884, "data_wait_seconds": 2.104043960571289e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 52420, "total_steps": 72479, "tokens": 27483176960, "loss": 4.449763596057892, "l_mix": 2.757085084915161, "l_a": 2.8454236686229706, "l_c": 2.7968378961086273, "z_loss": 39.56829357147217, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14922283589839935, "tokens_per_second": 1473088.1929391522, "eta_seconds": 7139.214775061608, "data_wait_seconds": 1.5407800674438477e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 52430, "total_steps": 72479, "tokens": 27488419840, "loss": 4.380450963973999, "l_mix": 2.7148554921150208, "l_a": 2.802708774805069, "l_c": 2.7492757439613342, "z_loss": 39.98886966705322, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1528746336698532, "tokens_per_second": 1304584.7128090058, "eta_seconds": 8057.315104794502, "data_wait_seconds": 3.796815872192383e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 52440, "total_steps": 72479, "tokens": 27493662720, "loss": 4.455343544483185, "l_mix": 2.760891765356064, "l_a": 2.8509950637817383, "l_c": 2.797177404165268, "z_loss": 39.467655658721924, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11390173435211182, "tokens_per_second": 1285024.8131939038, "eta_seconds": 8175.878881192208, "data_wait_seconds": 2.1398067474365234e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 52450, "total_steps": 72479, "tokens": 27498905600, "loss": 4.350605368614197, "l_mix": 2.695933222770691, "l_a": 2.7823233604431152, "l_c": 2.733250141143799, "z_loss": 39.92608976364136, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1461007297039032, "tokens_per_second": 1375521.933151467, "eta_seconds": 7634.167146968841, "data_wait_seconds": 2.5898218154907227e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 52460, "total_steps": 72479, "tokens": 27504148480, "loss": 3.143183469772339, "l_mix": 1.9349110424518585, "l_a": 2.0473306626081467, "l_c": 1.9802438914775848, "z_loss": 42.03949499130249, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15038447082042694, "tokens_per_second": 1319622.0049364984, "eta_seconds": 7953.581732296943, "data_wait_seconds": 3.540515899658203e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 52470, "total_steps": 72479, "tokens": 27509391360, "loss": 3.163490206003189, "l_mix": 1.9471469223499298, "l_a": 2.0604841709136963, "l_c": 1.9939933121204376, "z_loss": 41.82372999191284, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16421586275100708, "tokens_per_second": 1361265.9103943654, "eta_seconds": 7706.413943004609, "data_wait_seconds": 1.862645149230957e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 52480, "total_steps": 72479, "tokens": 27514634240, "loss": 3.133597433567047, "l_mix": 1.928562268614769, "l_a": 2.0408636778593063, "l_c": 1.975920021533966, "z_loss": 43.91728448867798, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.23020760715007782, "tokens_per_second": 1435810.4548826637, "eta_seconds": 7302.660094404221, "data_wait_seconds": 4.2825937271118164e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 52490, "total_steps": 72479, "tokens": 27519877120, "loss": 3.120980829000473, "l_mix": 1.9202774614095688, "l_a": 2.03258416056633, "l_c": 1.9697601050138474, "z_loss": 43.72492694854736, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17171736061573029, "tokens_per_second": 1432994.6263934488, "eta_seconds": 7313.351103329658, "data_wait_seconds": 1.9222497940063477e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 52500, "total_steps": 72479, "tokens": 27525120000, "loss": 3.1264328062534332, "l_mix": 1.923701897263527, "l_a": 2.0375019758939743, "l_c": 1.9716012328863144, "z_loss": 42.6164288520813, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14369302988052368, "tokens_per_second": 1370227.1272641532, "eta_seconds": 7644.535525226594, "data_wait_seconds": 2.753734588623047e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.17475481, "gate_a2b/L1_absmax": 0.70852184, "gate_a2b/L1_rms": 0.18595119, "gate_a2b/L3_absmean": 0.1683526, "gate_a2b/L3_absmax": 0.53669024, "gate_a2b/L3_rms": 0.17433773, "gate_a2b/L5_absmean": 0.25294617, "gate_a2b/L5_absmax": 0.73515671, "gate_a2b/L5_rms": 0.25894263, "gate_c2b/L1_absmean": 0.02539505, "gate_c2b/L1_absmax": 0.63127524, "gate_c2b/L1_rms": 0.06660448, "gate_c2b/L3_absmean": 0.17943093, "gate_c2b/L3_absmax": 0.40790346, "gate_c2b/L3_rms": 0.18325931, "gate_c2b/L5_absmean": 0.50972366, "gate_c2b/L5_absmax": 0.72307432, "gate_c2b/L5_rms": 0.52014476, "gate_b2a/L7_absmean": 0.05063467, "gate_b2a/L7_absmax": 0.32409135, "gate_b2a/L7_rms": 0.05626579, "gate_b2a/L11_absmean": 0.08193819, "gate_b2a/L11_absmax": 0.60506195, "gate_b2a/L11_rms": 0.09110163, "gate_b2a/L14_absmean": 0.21143031, "gate_b2a/L14_absmax": 0.80353814, "gate_b2a/L14_rms": 0.22035873, "gate_b2c/L3_absmean": 0.03523684, "gate_b2c/L3_absmax": 0.56214446, "gate_b2c/L3_rms": 0.04625208, "gate_b2c/L5_absmean": 0.03748015, "gate_b2c/L5_absmax": 0.55959004, "gate_b2c/L5_rms": 0.0475043, "gate_b2c/L6_absmean": 0.05422179, "gate_b2c/L6_absmax": 0.35478011, "gate_b2c/L6_rms": 0.06055988}, "streams": {"cos_a2b/round0": 0.356326, "cos_c2b/round0": 0.246984, "cos_b2a/round0": 0.140467, "cos_b2c/round0": 0.116652, "rms_a/round0": 6.881851, "rms_b/round0": 11.992686, "rms_c/round0": 4.599356, "cos_a2b/round1": 0.412237, "cos_c2b/round1": 0.304893, "cos_b2a/round1": 0.242819, "cos_b2c/round1": 0.121906, "rms_a/round1": 13.564058, "rms_b/round1": 28.619181, "rms_c/round1": 9.893726, "cos_a2b/round2": 0.519386, "cos_c2b/round2": 0.508754, "cos_b2a/round2": 0.087544, "cos_b2c/round2": 0.171272, "rms_a/round2": 36.764786, "rms_b/round2": 74.075958, "rms_c/round2": 19.255957}, "alpha_mean": 0.4099221270570155, "alpha_hist": [0.03206, 0.166, 0.19756, 0.15019, 0.11739, 0.09932, 0.09653, 0.09106, 0.04689, 0.00301], "alpha_by_type": {"special": {"share": 0.0011, "mean": 0.60785, "std": 0.12888}, "whitespace": {"share": 0.02934, "mean": 0.62615, "std": 0.16992}, "digit": {"share": 0.01038, "mean": 0.37937, "std": 0.16965}, "word_start": {"share": 0.57559, "mean": 0.41912, "std": 0.21327}, "word_cont": {"share": 0.274, "mean": 0.27872, "std": 0.14178}, "punct": {"share": 0.10959, "mean": 0.63267, "std": 0.18811}}, "alpha_p05": 0.12085, "alpha_p50": 0.34821, "alpha_p95": 0.79085}
{"step": 52510, "total_steps": 72479, "tokens": 27530362880, "loss": 3.160256505012512, "l_mix": 1.9439661800861359, "l_a": 2.0599038302898407, "l_c": 1.9943970143795013, "z_loss": 42.390392780303955, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17634828388690948, "tokens_per_second": 1421351.9813637454, "eta_seconds": 7365.87925388813, "data_wait_seconds": 1.8537044525146484e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 52520, "total_steps": 72479, "tokens": 27535605760, "loss": 4.52945739030838, "l_mix": 2.8082237243652344, "l_a": 2.892617791891098, "l_c": 2.8448278307914734, "z_loss": 39.997313499450684, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17215262353420258, "tokens_per_second": 1426715.114055227, "eta_seconds": 7334.515551781655, "data_wait_seconds": 2.193450927734375e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 52530, "total_steps": 72479, "tokens": 27540848640, "loss": 4.276019096374512, "l_mix": 2.6493186354637146, "l_a": 2.73602357506752, "l_c": 2.6863110661506653, "z_loss": 39.61212921142578, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15070249140262604, "tokens_per_second": 1378417.1835793448, "eta_seconds": 7587.703807377816, "data_wait_seconds": 1.8805265426635742e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 52540, "total_steps": 72479, "tokens": 27546091520, "loss": 4.411199152469635, "l_mix": 2.7338171303272247, "l_a": 2.8211827278137207, "l_c": 2.7700901329517365, "z_loss": 39.391921043395996, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1458462029695511, "tokens_per_second": 1445711.165053369, "eta_seconds": 7230.890017795563, "data_wait_seconds": 1.8775463104248047e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 52550, "total_steps": 72479, "tokens": 27551334400, "loss": 4.447774887084961, "l_mix": 2.7568125426769257, "l_a": 2.8418976962566376, "l_c": 2.794642925262451, "z_loss": 39.62645864486694, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11804629117250443, "tokens_per_second": 1443965.6666497034, "eta_seconds": 7235.999991774559, "data_wait_seconds": 1.761317253112793e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 52560, "total_steps": 72479, "tokens": 27556577280, "loss": 4.737569868564606, "l_mix": 2.939990818500519, "l_a": 3.0133793652057648, "l_c": 2.978550463914871, "z_loss": 38.55187129974365, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13151565194129944, "tokens_per_second": 1465630.2383660083, "eta_seconds": 7125.462069916724, "data_wait_seconds": 1.659989356994629e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 52570, "total_steps": 72479, "tokens": 27561820160, "loss": 4.6711366176605225, "l_mix": 2.8982695937156677, "l_a": 2.9760226905345917, "l_c": 2.9335336685180664, "z_loss": 40.519211769104004, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.29931506514549255, "tokens_per_second": 1456617.8906104118, "eta_seconds": 7165.949189066887, "data_wait_seconds": 2.6047229766845703e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 52580, "total_steps": 72479, "tokens": 27567063040, "loss": 3.1200483441352844, "l_mix": 1.9184680581092834, "l_a": 2.035253018140793, "l_c": 1.9700145721435547, "z_loss": 43.027416706085205, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2319723218679428, "tokens_per_second": 1441006.2635941515, "eta_seconds": 7239.945568299293, "data_wait_seconds": 2.3931264877319336e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 52590, "total_steps": 72479, "tokens": 27572305920, "loss": 3.1864932477474213, "l_mix": 1.9618526250123978, "l_a": 2.0719098150730133, "l_c": 2.010225370526314, "z_loss": 42.33184623718262, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1879938542842865, "tokens_per_second": 1354766.8378222012, "eta_seconds": 7696.943666529655, "data_wait_seconds": 1.6748905181884766e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 52600, "total_steps": 72479, "tokens": 27577548800, "loss": 3.13379630446434, "l_mix": 1.9285201877355576, "l_a": 2.039801761507988, "l_c": 1.9777852296829224, "z_loss": 42.857874393463135, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18468689918518066, "tokens_per_second": 1364069.411821847, "eta_seconds": 7640.609093403817, "data_wait_seconds": 2.0563602447509766e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.17445773, "gate_a2b/L1_absmax": 0.70595491, "gate_a2b/L1_rms": 0.1855779, "gate_a2b/L3_absmean": 0.16751973, "gate_a2b/L3_absmax": 0.53668863, "gate_a2b/L3_rms": 0.17352392, "gate_a2b/L5_absmean": 0.25243098, "gate_a2b/L5_absmax": 0.73351765, "gate_a2b/L5_rms": 0.25841525, "gate_c2b/L1_absmean": 0.02511519, "gate_c2b/L1_absmax": 0.62899441, "gate_c2b/L1_rms": 0.06638637, "gate_c2b/L3_absmean": 0.17907137, "gate_c2b/L3_absmax": 0.4061451, "gate_c2b/L3_rms": 0.18287058, "gate_c2b/L5_absmean": 0.50873834, "gate_c2b/L5_absmax": 0.71975601, "gate_c2b/L5_rms": 0.51909792, "gate_b2a/L7_absmean": 0.05093449, "gate_b2a/L7_absmax": 0.32339638, "gate_b2a/L7_rms": 0.05653104, "gate_b2a/L11_absmean": 0.08199746, "gate_b2a/L11_absmax": 0.60377783, "gate_b2a/L11_rms": 0.09113679, "gate_b2a/L14_absmean": 0.21128649, "gate_b2a/L14_absmax": 0.80258125, "gate_b2a/L14_rms": 0.22023486, "gate_b2c/L3_absmean": 0.03528177, "gate_b2c/L3_absmax": 0.560996, "gate_b2c/L3_rms": 0.04624078, "gate_b2c/L5_absmean": 0.03782145, "gate_b2c/L5_absmax": 0.55999035, "gate_b2c/L5_rms": 0.04774442, "gate_b2c/L6_absmean": 0.05448137, "gate_b2c/L6_absmax": 0.35458893, "gate_b2c/L6_rms": 0.06076064}, "streams": {"cos_a2b/round0": 0.355217, "cos_c2b/round0": 0.24953, "cos_b2a/round0": 0.143989, "cos_b2c/round0": 0.116546, "rms_a/round0": 6.919114, "rms_b/round0": 11.984743, "rms_c/round0": 4.600898, "cos_a2b/round1": 0.411277, "cos_c2b/round1": 0.302525, "cos_b2a/round1": 0.240631, "cos_b2c/round1": 0.117691, "rms_a/round1": 13.728884, "rms_b/round1": 28.544283, "rms_c/round1": 9.958257, "cos_a2b/round2": 0.521024, "cos_c2b/round2": 0.508471, "cos_b2a/round2": 0.088414, "cos_b2c/round2": 0.1689, "rms_a/round2": 37.132233, "rms_b/round2": 74.411636, "rms_c/round2": 19.402117}, "alpha_mean": 0.4181710918313968, "alpha_hist": [0.03012, 0.15729, 0.1929, 0.14954, 0.1187, 0.10384, 0.09962, 0.09084, 0.05365, 0.00351], "alpha_by_type": {"special": {"share": 0.00111, "mean": 0.66173, "std": 0.12028}, "whitespace": {"share": 0.02982, "mean": 0.63118, "std": 0.16881}, "digit": {"share": 0.00926, "mean": 0.42012, "std": 0.16749}, "word_start": {"share": 0.58026, "mean": 0.42414, "std": 0.21237}, "word_cont": {"share": 0.25923, "mean": 0.27766, "std": 0.14285}, "punct": {"share": 0.12032, "mean": 0.63693, "std": 0.18622}}, "alpha_p05": 0.1192, "alpha_p50": 0.3928, "alpha_p95": 0.81078}
{"step": 52610, "total_steps": 72479, "tokens": 27582791680, "loss": 3.0975834727287292, "l_mix": 1.9045556634664536, "l_a": 2.0221978276968002, "l_c": 1.954561322927475, "z_loss": 43.88569402694702, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19175659120082855, "tokens_per_second": 1418188.6057083325, "eta_seconds": 7345.340549254418, "data_wait_seconds": 1.7434358596801758e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 52620, "total_steps": 72479, "tokens": 27588034560, "loss": 3.1627952456474304, "l_mix": 1.9474278837442398, "l_a": 2.0590454638004303, "l_c": 1.9921791702508926, "z_loss": 43.259355545043945, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.28326934576034546, "tokens_per_second": 1353961.8052288357, "eta_seconds": 7689.9033279895775, "data_wait_seconds": 3.293156623840332e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 52630, "total_steps": 72479, "tokens": 27593277440, "loss": 4.538533985614777, "l_mix": 2.8135615289211273, "l_a": 2.89847594499588, "l_c": 2.8514317870140076, "z_loss": 39.469717502593994, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20164649188518524, "tokens_per_second": 1459209.652600959, "eta_seconds": 7131.663701272011, "data_wait_seconds": 2.0682811737060547e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 52640, "total_steps": 72479, "tokens": 27598520320, "loss": 4.445929229259491, "l_mix": 2.7554063200950623, "l_a": 2.8446521162986755, "l_c": 2.7904240787029266, "z_loss": 39.613380432128906, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.10901060700416565, "tokens_per_second": 1264367.440928964, "eta_seconds": 8226.524422645569, "data_wait_seconds": 1.946091651916504e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 52650, "total_steps": 72479, "tokens": 27603763200, "loss": 4.55305689573288, "l_mix": 2.8228273391723633, "l_a": 2.9057226181030273, "l_c": 2.861708641052246, "z_loss": 40.209389209747314, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12281997501850128, "tokens_per_second": 1436559.896500841, "eta_seconds": 7236.807025814056, "data_wait_seconds": 2.0563602447509766e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 52660, "total_steps": 72479, "tokens": 27609006080, "loss": 3.1069273948669434, "l_mix": 1.9125991612672806, "l_a": 2.023277387022972, "l_c": 1.9578165411949158, "z_loss": 41.62186241149902, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17549049854278564, "tokens_per_second": 1463373.0422198954, "eta_seconds": 7100.625453805924, "data_wait_seconds": 3.0100345611572266e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 52670, "total_steps": 72479, "tokens": 27614248960, "loss": 3.1314826011657715, "l_mix": 1.925507813692093, "l_a": 2.045052617788315, "l_c": 1.9748631864786148, "z_loss": 43.906375885009766, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.21866801381111145, "tokens_per_second": 1332238.9110773555, "eta_seconds": 7795.614514517784, "data_wait_seconds": 3.0159950256347656e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 52680, "total_steps": 72479, "tokens": 27619491840, "loss": 3.144235610961914, "l_mix": 1.9348892569541931, "l_a": 2.04858560860157, "l_c": 1.9825686067342758, "z_loss": 43.131351947784424, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17651578783988953, "tokens_per_second": 1451641.7343522399, "eta_seconds": 7150.785118913651, "data_wait_seconds": 2.09808349609375e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 52690, "total_steps": 72479, "tokens": 27624734720, "loss": 3.123104453086853, "l_mix": 1.9222362190485, "l_a": 2.032319724559784, "l_c": 1.970574215054512, "z_loss": 40.93288278579712, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20453758537769318, "tokens_per_second": 1476557.5338701527, "eta_seconds": 7026.570244646073, "data_wait_seconds": 2.568960189819336e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 52700, "total_steps": 72479, "tokens": 27629977600, "loss": 3.1390810310840607, "l_mix": 1.9309886991977692, "l_a": 2.047763839364052, "l_c": 1.9792104363441467, "z_loss": 43.07542705535889, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17863816022872925, "tokens_per_second": 1451472.235723463, "eta_seconds": 7144.395942807198, "data_wait_seconds": 1.946091651916504e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.17423984, "gate_a2b/L1_absmax": 0.704454, "gate_a2b/L1_rms": 0.18529923, "gate_a2b/L3_absmean": 0.16733156, "gate_a2b/L3_absmax": 0.53580922, "gate_a2b/L3_rms": 0.17333767, "gate_a2b/L5_absmean": 0.25208351, "gate_a2b/L5_absmax": 0.73364723, "gate_a2b/L5_rms": 0.25805232, "gate_c2b/L1_absmean": 0.02541896, "gate_c2b/L1_absmax": 0.62817568, "gate_c2b/L1_rms": 0.06655469, "gate_c2b/L3_absmean": 0.17904393, "gate_c2b/L3_absmax": 0.40562731, "gate_c2b/L3_rms": 0.18285541, "gate_c2b/L5_absmean": 0.50898045, "gate_c2b/L5_absmax": 0.72410715, "gate_c2b/L5_rms": 0.51942754, "gate_b2a/L7_absmean": 0.05061215, "gate_b2a/L7_absmax": 0.32352448, "gate_b2a/L7_rms": 0.05625628, "gate_b2a/L11_absmean": 0.08188608, "gate_b2a/L11_absmax": 0.60567695, "gate_b2a/L11_rms": 0.09105887, "gate_b2a/L14_absmean": 0.21175286, "gate_b2a/L14_absmax": 0.80263269, "gate_b2a/L14_rms": 0.22066936, "gate_b2c/L3_absmean": 0.03547134, "gate_b2c/L3_absmax": 0.56028235, "gate_b2c/L3_rms": 0.04631868, "gate_b2c/L5_absmean": 0.03792756, "gate_b2c/L5_absmax": 0.55683827, "gate_b2c/L5_rms": 0.04772274, "gate_b2c/L6_absmean": 0.05432029, "gate_b2c/L6_absmax": 0.35484469, "gate_b2c/L6_rms": 0.06062224}, "streams": {"cos_a2b/round0": 0.355246, "cos_c2b/round0": 0.249171, "cos_b2a/round0": 0.143568, "cos_b2c/round0": 0.11646, "rms_a/round0": 6.926706, "rms_b/round0": 11.92824, "rms_c/round0": 4.591387, "cos_a2b/round1": 0.411287, "cos_c2b/round1": 0.302733, "cos_b2a/round1": 0.239534, "cos_b2c/round1": 0.119117, "rms_a/round1": 13.692312, "rms_b/round1": 28.632753, "rms_c/round1": 9.937486, "cos_a2b/round2": 0.517342, "cos_c2b/round2": 0.509434, "cos_b2a/round2": 0.089064, "cos_b2c/round2": 0.170205, "rms_a/round2": 36.957253, "rms_b/round2": 74.566307, "rms_c/round2": 19.191549}, "alpha_mean": 0.4266581441880106, "alpha_hist": [0.03023, 0.1496, 0.18863, 0.14926, 0.11395, 0.10522, 0.10152, 0.09668, 0.05904, 0.00587], "alpha_by_type": {"special": {"share": 0.00108, "mean": 0.63345, "std": 0.12097}, "whitespace": {"share": 0.03937, "mean": 0.63216, "std": 0.17081}, "digit": {"share": 0.01131, "mean": 0.38457, "std": 0.18943}, "word_start": {"share": 0.57399, "mean": 0.43345, "std": 0.2157}, "word_cont": {"share": 0.25961, "mean": 0.28358, "std": 0.14782}, "punct": {"share": 0.11464, "mean": 0.64828, "std": 0.18812}}, "alpha_p05": 0.12464, "alpha_p50": 0.41206, "alpha_p95": 0.84312}
{"step": 52710, "total_steps": 72479, "tokens": 27635220480, "loss": 3.0892873108386993, "l_mix": 1.9011317044496536, "l_a": 2.0129891633987427, "l_c": 1.947529599070549, "z_loss": 42.76565074920654, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15864212810993195, "tokens_per_second": 1442009.9790441904, "eta_seconds": 7187.640600705146, "data_wait_seconds": 1.9609928131103516e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 52720, "total_steps": 72479, "tokens": 27640463360, "loss": 3.11292228102684, "l_mix": 1.9156050384044647, "l_a": 2.026590123772621, "l_c": 1.964467167854309, "z_loss": 42.40743017196655, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19670391082763672, "tokens_per_second": 1283163.0571704658, "eta_seconds": 8073.336069107055, "data_wait_seconds": 1.8596649169921875e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 52730, "total_steps": 72479, "tokens": 27645706240, "loss": 3.1761548817157745, "l_mix": 1.9532667398452759, "l_a": 2.073457896709442, "l_c": 2.0028360038995743, "z_loss": 44.17361068725586, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.25490838289260864, "tokens_per_second": 1429059.3617130646, "eta_seconds": 7245.439895224572, "data_wait_seconds": 1.8328428268432617e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 52740, "total_steps": 72479, "tokens": 27650949120, "loss": 4.473754107952118, "l_mix": 2.772002637386322, "l_a": 2.8597847521305084, "l_c": 2.8127199709415436, "z_loss": 39.158376693725586, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17347238957881927, "tokens_per_second": 1423471.2689066736, "eta_seconds": 7270.20000898838, "data_wait_seconds": 1.6182661056518555e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 52750, "total_steps": 72479, "tokens": 27656192000, "loss": 4.458535194396973, "l_mix": 2.763395607471466, "l_a": 2.8513344526290894, "l_c": 2.7991306483745575, "z_loss": 39.23522663116455, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11023306101560593, "tokens_per_second": 1405843.1668126837, "eta_seconds": 7357.632911109924, "data_wait_seconds": 2.339482307434082e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 52760, "total_steps": 72479, "tokens": 27661434880, "loss": 4.466404676437378, "l_mix": 2.768316149711609, "l_a": 2.8568420112133026, "l_c": 2.8034528195858, "z_loss": 40.21570825576782, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2035364806652069, "tokens_per_second": 1410150.691862375, "eta_seconds": 7331.439917492867, "data_wait_seconds": 1.7076730728149414e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 52770, "total_steps": 72479, "tokens": 27666677760, "loss": 3.1192495822906494, "l_mix": 1.9191642850637436, "l_a": 2.0338186621665955, "l_c": 1.9664655327796936, "z_loss": 45.2952938079834, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2311374843120575, "tokens_per_second": 1436962.328756122, "eta_seconds": 7190.99727613926, "data_wait_seconds": 2.5928020477294922e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 52780, "total_steps": 72479, "tokens": 27671920640, "loss": 3.0810786485671997, "l_mix": 1.8956033736467361, "l_a": 2.0065972357988358, "l_c": 1.9449867010116577, "z_loss": 43.135419845581055, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18502052128314972, "tokens_per_second": 1467774.0927676815, "eta_seconds": 7036.470641422271, "data_wait_seconds": 1.6510486602783203e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 52790, "total_steps": 72479, "tokens": 27677163520, "loss": 3.16103059053421, "l_mix": 1.9465898424386978, "l_a": 2.057522103190422, "l_c": 1.9906134903430939, "z_loss": 42.25552749633789, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20867349207401276, "tokens_per_second": 1327457.8434283957, "eta_seconds": 7776.296989846229, "data_wait_seconds": 2.3365020751953125e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 52800, "total_steps": 72479, "tokens": 27682406400, "loss": 3.1354081332683563, "l_mix": 1.9290031790733337, "l_a": 2.0417279303073883, "l_c": 1.9796217232942581, "z_loss": 43.46780490875244, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1691848784685135, "tokens_per_second": 1421629.5746715483, "eta_seconds": 7257.4907949447625, "data_wait_seconds": 3.5315752029418945e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.17455934, "gate_a2b/L1_absmax": 0.7041837, "gate_a2b/L1_rms": 0.18563254, "gate_a2b/L3_absmean": 0.16726004, "gate_a2b/L3_absmax": 0.53454792, "gate_a2b/L3_rms": 0.17319235, "gate_a2b/L5_absmean": 0.25201014, "gate_a2b/L5_absmax": 0.73359853, "gate_a2b/L5_rms": 0.25801721, "gate_c2b/L1_absmean": 0.0256091, "gate_c2b/L1_absmax": 0.62838161, "gate_c2b/L1_rms": 0.06657463, "gate_c2b/L3_absmean": 0.17865478, "gate_c2b/L3_absmax": 0.40598026, "gate_c2b/L3_rms": 0.18246272, "gate_c2b/L5_absmean": 0.50814676, "gate_c2b/L5_absmax": 0.71421403, "gate_c2b/L5_rms": 0.51857835, "gate_b2a/L7_absmean": 0.05050812, "gate_b2a/L7_absmax": 0.32309961, "gate_b2a/L7_rms": 0.05613127, "gate_b2a/L11_absmean": 0.08192423, "gate_b2a/L11_absmax": 0.60576189, "gate_b2a/L11_rms": 0.09109915, "gate_b2a/L14_absmean": 0.21170029, "gate_b2a/L14_absmax": 0.80354154, "gate_b2a/L14_rms": 0.22059914, "gate_b2c/L3_absmean": 0.03550985, "gate_b2c/L3_absmax": 0.56117785, "gate_b2c/L3_rms": 0.04638568, "gate_b2c/L5_absmean": 0.03752431, "gate_b2c/L5_absmax": 0.55888599, "gate_b2c/L5_rms": 0.04750693, "gate_b2c/L6_absmean": 0.05431247, "gate_b2c/L6_absmax": 0.35497946, "gate_b2c/L6_rms": 0.06061774}, "streams": {"cos_a2b/round0": 0.35588, "cos_c2b/round0": 0.248531, "cos_b2a/round0": 0.139495, "cos_b2c/round0": 0.117158, "rms_a/round0": 6.941901, "rms_b/round0": 11.995481, "rms_c/round0": 4.56822, "cos_a2b/round1": 0.408183, "cos_c2b/round1": 0.302048, "cos_b2a/round1": 0.235861, "cos_b2c/round1": 0.118374, "rms_a/round1": 13.765985, "rms_b/round1": 28.596142, "rms_c/round1": 9.962534, "cos_a2b/round2": 0.517613, "cos_c2b/round2": 0.510376, "cos_b2a/round2": 0.090771, "cos_b2c/round2": 0.170434, "rms_a/round2": 37.210491, "rms_b/round2": 74.802368, "rms_c/round2": 19.342781}, "alpha_mean": 0.4203002618325513, "alpha_hist": [0.03343, 0.1541, 0.18672, 0.14606, 0.11919, 0.11107, 0.10422, 0.08704, 0.05299, 0.00519], "alpha_by_type": {"special": {"share": 0.0011, "mean": 0.62963, "std": 0.1139}, "whitespace": {"share": 0.03758, "mean": 0.62553, "std": 0.17082}, "digit": {"share": 0.01094, "mean": 0.38522, "std": 0.16087}, "word_start": {"share": 0.55835, "mean": 0.42457, "std": 0.21295}, "word_cont": {"share": 0.26379, "mean": 0.27885, "std": 0.1437}, "punct": {"share": 0.12823, "mean": 0.63372, "std": 0.18388}}, "alpha_p05": 0.11941, "alpha_p50": 0.39911, "alpha_p95": 0.81405}
{"step": 52810, "total_steps": 72479, "tokens": 27687649280, "loss": 4.5001532435417175, "l_mix": 2.791562020778656, "l_a": 2.8673042356967926, "l_c": 2.827999919652939, "z_loss": 41.00188493728638, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2374102771282196, "tokens_per_second": 1480509.7932781796, "eta_seconds": 6965.317432427406, "data_wait_seconds": 1.531839370727539e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 52820, "total_steps": 72479, "tokens": 27692892160, "loss": 4.670307159423828, "l_mix": 2.896628260612488, "l_a": 2.9734369218349457, "l_c": 2.9388259053230286, "z_loss": 39.94595718383789, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16618327796459198, "tokens_per_second": 1358396.7897154822, "eta_seconds": 7587.6046454429625, "data_wait_seconds": 1.58846378326416e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 52830, "total_steps": 72479, "tokens": 27698135040, "loss": 4.679277420043945, "l_mix": 2.903631567955017, "l_a": 2.977573186159134, "l_c": 2.9412459433078766, "z_loss": 38.756468772888184, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14963486790657043, "tokens_per_second": 1449391.1801175424, "eta_seconds": 7107.62908821106, "data_wait_seconds": 3.93986701965332e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 52840, "total_steps": 72479, "tokens": 27703377920, "loss": 3.1649014055728912, "l_mix": 1.9493456035852432, "l_a": 2.0578834116458893, "l_c": 1.9939692914485931, "z_loss": 41.907073974609375, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20688827335834503, "tokens_per_second": 1442421.5263111566, "eta_seconds": 7138.337749528884, "data_wait_seconds": 3.090500831604004e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 52850, "total_steps": 72479, "tokens": 27708620800, "loss": 4.505808711051941, "l_mix": 2.792826771736145, "l_a": 2.8801684081554413, "l_c": 2.8297708928585052, "z_loss": 39.70848035812378, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16497711837291718, "tokens_per_second": 1226075.2706087888, "eta_seconds": 8393.65200383663, "data_wait_seconds": 2.2083520889282227e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 52860, "total_steps": 72479, "tokens": 27713863680, "loss": 4.403774321079254, "l_mix": 2.7292101979255676, "l_a": 2.8155684173107147, "l_c": 2.7663120329380035, "z_loss": 39.04742956161499, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16389459371566772, "tokens_per_second": 1408248.1325873048, "eta_seconds": 7304.114973759652, "data_wait_seconds": 2.2023916244506836e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 52870, "total_steps": 72479, "tokens": 27719106560, "loss": 4.507320582866669, "l_mix": 2.7935694456100464, "l_a": 2.881045639514923, "l_c": 2.831458032131195, "z_loss": 39.72917556762695, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14106081426143646, "tokens_per_second": 1409937.2245086173, "eta_seconds": 7291.646190547944, "data_wait_seconds": 1.9788742065429688e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 52880, "total_steps": 72479, "tokens": 27724349440, "loss": 3.119824230670929, "l_mix": 1.9200441539287567, "l_a": 2.032381847500801, "l_c": 1.9668850004673004, "z_loss": 42.130929470062256, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1803685426712036, "tokens_per_second": 1426781.2082068315, "eta_seconds": 7201.889436793327, "data_wait_seconds": 1.9848346710205078e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 52890, "total_steps": 72479, "tokens": 27729592320, "loss": 3.0950236916542053, "l_mix": 1.9040669202804565, "l_a": 2.018964722752571, "l_c": 1.95089091360569, "z_loss": 43.746009826660156, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1991073191165924, "tokens_per_second": 1307550.728627465, "eta_seconds": 7854.592106556893, "data_wait_seconds": 1.659989356994629e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 52900, "total_steps": 72479, "tokens": 27734835200, "loss": 3.1411462128162384, "l_mix": 1.9332647919654846, "l_a": 2.0446331799030304, "l_c": 1.9816379249095917, "z_loss": 41.3498010635376, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16819186508655548, "tokens_per_second": 1441529.4915538672, "eta_seconds": 7120.932878684997, "data_wait_seconds": 1.9997358322143555e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.17375837, "gate_a2b/L1_absmax": 0.70336801, "gate_a2b/L1_rms": 0.18477425, "gate_a2b/L3_absmean": 0.16749214, "gate_a2b/L3_absmax": 0.53685218, "gate_a2b/L3_rms": 0.17349121, "gate_a2b/L5_absmean": 0.25118473, "gate_a2b/L5_absmax": 0.73295045, "gate_a2b/L5_rms": 0.25719509, "gate_c2b/L1_absmean": 0.02534454, "gate_c2b/L1_absmax": 0.62819809, "gate_c2b/L1_rms": 0.06657887, "gate_c2b/L3_absmean": 0.17867401, "gate_c2b/L3_absmax": 0.40692997, "gate_c2b/L3_rms": 0.18250705, "gate_c2b/L5_absmean": 0.50766945, "gate_c2b/L5_absmax": 0.71724677, "gate_c2b/L5_rms": 0.51815337, "gate_b2a/L7_absmean": 0.05052423, "gate_b2a/L7_absmax": 0.32416815, "gate_b2a/L7_rms": 0.0562146, "gate_b2a/L11_absmean": 0.08201206, "gate_b2a/L11_absmax": 0.60649812, "gate_b2a/L11_rms": 0.0911788, "gate_b2a/L14_absmean": 0.21150406, "gate_b2a/L14_absmax": 0.80261648, "gate_b2a/L14_rms": 0.2204504, "gate_b2c/L3_absmean": 0.03530134, "gate_b2c/L3_absmax": 0.56100875, "gate_b2c/L3_rms": 0.04621643, "gate_b2c/L5_absmean": 0.03768857, "gate_b2c/L5_absmax": 0.55792797, "gate_b2c/L5_rms": 0.04766982, "gate_b2c/L6_absmean": 0.0543819, "gate_b2c/L6_absmax": 0.35573548, "gate_b2c/L6_rms": 0.06070554}, "streams": {"cos_a2b/round0": 0.356387, "cos_c2b/round0": 0.249177, "cos_b2a/round0": 0.142698, "cos_b2c/round0": 0.116764, "rms_a/round0": 6.966249, "rms_b/round0": 11.978843, "rms_c/round0": 4.60356, "cos_a2b/round1": 0.408459, "cos_c2b/round1": 0.302289, "cos_b2a/round1": 0.237355, "cos_b2c/round1": 0.118085, "rms_a/round1": 13.678794, "rms_b/round1": 28.658159, "rms_c/round1": 10.003554, "cos_a2b/round2": 0.512228, "cos_c2b/round2": 0.516666, "cos_b2a/round2": 0.088595, "cos_b2c/round2": 0.171215, "rms_a/round2": 36.879131, "rms_b/round2": 74.735771, "rms_c/round2": 19.398659}, "alpha_mean": 0.40872496638523614, "alpha_hist": [0.03313, 0.15451, 0.19943, 0.15404, 0.12357, 0.10892, 0.10025, 0.08151, 0.0415, 0.00314], "alpha_by_type": {"special": {"share": 0.00105, "mean": 0.65827, "std": 0.12163}, "whitespace": {"share": 0.03534, "mean": 0.61685, "std": 0.1756}, "digit": {"share": 0.00955, "mean": 0.40525, "std": 0.17212}, "word_start": {"share": 0.56879, "mean": 0.41201, "std": 0.20367}, "word_cont": {"share": 0.26379, "mean": 0.27621, "std": 0.14265}, "punct": {"share": 0.12148, "mean": 0.61866, "std": 0.18303}}, "alpha_p05": 0.10819, "alpha_p50": 0.37777, "alpha_p95": 0.80162}
{"step": 52910, "total_steps": 72479, "tokens": 27740078080, "loss": 3.1277171671390533, "l_mix": 1.9249275922775269, "l_a": 2.03903591632843, "l_c": 1.970262348651886, "z_loss": 43.826236724853516, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1933337301015854, "tokens_per_second": 1416339.2002811264, "eta_seconds": 7243.880470132828, "data_wait_seconds": 2.899765968322754e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 52920, "total_steps": 72479, "tokens": 27745320960, "loss": 4.506208777427673, "l_mix": 2.7924039363861084, "l_a": 2.876476436853409, "l_c": 2.8362060487270355, "z_loss": 37.75220966339111, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.9647440910339355, "tokens_per_second": 1359903.7549285167, "eta_seconds": 7540.643192458153, "data_wait_seconds": 2.2083520889282227e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 52930, "total_steps": 72479, "tokens": 27750563840, "loss": 4.452260732650757, "l_mix": 2.75950488448143, "l_a": 2.8455242216587067, "l_c": 2.796995311975479, "z_loss": 39.54595947265625, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1340385377407074, "tokens_per_second": 1401529.7197652159, "eta_seconds": 7312.942399621011, "data_wait_seconds": 2.518296241760254e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 52940, "total_steps": 72479, "tokens": 27755806720, "loss": 4.423866450786591, "l_mix": 2.740896314382553, "l_a": 2.8314845860004425, "l_c": 2.778415709733963, "z_loss": 40.28992557525635, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14533206820487976, "tokens_per_second": 1400884.284074159, "eta_seconds": 7312.569173955917, "data_wait_seconds": 2.0802021026611328e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 52950, "total_steps": 72479, "tokens": 27761049600, "loss": 4.43668407201767, "l_mix": 2.7498659193515778, "l_a": 2.834742158651352, "l_c": 2.7879848182201385, "z_loss": 40.263850688934326, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15467526018619537, "tokens_per_second": 1448949.7732717388, "eta_seconds": 7066.373549222946, "data_wait_seconds": 1.8984079360961914e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 52960, "total_steps": 72479, "tokens": 27766292480, "loss": 4.4365779757499695, "l_mix": 2.7493850886821747, "l_a": 2.8376695215702057, "l_c": 2.786306381225586, "z_loss": 40.57368755340576, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14425726234912872, "tokens_per_second": 1414485.9684681671, "eta_seconds": 7234.838450241089, "data_wait_seconds": 3.546476364135742e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 52970, "total_steps": 72479, "tokens": 27771535360, "loss": 4.45842319726944, "l_mix": 2.763303816318512, "l_a": 2.8486257195472717, "l_c": 2.801772564649582, "z_loss": 39.408509731292725, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2463986575603485, "tokens_per_second": 1418665.0970672134, "eta_seconds": 7209.83029267788, "data_wait_seconds": 2.2590160369873047e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 52980, "total_steps": 72479, "tokens": 27776778240, "loss": 4.41938704252243, "l_mix": 2.7393800020217896, "l_a": 2.8258036971092224, "l_c": 2.7742197811603546, "z_loss": 40.24056339263916, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.21629203855991364, "tokens_per_second": 1438527.8262909995, "eta_seconds": 7106.634661602974, "data_wait_seconds": 1.5944242477416992e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 52990, "total_steps": 72479, "tokens": 27782021120, "loss": 3.098445951938629, "l_mix": 1.9060376286506653, "l_a": 2.0187607556581497, "l_c": 1.9559334814548492, "z_loss": 44.449190616607666, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15956130623817444, "tokens_per_second": 1458851.5699252868, "eta_seconds": 7004.035943508148, "data_wait_seconds": 1.6361474990844727e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 53000, "total_steps": 72479, "tokens": 27787264000, "loss": 3.1269558370113373, "l_mix": 1.9246934354305267, "l_a": 2.0357051342725754, "l_c": 1.9718359857797623, "z_loss": 42.62060260772705, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15602809190750122, "tokens_per_second": 1369607.9764726383, "eta_seconds": 7456.590591931344, "data_wait_seconds": 3.501772880554199e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.17327711, "gate_a2b/L1_absmax": 0.70348346, "gate_a2b/L1_rms": 0.18437378, "gate_a2b/L3_absmean": 0.1668781, "gate_a2b/L3_absmax": 0.54065144, "gate_a2b/L3_rms": 0.17292213, "gate_a2b/L5_absmean": 0.25078747, "gate_a2b/L5_absmax": 0.73661369, "gate_a2b/L5_rms": 0.25677505, "gate_c2b/L1_absmean": 0.02566294, "gate_c2b/L1_absmax": 0.62658858, "gate_c2b/L1_rms": 0.06660623, "gate_c2b/L3_absmean": 0.17857547, "gate_c2b/L3_absmax": 0.4019708, "gate_c2b/L3_rms": 0.18237956, "gate_c2b/L5_absmean": 0.50816232, "gate_c2b/L5_absmax": 0.72014791, "gate_c2b/L5_rms": 0.51860982, "gate_b2a/L7_absmean": 0.05072001, "gate_b2a/L7_absmax": 0.3237862, "gate_b2a/L7_rms": 0.05635059, "gate_b2a/L11_absmean": 0.08204196, "gate_b2a/L11_absmax": 0.60674506, "gate_b2a/L11_rms": 0.09123277, "gate_b2a/L14_absmean": 0.21141644, "gate_b2a/L14_absmax": 0.80095315, "gate_b2a/L14_rms": 0.22035301, "gate_b2c/L3_absmean": 0.03541803, "gate_b2c/L3_absmax": 0.56115872, "gate_b2c/L3_rms": 0.04630484, "gate_b2c/L5_absmean": 0.03770126, "gate_b2c/L5_absmax": 0.56176662, "gate_b2c/L5_rms": 0.04768816, "gate_b2c/L6_absmean": 0.05419434, "gate_b2c/L6_absmax": 0.35341892, "gate_b2c/L6_rms": 0.06043973}, "streams": {"cos_a2b/round0": 0.356897, "cos_c2b/round0": 0.251145, "cos_b2a/round0": 0.143968, "cos_b2c/round0": 0.119144, "rms_a/round0": 6.96259, "rms_b/round0": 12.062234, "rms_c/round0": 4.594659, "cos_a2b/round1": 0.406983, "cos_c2b/round1": 0.30366, "cos_b2a/round1": 0.240494, "cos_b2c/round1": 0.115318, "rms_a/round1": 13.69127, "rms_b/round1": 28.677132, "rms_c/round1": 10.008395, "cos_a2b/round2": 0.516732, "cos_c2b/round2": 0.511127, "cos_b2a/round2": 0.088628, "cos_b2c/round2": 0.172013, "rms_a/round2": 37.210075, "rms_b/round2": 74.735878, "rms_c/round2": 19.297539}, "alpha_mean": 0.40551333963017555, "alpha_hist": [0.03297, 0.15726, 0.19324, 0.15637, 0.127, 0.11511, 0.10469, 0.0797, 0.0325, 0.00116], "alpha_by_type": {"special": {"share": 0.00101, "mean": 0.63447, "std": 0.10969}, "whitespace": {"share": 0.03888, "mean": 0.57822, "std": 0.1722}, "digit": {"share": 0.01218, "mean": 0.40114, "std": 0.17403}, "word_start": {"share": 0.55856, "mean": 0.40924, "std": 0.20159}, "word_cont": {"share": 0.26242, "mean": 0.27405, "std": 0.13659}, "punct": {"share": 0.12695, "mean": 0.60655, "std": 0.17521}}, "alpha_p05": 0.11596, "alpha_p50": 0.38191, "alpha_p95": 0.75707}
{"step": 53010, "total_steps": 72479, "tokens": 27792506880, "loss": 3.136036217212677, "l_mix": 1.930005058646202, "l_a": 2.0419827848672867, "l_c": 1.978120744228363, "z_loss": 43.10515546798706, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14757390320301056, "tokens_per_second": 1349126.7981430714, "eta_seconds": 7565.903431797028, "data_wait_seconds": 2.9653310775756836e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 53020, "total_steps": 72479, "tokens": 27797749760, "loss": 3.1547632813453674, "l_mix": 1.9399351328611374, "l_a": 2.056719496846199, "l_c": 1.9927075505256653, "z_loss": 42.931286334991455, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2203793078660965, "tokens_per_second": 1459355.6853858854, "eta_seconds": 6990.838692831992, "data_wait_seconds": 1.958012580871582e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 53030, "total_steps": 72479, "tokens": 27802992640, "loss": 4.46505331993103, "l_mix": 2.767396867275238, "l_a": 2.8526673018932343, "l_c": 2.806187331676483, "z_loss": 40.920477867126465, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14148789644241333, "tokens_per_second": 1455257.5606924994, "eta_seconds": 7006.922751975059, "data_wait_seconds": 1.5556812286376953e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 53040, "total_steps": 72479, "tokens": 27808235520, "loss": 4.467629075050354, "l_mix": 2.7684094607830048, "l_a": 2.8553826212882996, "l_c": 2.808682769536972, "z_loss": 40.17756462097168, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18581631779670715, "tokens_per_second": 1411744.6511249284, "eta_seconds": 7219.176941013337, "data_wait_seconds": 2.8759241104125977e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 53050, "total_steps": 72479, "tokens": 27813478400, "loss": 4.423743009567261, "l_mix": 2.7427179515361786, "l_a": 2.827433258295059, "l_c": 2.775983303785324, "z_loss": 40.36547374725342, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.229559063911438, "tokens_per_second": 1467853.2559778176, "eta_seconds": 6939.652523517609, "data_wait_seconds": 2.1278858184814453e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 53060, "total_steps": 72479, "tokens": 27818721280, "loss": 4.586894333362579, "l_mix": 2.8429310619831085, "l_a": 2.927237808704376, "l_c": 2.885972708463669, "z_loss": 39.876070499420166, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.22889335453510284, "tokens_per_second": 1452585.5753449623, "eta_seconds": 7008.9837354898455, "data_wait_seconds": 1.7702579498291016e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 53070, "total_steps": 72479, "tokens": 27823964160, "loss": 4.386820018291473, "l_mix": 2.7188459634780884, "l_a": 2.8058994710445404, "l_c": 2.754013776779175, "z_loss": 40.124606132507324, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13689792156219482, "tokens_per_second": 1462129.553250009, "eta_seconds": 6959.647159433365, "data_wait_seconds": 1.6629695892333984e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 53080, "total_steps": 72479, "tokens": 27829207040, "loss": 4.414013504981995, "l_mix": 2.7358613312244415, "l_a": 2.82243075966835, "l_c": 2.7714098393917084, "z_loss": 40.58931493759155, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15637509524822235, "tokens_per_second": 1455305.907527836, "eta_seconds": 6988.676991820335, "data_wait_seconds": 2.396106719970703e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 53090, "total_steps": 72479, "tokens": 27834449920, "loss": 4.685873746871948, "l_mix": 2.9077146351337433, "l_a": 2.9793164432048798, "l_c": 2.9478805363178253, "z_loss": 37.67523193359375, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2238527238368988, "tokens_per_second": 1404312.359160423, "eta_seconds": 7238.717202544211, "data_wait_seconds": 2.1666288375854492e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 53100, "total_steps": 72479, "tokens": 27839692800, "loss": 4.648551344871521, "l_mix": 2.8838393688201904, "l_a": 2.9612467885017395, "l_c": 2.921126365661621, "z_loss": 38.50119400024414, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.24197234213352203, "tokens_per_second": 1432887.0593630555, "eta_seconds": 7090.703405833245, "data_wait_seconds": 1.6629695892333984e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.17381816, "gate_a2b/L1_absmax": 0.70347917, "gate_a2b/L1_rms": 0.18470126, "gate_a2b/L3_absmean": 0.1664965, "gate_a2b/L3_absmax": 0.53871471, "gate_a2b/L3_rms": 0.17250206, "gate_a2b/L5_absmean": 0.25032529, "gate_a2b/L5_absmax": 0.73436624, "gate_a2b/L5_rms": 0.25627175, "gate_c2b/L1_absmean": 0.02605589, "gate_c2b/L1_absmax": 0.62562096, "gate_c2b/L1_rms": 0.066781, "gate_c2b/L3_absmean": 0.17853586, "gate_c2b/L3_absmax": 0.40065768, "gate_c2b/L3_rms": 0.18230942, "gate_c2b/L5_absmean": 0.50738347, "gate_c2b/L5_absmax": 0.71486604, "gate_c2b/L5_rms": 0.51784796, "gate_b2a/L7_absmean": 0.05053686, "gate_b2a/L7_absmax": 0.32388759, "gate_b2a/L7_rms": 0.05620744, "gate_b2a/L11_absmean": 0.08189756, "gate_b2a/L11_absmax": 0.60735744, "gate_b2a/L11_rms": 0.09111356, "gate_b2a/L14_absmean": 0.211291, "gate_b2a/L14_absmax": 0.80098438, "gate_b2a/L14_rms": 0.22021993, "gate_b2c/L3_absmean": 0.0354277, "gate_b2c/L3_absmax": 0.56034636, "gate_b2c/L3_rms": 0.04630341, "gate_b2c/L5_absmean": 0.03754507, "gate_b2c/L5_absmax": 0.56235272, "gate_b2c/L5_rms": 0.04760844, "gate_b2c/L6_absmean": 0.05413286, "gate_b2c/L6_absmax": 0.35257831, "gate_b2c/L6_rms": 0.06039072}, "streams": {"cos_a2b/round0": 0.358457, "cos_c2b/round0": 0.282285, "cos_b2a/round0": 0.152568, "cos_b2c/round0": 0.137374, "rms_a/round0": 6.702454, "rms_b/round0": 11.854793, "rms_c/round0": 4.484746, "cos_a2b/round1": 0.404989, "cos_c2b/round1": 0.300327, "cos_b2a/round1": 0.249951, "cos_b2c/round1": 0.112458, "rms_a/round1": 12.941329, "rms_b/round1": 27.111929, "rms_c/round1": 9.485703, "cos_a2b/round2": 0.538247, "cos_c2b/round2": 0.518131, "cos_b2a/round2": 0.093838, "cos_b2c/round2": 0.180709, "rms_a/round2": 36.092609, "rms_b/round2": 72.440742, "rms_c/round2": 18.193146}, "alpha_mean": 0.4341490619490571, "alpha_hist": [0.00751, 0.05908, 0.17125, 0.21693, 0.19768, 0.16254, 0.11746, 0.05946, 0.00755, 0.00053], "alpha_by_type": {"special": {"share": 0.00078, "mean": 0.51288, "std": 0.18195}, "whitespace": {"share": 0.06821, "mean": 0.50642, "std": 0.11769}, "digit": {"share": 0.04774, "mean": 0.42106, "std": 0.12043}, "word_start": {"share": 0.50737, "mean": 0.45394, "std": 0.16325}, "word_cont": {"share": 0.26646, "mean": 0.32683, "std": 0.12104}, "punct": {"share": 0.10944, "mean": 0.56383, "std": 0.16051}}, "alpha_p05": 0.18803, "alpha_p50": 0.40993, "alpha_p95": 0.71463}
{"step": 53110, "total_steps": 72479, "tokens": 27844935680, "loss": 4.653223037719727, "l_mix": 2.886507421731949, "l_a": 2.9645060896873474, "l_c": 2.924545645713806, "z_loss": 39.95694828033447, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1485983282327652, "tokens_per_second": 1367552.6263088472, "eta_seconds": 7425.625951528549, "data_wait_seconds": 2.2619962692260742e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 53120, "total_steps": 72479, "tokens": 27850178560, "loss": 3.1391944885253906, "l_mix": 1.9320961683988571, "l_a": 2.0446535646915436, "l_c": 1.979007050395012, "z_loss": 42.796095848083496, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15481741726398468, "tokens_per_second": 1391509.1852568914, "eta_seconds": 7294.016812491417, "data_wait_seconds": 3.287196159362793e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 53130, "total_steps": 72479, "tokens": 27855421440, "loss": 3.0925933122634888, "l_mix": 1.9027761369943619, "l_a": 2.014683276414871, "l_c": 1.9513740539550781, "z_loss": 43.621323585510254, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.138454407453537, "tokens_per_second": 1368161.7479534252, "eta_seconds": 7414.655852770806, "data_wait_seconds": 2.8252601623535156e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 53140, "total_steps": 72479, "tokens": 27860664320, "loss": 4.440156579017639, "l_mix": 2.751416027545929, "l_a": 2.84134778380394, "l_c": 2.787786841392517, "z_loss": 40.77551317214966, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16292314231395721, "tokens_per_second": 1292637.1049655492, "eta_seconds": 7843.814472794533, "data_wait_seconds": 1.5556812286376953e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 53150, "total_steps": 72479, "tokens": 27865907200, "loss": 4.411154389381409, "l_mix": 2.734301060438156, "l_a": 2.8193475902080536, "l_c": 2.770163416862488, "z_loss": 39.93920850753784, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19179043173789978, "tokens_per_second": 1445864.7754981704, "eta_seconds": 7008.928444576263, "data_wait_seconds": 1.6450881958007812e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 53160, "total_steps": 72479, "tokens": 27871150080, "loss": 4.480809211730957, "l_mix": 2.777589291334152, "l_a": 2.8615754544734955, "l_c": 2.815823942422867, "z_loss": 39.87846899032593, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16159236431121826, "tokens_per_second": 1415244.7311665195, "eta_seconds": 7156.868101286887, "data_wait_seconds": 1.609325408935547e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 53170, "total_steps": 72479, "tokens": 27876392960, "loss": 4.435780346393585, "l_mix": 2.749444156885147, "l_a": 2.836113780736923, "l_c": 2.785006582736969, "z_loss": 40.32791614532471, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1476670205593109, "tokens_per_second": 1318838.0958650215, "eta_seconds": 7676.057450675965, "data_wait_seconds": 1.5616416931152344e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 53180, "total_steps": 72479, "tokens": 27881635840, "loss": 4.49053817987442, "l_mix": 2.783374011516571, "l_a": 2.8691364526748657, "l_c": 2.821410447359085, "z_loss": 40.24969434738159, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1435098499059677, "tokens_per_second": 1467724.4241725449, "eta_seconds": 6893.8241711854935, "data_wait_seconds": 1.6927719116210938e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 53190, "total_steps": 72479, "tokens": 27886878720, "loss": 4.5262293219566345, "l_mix": 2.8055860102176666, "l_a": 2.890198051929474, "l_c": 2.8452795445919037, "z_loss": 40.513136863708496, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1478259265422821, "tokens_per_second": 1419496.1542660347, "eta_seconds": 7124.352680778503, "data_wait_seconds": 2.4527311325073242e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 53200, "total_steps": 72479, "tokens": 27892121600, "loss": 4.649889647960663, "l_mix": 2.8846923410892487, "l_a": 2.962654769420624, "l_c": 2.921335995197296, "z_loss": 40.01586437225342, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1604505479335785, "tokens_per_second": 1404451.8260567447, "eta_seconds": 7196.934892654419, "data_wait_seconds": 2.142786979675293e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.17343485, "gate_a2b/L1_absmax": 0.70212954, "gate_a2b/L1_rms": 0.18438427, "gate_a2b/L3_absmean": 0.16617732, "gate_a2b/L3_absmax": 0.53842473, "gate_a2b/L3_rms": 0.17216371, "gate_a2b/L5_absmean": 0.24955039, "gate_a2b/L5_absmax": 0.73419207, "gate_a2b/L5_rms": 0.25560185, "gate_c2b/L1_absmean": 0.02572531, "gate_c2b/L1_absmax": 0.62481761, "gate_c2b/L1_rms": 0.06652404, "gate_c2b/L3_absmean": 0.17829742, "gate_c2b/L3_absmax": 0.39779758, "gate_c2b/L3_rms": 0.18202956, "gate_c2b/L5_absmean": 0.50748402, "gate_c2b/L5_absmax": 0.71809393, "gate_c2b/L5_rms": 0.5178262, "gate_b2a/L7_absmean": 0.050664, "gate_b2a/L7_absmax": 0.32434222, "gate_b2a/L7_rms": 0.05630359, "gate_b2a/L11_absmean": 0.08194363, "gate_b2a/L11_absmax": 0.60642743, "gate_b2a/L11_rms": 0.09118158, "gate_b2a/L14_absmean": 0.21156865, "gate_b2a/L14_absmax": 0.80212897, "gate_b2a/L14_rms": 0.22048253, "gate_b2c/L3_absmean": 0.03549203, "gate_b2c/L3_absmax": 0.56094468, "gate_b2c/L3_rms": 0.04637781, "gate_b2c/L5_absmean": 0.03785394, "gate_b2c/L5_absmax": 0.55760115, "gate_b2c/L5_rms": 0.04767458, "gate_b2c/L6_absmean": 0.05410033, "gate_b2c/L6_absmax": 0.35229173, "gate_b2c/L6_rms": 0.06037698}, "streams": {"cos_a2b/round0": 0.353593, "cos_c2b/round0": 0.273564, "cos_b2a/round0": 0.137198, "cos_b2c/round0": 0.130143, "rms_a/round0": 6.828413, "rms_b/round0": 12.041582, "rms_c/round0": 4.477169, "cos_a2b/round1": 0.412002, "cos_c2b/round1": 0.2941, "cos_b2a/round1": 0.237204, "cos_b2c/round1": 0.100222, "rms_a/round1": 13.036493, "rms_b/round1": 27.190277, "rms_c/round1": 9.423141, "cos_a2b/round2": 0.537342, "cos_c2b/round2": 0.515229, "cos_b2a/round2": 0.107038, "cos_b2c/round2": 0.176419, "rms_a/round2": 37.056076, "rms_b/round2": 74.8694, "rms_c/round2": 17.878544}, "alpha_mean": 0.46195961077465597, "alpha_hist": [0.00606, 0.05049, 0.13617, 0.18352, 0.18268, 0.20927, 0.16203, 0.0621, 0.00752, 0.00015], "alpha_by_type": {"special": {"share": 0.00018, "mean": 0.52063, "std": 0.09189}, "whitespace": {"share": 0.05533, "mean": 0.57902, "std": 0.13163}, "digit": {"share": 0.03313, "mean": 0.48292, "std": 0.15708}, "word_start": {"share": 0.46283, "mean": 0.47856, "std": 0.14926}, "word_cont": {"share": 0.32452, "mean": 0.35979, "std": 0.13482}, "punct": {"share": 0.12401, "mean": 0.60944, "std": 0.13391}}, "alpha_p05": 0.19346, "alpha_p50": 0.46733, "alpha_p95": 0.72213}
{"step": 53210, "total_steps": 72479, "tokens": 27897364480, "loss": 4.610674858093262, "l_mix": 2.8602309226989746, "l_a": 2.9337574243545532, "l_c": 2.901055246591568, "z_loss": 40.08986186981201, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1275632083415985, "tokens_per_second": 1334166.2565632216, "eta_seconds": 7572.148839998245, "data_wait_seconds": 2.1696090698242188e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 53220, "total_steps": 72479, "tokens": 27902607360, "loss": 4.723925709724426, "l_mix": 2.9319044947624207, "l_a": 3.0033479630947113, "l_c": 2.9700558185577393, "z_loss": 39.00916385650635, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14803367853164673, "tokens_per_second": 1201490.934249582, "eta_seconds": 8403.944053316116, "data_wait_seconds": 2.2709369659423828e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 53230, "total_steps": 72479, "tokens": 27907850240, "loss": 4.689879894256592, "l_mix": 2.9094553887844086, "l_a": 2.9883395433425903, "l_c": 2.946408450603485, "z_loss": 39.250479221343994, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17037451267242432, "tokens_per_second": 1390657.6214074267, "eta_seconds": 7257.0124785900125, "data_wait_seconds": 2.2202730178833008e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 53240, "total_steps": 72479, "tokens": 27913093120, "loss": 4.569754242897034, "l_mix": 2.835074633359909, "l_a": 2.9099121689796448, "l_c": 2.8723527789115906, "z_loss": 40.3174729347229, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1617812067270279, "tokens_per_second": 1245911.456775205, "eta_seconds": 8095.901821231842, "data_wait_seconds": 2.6792287826538086e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 53250, "total_steps": 72479, "tokens": 27918336000, "loss": 4.601124107837677, "l_mix": 2.854577362537384, "l_a": 2.9297701120376587, "l_c": 2.8920525312423706, "z_loss": 38.88728141784668, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15592332184314728, "tokens_per_second": 1455525.1458352597, "eta_seconds": 6926.38940718174, "data_wait_seconds": 3.17990779876709e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 53260, "total_steps": 72479, "tokens": 27923578880, "loss": 4.515117943286896, "l_mix": 2.798967272043228, "l_a": 2.886706680059433, "l_c": 2.8337953090667725, "z_loss": 38.459516525268555, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14153966307640076, "tokens_per_second": 1353078.7141076438, "eta_seconds": 7446.936358499527, "data_wait_seconds": 1.8715858459472656e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 53270, "total_steps": 72479, "tokens": 27928821760, "loss": 4.412636995315552, "l_mix": 2.734497606754303, "l_a": 2.822195589542389, "l_c": 2.7716020345687866, "z_loss": 39.366517543792725, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13212339580059052, "tokens_per_second": 1408830.6902941114, "eta_seconds": 7148.515617513656, "data_wait_seconds": 1.9848346710205078e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 53280, "total_steps": 72479, "tokens": 27934064640, "loss": 4.463960289955139, "l_mix": 2.7664358019828796, "l_a": 2.8554265797138214, "l_c": 2.8029879331588745, "z_loss": 39.87233018875122, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1647329479455948, "tokens_per_second": 1461951.1819520944, "eta_seconds": 6885.18565890789, "data_wait_seconds": 1.5437602996826172e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 53290, "total_steps": 72479, "tokens": 27939307520, "loss": 4.461463749408722, "l_mix": 2.7654586732387543, "l_a": 2.8523303270339966, "l_c": 2.801019936800003, "z_loss": 40.92346239089966, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.27486398816108704, "tokens_per_second": 1434870.3981550334, "eta_seconds": 7011.478141117095, "data_wait_seconds": 1.677870750427246e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 53300, "total_steps": 72479, "tokens": 27944550400, "loss": 4.405413925647736, "l_mix": 2.7287687063217163, "l_a": 2.8186800479888916, "l_c": 2.7701372504234314, "z_loss": 41.77964448928833, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1948995441198349, "tokens_per_second": 1462010.9582193734, "eta_seconds": 6877.7320002079005, "data_wait_seconds": 2.2172927856445312e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.17290473, "gate_a2b/L1_absmax": 0.70135176, "gate_a2b/L1_rms": 0.1839077, "gate_a2b/L3_absmean": 0.16632824, "gate_a2b/L3_absmax": 0.53788769, "gate_a2b/L3_rms": 0.1723092, "gate_a2b/L5_absmean": 0.25005871, "gate_a2b/L5_absmax": 0.73437291, "gate_a2b/L5_rms": 0.25605056, "gate_c2b/L1_absmean": 0.0262058, "gate_c2b/L1_absmax": 0.62381285, "gate_c2b/L1_rms": 0.06648702, "gate_c2b/L3_absmean": 0.17856714, "gate_c2b/L3_absmax": 0.40493092, "gate_c2b/L3_rms": 0.18229496, "gate_c2b/L5_absmean": 0.50660563, "gate_c2b/L5_absmax": 0.71945399, "gate_c2b/L5_rms": 0.51699531, "gate_b2a/L7_absmean": 0.05052815, "gate_b2a/L7_absmax": 0.32310307, "gate_b2a/L7_rms": 0.05616983, "gate_b2a/L11_absmean": 0.0822003, "gate_b2a/L11_absmax": 0.60640734, "gate_b2a/L11_rms": 0.09134721, "gate_b2a/L14_absmean": 0.21183956, "gate_b2a/L14_absmax": 0.800825, "gate_b2a/L14_rms": 0.22071509, "gate_b2c/L3_absmean": 0.0355586, "gate_b2c/L3_absmax": 0.56025851, "gate_b2c/L3_rms": 0.04636952, "gate_b2c/L5_absmean": 0.03779413, "gate_b2c/L5_absmax": 0.55845106, "gate_b2c/L5_rms": 0.04766943, "gate_b2c/L6_absmean": 0.05449834, "gate_b2c/L6_absmax": 0.35198769, "gate_b2c/L6_rms": 0.06069069}, "streams": {"cos_a2b/round0": 0.356781, "cos_c2b/round0": 0.276559, "cos_b2a/round0": 0.149585, "cos_b2c/round0": 0.135262, "rms_a/round0": 6.78171, "rms_b/round0": 11.994953, "rms_c/round0": 4.535693, "cos_a2b/round1": 0.408276, "cos_c2b/round1": 0.302443, "cos_b2a/round1": 0.245675, "cos_b2c/round1": 0.11053, "rms_a/round1": 13.231193, "rms_b/round1": 27.717699, "rms_c/round1": 9.624394, "cos_a2b/round2": 0.530003, "cos_c2b/round2": 0.52801, "cos_b2a/round2": 0.100014, "cos_b2c/round2": 0.179033, "rms_a/round2": 36.827976, "rms_b/round2": 74.207069, "rms_c/round2": 18.684673}, "alpha_mean": 0.42594871323188954, "alpha_hist": [0.01302, 0.08574, 0.18744, 0.19493, 0.16966, 0.1505, 0.11845, 0.0705, 0.0097, 6e-05], "alpha_by_type": {"special": {"share": 0.00134, "mean": 0.56736, "std": 0.13901}, "whitespace": {"share": 0.02858, "mean": 0.53672, "std": 0.11801}, "digit": {"share": 0.03389, "mean": 0.4205, "std": 0.12835}, "word_start": {"share": 0.56889, "mean": 0.44427, "std": 0.17383}, "word_cont": {"share": 0.26457, "mean": 0.31336, "std": 0.13165}, "punct": {"share": 0.10272, "mean": 0.58356, "std": 0.16325}}, "alpha_p05": 0.15304, "alpha_p50": 0.41182, "alpha_p95": 0.73259}
{"step": 53310, "total_steps": 72479, "tokens": 27949793280, "loss": 2.867679238319397, "l_mix": 1.7754593193531036, "l_a": 1.8385000824928284, "l_c": 1.802232712507248, "z_loss": 47.46145582199097, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.24544067680835724, "tokens_per_second": 1397749.333089932, "eta_seconds": 7190.185274338723, "data_wait_seconds": 1.8596649169921875e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 53320, "total_steps": 72479, "tokens": 27955036160, "loss": 4.679933786392212, "l_mix": 2.903992682695389, "l_a": 2.9789006412029266, "l_c": 2.94090273976326, "z_loss": 38.86610651016235, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11874602735042572, "tokens_per_second": 1364747.76237231, "eta_seconds": 7360.212684679031, "data_wait_seconds": 1.862645149230957e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 53330, "total_steps": 72479, "tokens": 27960279040, "loss": 4.605576395988464, "l_mix": 2.8560914397239685, "l_a": 2.937910318374634, "l_c": 2.8937058448791504, "z_loss": 39.113322257995605, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.21481089293956757, "tokens_per_second": 1481457.1263848767, "eta_seconds": 6776.835274672508, "data_wait_seconds": 1.519918441772461e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 53340, "total_steps": 72479, "tokens": 27965521920, "loss": 4.784054934978485, "l_mix": 2.968933880329132, "l_a": 3.044701963663101, "l_c": 3.0057011544704437, "z_loss": 38.1826810836792, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14250284433364868, "tokens_per_second": 893369.2251702553, "eta_seconds": 11232.027866292, "data_wait_seconds": 1.963973045349121e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 53350, "total_steps": 72479, "tokens": 27970764800, "loss": 4.655613660812378, "l_mix": 2.8882817924022675, "l_a": 2.9636974036693573, "l_c": 2.9274084866046906, "z_loss": 39.37520360946655, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17048165202140808, "tokens_per_second": 1410854.932189537, "eta_seconds": 7108.530383372307, "data_wait_seconds": 1.7881393432617188e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 53360, "total_steps": 72479, "tokens": 27976007680, "loss": 4.625362873077393, "l_mix": 2.868931919336319, "l_a": 2.9481259882450104, "l_c": 2.906643897294998, "z_loss": 41.111382484436035, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18004508316516876, "tokens_per_second": 1375586.8110907031, "eta_seconds": 7286.97177901268, "data_wait_seconds": 2.22623348236084e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 53370, "total_steps": 72479, "tokens": 27981250560, "loss": 4.454939663410187, "l_mix": 2.7614071369171143, "l_a": 2.847930073738098, "l_c": 2.797177702188492, "z_loss": 39.76072549819946, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1166868731379509, "tokens_per_second": 1291997.6343309341, "eta_seconds": 7754.3635729551315, "data_wait_seconds": 1.677870750427246e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 53380, "total_steps": 72479, "tokens": 27986493440, "loss": 4.377372205257416, "l_mix": 2.712847888469696, "l_a": 2.7988112568855286, "l_c": 2.749602824449539, "z_loss": 39.86109638214111, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12490443140268326, "tokens_per_second": 1426793.2428149977, "eta_seconds": 7018.099197220802, "data_wait_seconds": 1.659989356994629e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 53390, "total_steps": 72479, "tokens": 27991736320, "loss": 4.494884252548218, "l_mix": 2.78609699010849, "l_a": 2.8740255534648895, "l_c": 2.8219314515590668, "z_loss": 40.11668109893799, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17688345909118652, "tokens_per_second": 1444787.0548659144, "eta_seconds": 6927.064855885506, "data_wait_seconds": 1.677870750427246e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 53400, "total_steps": 72479, "tokens": 27996979200, "loss": 4.471083760261536, "l_mix": 2.7723390758037567, "l_a": 2.8529094755649567, "l_c": 2.809572398662567, "z_loss": 39.2521858215332, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1685282588005066, "tokens_per_second": 1438854.5344258829, "eta_seconds": 6951.981949996948, "data_wait_seconds": 1.901388168334961e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.17323755, "gate_a2b/L1_absmax": 0.70170921, "gate_a2b/L1_rms": 0.1841688, "gate_a2b/L3_absmean": 0.1661856, "gate_a2b/L3_absmax": 0.53815681, "gate_a2b/L3_rms": 0.17216574, "gate_a2b/L5_absmean": 0.24941531, "gate_a2b/L5_absmax": 0.73603606, "gate_a2b/L5_rms": 0.25541496, "gate_c2b/L1_absmean": 0.02589593, "gate_c2b/L1_absmax": 0.62401849, "gate_c2b/L1_rms": 0.06652422, "gate_c2b/L3_absmean": 0.17817876, "gate_c2b/L3_absmax": 0.40461653, "gate_c2b/L3_rms": 0.1819887, "gate_c2b/L5_absmean": 0.50617236, "gate_c2b/L5_absmax": 0.71752709, "gate_c2b/L5_rms": 0.51651859, "gate_b2a/L7_absmean": 0.05047548, "gate_b2a/L7_absmax": 0.32392341, "gate_b2a/L7_rms": 0.05613903, "gate_b2a/L11_absmean": 0.0819006, "gate_b2a/L11_absmax": 0.60666716, "gate_b2a/L11_rms": 0.09113999, "gate_b2a/L14_absmean": 0.21193574, "gate_b2a/L14_absmax": 0.80235636, "gate_b2a/L14_rms": 0.22083007, "gate_b2c/L3_absmean": 0.03526615, "gate_b2c/L3_absmax": 0.56085497, "gate_b2c/L3_rms": 0.04627514, "gate_b2c/L5_absmean": 0.03788845, "gate_b2c/L5_absmax": 0.56067312, "gate_b2c/L5_rms": 0.04780855, "gate_b2c/L6_absmean": 0.05401048, "gate_b2c/L6_absmax": 0.3521848, "gate_b2c/L6_rms": 0.06028881}, "streams": {"cos_a2b/round0": 0.358428, "cos_c2b/round0": 0.281971, "cos_b2a/round0": 0.156371, "cos_b2c/round0": 0.140431, "rms_a/round0": 6.740794, "rms_b/round0": 11.844634, "rms_c/round0": 4.502758, "cos_a2b/round1": 0.411678, "cos_c2b/round1": 0.302239, "cos_b2a/round1": 0.251783, "cos_b2c/round1": 0.112076, "rms_a/round1": 13.093983, "rms_b/round1": 27.406374, "rms_c/round1": 9.420566, "cos_a2b/round2": 0.544991, "cos_c2b/round2": 0.518747, "cos_b2a/round2": 0.095978, "cos_b2c/round2": 0.188122, "rms_a/round2": 36.772854, "rms_b/round2": 74.249596, "rms_c/round2": 18.105263}, "alpha_mean": 0.4346287541175684, "alpha_hist": [0.00925, 0.0589, 0.17015, 0.21211, 0.19572, 0.1669, 0.12424, 0.05719, 0.00551, 3e-05], "alpha_by_type": {"special": {"share": 0.00099, "mean": 0.53585, "std": 0.1084}, "whitespace": {"share": 0.02556, "mean": 0.51901, "std": 0.11122}, "digit": {"share": 0.04594, "mean": 0.45124, "std": 0.11657}, "word_start": {"share": 0.56032, "mean": 0.45094, "std": 0.15628}, "word_cont": {"share": 0.25563, "mean": 0.32165, "std": 0.12534}, "punct": {"share": 0.11156, "mean": 0.5845, "std": 0.1491}}, "alpha_p05": 0.19012, "alpha_p50": 0.41989, "alpha_p95": 0.70902}
{"step": 53410, "total_steps": 72479, "tokens": 28002222080, "loss": 4.409462571144104, "l_mix": 2.7329045832157135, "l_a": 2.8177471458911896, "l_c": 2.770779103040695, "z_loss": 40.696757316589355, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13526855409145355, "tokens_per_second": 1391398.0717109744, "eta_seconds": 7185.3253754377365, "data_wait_seconds": 2.1398067474365234e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 53420, "total_steps": 72479, "tokens": 28007464960, "loss": 3.754174590110779, "l_mix": 2.329968348145485, "l_a": 2.4087977558374405, "l_c": 2.33855602145195, "z_loss": 49.5500750541687, "lr": 0.0023, "wd": 0.01, "grad_norm": 1.1195799112319946, "tokens_per_second": 1403229.5861939494, "eta_seconds": 7121.005065965653, "data_wait_seconds": 2.753734588623047e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 53430, "total_steps": 72479, "tokens": 28012707840, "loss": 3.6334570348262787, "l_mix": 2.253660649061203, "l_a": 2.3113464266061783, "l_c": 2.2879747301340103, "z_loss": 46.353291034698486, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.8056454658508301, "tokens_per_second": 1439973.7700661889, "eta_seconds": 6935.655579018593, "data_wait_seconds": 2.1785497665405273e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 53440, "total_steps": 72479, "tokens": 28017950720, "loss": 4.383998155593872, "l_mix": 2.717059016227722, "l_a": 2.803919106721878, "l_c": 2.7525444626808167, "z_loss": 40.46748352050781, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14346887171268463, "tokens_per_second": 1465187.9655383972, "eta_seconds": 6812.722644996643, "data_wait_seconds": 2.142786979675293e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 53450, "total_steps": 72479, "tokens": 28023193600, "loss": 4.442913711071014, "l_mix": 2.7531566619873047, "l_a": 2.839624971151352, "l_c": 2.792898714542389, "z_loss": 40.12174081802368, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13891233503818512, "tokens_per_second": 1453740.8909466548, "eta_seconds": 6862.761042308808, "data_wait_seconds": 1.52587890625e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 53460, "total_steps": 72479, "tokens": 28028436480, "loss": 4.432868123054504, "l_mix": 2.748252034187317, "l_a": 2.831955850124359, "l_c": 2.7834307849407196, "z_loss": 40.27630853652954, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1856027990579605, "tokens_per_second": 1014297.5959888299, "eta_seconds": 9830.875584673882, "data_wait_seconds": 1.9609928131103516e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 53470, "total_steps": 72479, "tokens": 28033679360, "loss": 4.374074995517731, "l_mix": 2.70979967713356, "l_a": 2.7977894842624664, "l_c": 2.7497947812080383, "z_loss": 40.1019492149353, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.21155808866024017, "tokens_per_second": 1451353.6389697334, "eta_seconds": 6866.824407505988, "data_wait_seconds": 2.0235776901245117e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 53480, "total_steps": 72479, "tokens": 28038922240, "loss": 4.30814516544342, "l_mix": 2.6694173216819763, "l_a": 2.7575084567070007, "l_c": 2.7049176394939423, "z_loss": 41.03574180603027, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.22244305908679962, "tokens_per_second": 1436440.3461268581, "eta_seconds": 6934.466675806046, "data_wait_seconds": 1.811981201171875e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 53490, "total_steps": 72479, "tokens": 28044165120, "loss": 4.3792009353637695, "l_mix": 2.713135749101639, "l_a": 2.8025734424591064, "l_c": 2.7509768307209015, "z_loss": 40.952391624450684, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1716035157442093, "tokens_per_second": 1169275.4538455596, "eta_seconds": 8514.422157120704, "data_wait_seconds": 1.806020736694336e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 53500, "total_steps": 72479, "tokens": 28049408000, "loss": 4.414857029914856, "l_mix": 2.7353188693523407, "l_a": 2.8204712569713593, "l_c": 2.7779887914657593, "z_loss": 40.764408111572266, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15926721692085266, "tokens_per_second": 1438083.8879622207, "eta_seconds": 6919.2500071048735, "data_wait_seconds": 1.6361474990844727e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.17164271, "gate_a2b/L1_absmax": 0.70410043, "gate_a2b/L1_rms": 0.18277438, "gate_a2b/L3_absmean": 0.16535926, "gate_a2b/L3_absmax": 0.54012942, "gate_a2b/L3_rms": 0.17141667, "gate_a2b/L5_absmean": 0.24868174, "gate_a2b/L5_absmax": 0.73794484, "gate_a2b/L5_rms": 0.2548506, "gate_c2b/L1_absmean": 0.02684067, "gate_c2b/L1_absmax": 0.62863964, "gate_c2b/L1_rms": 0.06692891, "gate_c2b/L3_absmean": 0.17823428, "gate_c2b/L3_absmax": 0.39961529, "gate_c2b/L3_rms": 0.18214251, "gate_c2b/L5_absmean": 0.50712699, "gate_c2b/L5_absmax": 0.72490567, "gate_c2b/L5_rms": 0.51751924, "gate_b2a/L7_absmean": 0.05064999, "gate_b2a/L7_absmax": 0.32648855, "gate_b2a/L7_rms": 0.05647088, "gate_b2a/L11_absmean": 0.08235469, "gate_b2a/L11_absmax": 0.60702699, "gate_b2a/L11_rms": 0.091568, "gate_b2a/L14_absmean": 0.21117941, "gate_b2a/L14_absmax": 0.80929142, "gate_b2a/L14_rms": 0.22034341, "gate_b2c/L3_absmean": 0.03545718, "gate_b2c/L3_absmax": 0.56277639, "gate_b2c/L3_rms": 0.04636671, "gate_b2c/L5_absmean": 0.03727616, "gate_b2c/L5_absmax": 0.56255341, "gate_b2c/L5_rms": 0.04741379, "gate_b2c/L6_absmean": 0.05396133, "gate_b2c/L6_absmax": 0.35293642, "gate_b2c/L6_rms": 0.06034128}, "streams": {"cos_a2b/round0": 0.355579, "cos_c2b/round0": 0.275683, "cos_b2a/round0": 0.155146, "cos_b2c/round0": 0.13718, "rms_a/round0": 6.817828, "rms_b/round0": 12.040612, "rms_c/round0": 4.539011, "cos_a2b/round1": 0.410369, "cos_c2b/round1": 0.304933, "cos_b2a/round1": 0.247554, "cos_b2c/round1": 0.112179, "rms_a/round1": 13.193968, "rms_b/round1": 27.781801, "rms_c/round1": 9.538582, "cos_a2b/round2": 0.539678, "cos_c2b/round2": 0.523543, "cos_b2a/round2": 0.102248, "cos_b2c/round2": 0.182178, "rms_a/round2": 37.255226, "rms_b/round2": 75.073479, "rms_c/round2": 18.341429}, "alpha_mean": 0.43511459553145926, "alpha_hist": [0.01166, 0.08231, 0.17442, 0.19798, 0.16905, 0.14508, 0.12709, 0.07936, 0.01288, 0.00017], "alpha_by_type": {"special": {"share": 0.00085, "mean": 0.55733, "std": 0.12588}, "whitespace": {"share": 0.0226, "mean": 0.4953, "std": 0.12587}, "digit": {"share": 0.03085, "mean": 0.44311, "std": 0.10238}, "word_start": {"share": 0.56915, "mean": 0.45471, "std": 0.17754}, "word_cont": {"share": 0.26918, "mean": 0.32672, "std": 0.13997}, "punct": {"share": 0.10736, "mean": 0.58704, "std": 0.16627}}, "alpha_p05": 0.17953, "alpha_p50": 0.4206, "alpha_p95": 0.75128}
{"step": 53510, "total_steps": 72479, "tokens": 28054650880, "loss": 4.480109691619873, "l_mix": 2.778008133172989, "l_a": 2.860285371541977, "l_c": 2.813386231660843, "z_loss": 39.03977298736572, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16390931606292725, "tokens_per_second": 1388403.2966192202, "eta_seconds": 7163.062127709388, "data_wait_seconds": 1.722574234008789e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 53520, "total_steps": 72479, "tokens": 28059893760, "loss": 4.40431672334671, "l_mix": 2.728877991437912, "l_a": 2.816902369260788, "l_c": 2.7678932547569275, "z_loss": 40.07575464248657, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18005049228668213, "tokens_per_second": 1301421.5145794065, "eta_seconds": 7637.783823800087, "data_wait_seconds": 2.9355287551879883e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 53530, "total_steps": 72479, "tokens": 28065136640, "loss": 4.391305923461914, "l_mix": 2.7211323380470276, "l_a": 2.811482399702072, "l_c": 2.7557626962661743, "z_loss": 40.98704433441162, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19181877374649048, "tokens_per_second": 1387318.125748617, "eta_seconds": 7161.106834554672, "data_wait_seconds": 2.0116567611694336e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 53540, "total_steps": 72479, "tokens": 28070379520, "loss": 2.914401054382324, "l_mix": 1.8033405095338821, "l_a": 1.873360738158226, "l_c": 1.8301742523908615, "z_loss": 49.70733070373535, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13800999522209167, "tokens_per_second": 1388323.7059050105, "eta_seconds": 7152.14354531765, "data_wait_seconds": 2.2590160369873047e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 53550, "total_steps": 72479, "tokens": 28075622400, "loss": 4.243893563747406, "l_mix": 2.6290130615234375, "l_a": 2.7157800495624542, "l_c": 2.6671546697616577, "z_loss": 41.67236375808716, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16781079769134521, "tokens_per_second": 1280602.453053954, "eta_seconds": 7749.670889925957, "data_wait_seconds": 2.0951032638549805e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 53560, "total_steps": 72479, "tokens": 28080865280, "loss": 4.371025383472443, "l_mix": 2.7087950706481934, "l_a": 2.7957431077957153, "l_c": 2.74502432346344, "z_loss": 40.52936887741089, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14378130435943604, "tokens_per_second": 1340168.3362537601, "eta_seconds": 7401.312509536744, "data_wait_seconds": 1.6957521438598633e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 53570, "total_steps": 72479, "tokens": 28086108160, "loss": 4.351385295391083, "l_mix": 2.6962304413318634, "l_a": 2.784641057252884, "l_c": 2.7325416803359985, "z_loss": 40.340999126434326, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19138763844966888, "tokens_per_second": 1443259.0607862915, "eta_seconds": 6869.0106034040455, "data_wait_seconds": 1.5735626220703125e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 53580, "total_steps": 72479, "tokens": 28091351040, "loss": 4.393188416957855, "l_mix": 2.722185343503952, "l_a": 2.807364672422409, "l_c": 2.7626453042030334, "z_loss": 41.93168926239014, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1527693122625351, "tokens_per_second": 1395852.5119061978, "eta_seconds": 7098.5428814888, "data_wait_seconds": 1.7076730728149414e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 53590, "total_steps": 72479, "tokens": 28096593920, "loss": 4.287922114133835, "l_mix": 2.657225549221039, "l_a": 2.742044508457184, "l_c": 2.6936101615428925, "z_loss": 41.571335792541504, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14227358996868134, "tokens_per_second": 1441774.8470790698, "eta_seconds": 6868.8089905738825, "data_wait_seconds": 1.6391277313232422e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 53600, "total_steps": 72479, "tokens": 28101836800, "loss": 4.586328327655792, "l_mix": 2.8450748920440674, "l_a": 2.9216078519821167, "l_c": 2.8825697898864746, "z_loss": 40.62085008621216, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11125469207763672, "tokens_per_second": 1466742.910402032, "eta_seconds": 6748.308160758018, "data_wait_seconds": 2.3156404495239258e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.17187312, "gate_a2b/L1_absmax": 0.70264363, "gate_a2b/L1_rms": 0.18279824, "gate_a2b/L3_absmean": 0.16540478, "gate_a2b/L3_absmax": 0.53520471, "gate_a2b/L3_rms": 0.1713693, "gate_a2b/L5_absmean": 0.24828056, "gate_a2b/L5_absmax": 0.73670727, "gate_a2b/L5_rms": 0.25433353, "gate_c2b/L1_absmean": 0.02677085, "gate_c2b/L1_absmax": 0.62719047, "gate_c2b/L1_rms": 0.0668217, "gate_c2b/L3_absmean": 0.17803721, "gate_c2b/L3_absmax": 0.39938653, "gate_c2b/L3_rms": 0.1818295, "gate_c2b/L5_absmean": 0.50685596, "gate_c2b/L5_absmax": 0.72141969, "gate_c2b/L5_rms": 0.51725996, "gate_b2a/L7_absmean": 0.05079983, "gate_b2a/L7_absmax": 0.32420847, "gate_b2a/L7_rms": 0.05642924, "gate_b2a/L11_absmean": 0.08213013, "gate_b2a/L11_absmax": 0.6053037, "gate_b2a/L11_rms": 0.09130896, "gate_b2a/L14_absmean": 0.21180713, "gate_b2a/L14_absmax": 0.80653065, "gate_b2a/L14_rms": 0.22085409, "gate_b2c/L3_absmean": 0.03536664, "gate_b2c/L3_absmax": 0.56112587, "gate_b2c/L3_rms": 0.04625582, "gate_b2c/L5_absmean": 0.03763949, "gate_b2c/L5_absmax": 0.5573377, "gate_b2c/L5_rms": 0.04750028, "gate_b2c/L6_absmean": 0.05406019, "gate_b2c/L6_absmax": 0.3520821, "gate_b2c/L6_rms": 0.06030392}, "streams": {"cos_a2b/round0": 0.357704, "cos_c2b/round0": 0.284763, "cos_b2a/round0": 0.15542, "cos_b2c/round0": 0.14367, "rms_a/round0": 6.766984, "rms_b/round0": 11.91513, "rms_c/round0": 4.45072, "cos_a2b/round1": 0.414971, "cos_c2b/round1": 0.30851, "cos_b2a/round1": 0.24838, "cos_b2c/round1": 0.109021, "rms_a/round1": 13.117466, "rms_b/round1": 27.001959, "rms_c/round1": 9.426043, "cos_a2b/round2": 0.555398, "cos_c2b/round2": 0.51037, "cos_b2a/round2": 0.103133, "cos_b2c/round2": 0.185724, "rms_a/round2": 36.727921, "rms_b/round2": 72.372528, "rms_c/round2": 17.901718}, "alpha_mean": 0.46965067733000154, "alpha_hist": [0.00546, 0.04578, 0.13908, 0.17239, 0.18001, 0.20892, 0.16197, 0.07584, 0.00935, 0.00119], "alpha_by_type": {"special": {"share": 0.00044, "mean": 0.47801, "std": 0.22851}, "whitespace": {"share": 0.04831, "mean": 0.57287, "std": 0.12139}, "digit": {"share": 0.02696, "mean": 0.46041, "std": 0.15331}, "word_start": {"share": 0.60573, "mean": 0.48391, "std": 0.16151}, "word_cont": {"share": 0.22289, "mean": 0.35649, "std": 0.13173}, "punct": {"share": 0.09567, "mean": 0.59345, "std": 0.15564}}, "alpha_p05": 0.18952, "alpha_p50": 0.48273, "alpha_p95": 0.73905}
{"step": 53610, "total_steps": 72479, "tokens": 28107079680, "loss": 4.714717328548431, "l_mix": 2.9243273735046387, "l_a": 3.001636892557144, "l_c": 2.96632918715477, "z_loss": 40.52024698257446, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17918716371059418, "tokens_per_second": 1181649.1217102918, "eta_seconds": 8372.020162534714, "data_wait_seconds": 3.406405448913574e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 53620, "total_steps": 72479, "tokens": 28112322560, "loss": 4.6507983803749084, "l_mix": 2.8863348960876465, "l_a": 2.9590703547000885, "l_c": 2.9224743247032166, "z_loss": 39.52616310119629, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1455441564321518, "tokens_per_second": 1431938.6104989748, "eta_seconds": 6905.007881975174, "data_wait_seconds": 2.5540590286254883e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 53630, "total_steps": 72479, "tokens": 28117565440, "loss": 4.6709885597229, "l_mix": 2.897462248802185, "l_a": 2.9747620224952698, "l_c": 2.9369922876358032, "z_loss": 39.04729652404785, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20371080935001373, "tokens_per_second": 1272807.9272996471, "eta_seconds": 7764.175803780555, "data_wait_seconds": 2.2530555725097656e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 53640, "total_steps": 72479, "tokens": 28122808320, "loss": 4.570972383022308, "l_mix": 2.836108297109604, "l_a": 2.9133268892765045, "l_c": 2.869553357362747, "z_loss": 39.904231548309326, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19667090475559235, "tokens_per_second": 1403332.925474684, "eta_seconds": 7038.288244152069, "data_wait_seconds": 2.568960189819336e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 53650, "total_steps": 72479, "tokens": 28128051200, "loss": 2.9251382052898407, "l_mix": 1.8106829524040222, "l_a": 1.8770023733377457, "l_c": 1.8378482162952423, "z_loss": 49.57947015762329, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15540336072444916, "tokens_per_second": 1431092.0864841647, "eta_seconds": 6898.1016981601715, "data_wait_seconds": 3.3527612686157227e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 53660, "total_steps": 72479, "tokens": 28133294080, "loss": 4.3252629935741425, "l_mix": 2.680093288421631, "l_a": 2.7666760087013245, "l_c": 2.717222571372986, "z_loss": 42.52567911148071, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.123428575694561, "tokens_per_second": 1461226.091648274, "eta_seconds": 6752.258208632469, "data_wait_seconds": 2.676248550415039e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 53670, "total_steps": 72479, "tokens": 28138536960, "loss": 4.261174708604813, "l_mix": 2.6402252912521362, "l_a": 2.727884531021118, "l_c": 2.6752796173095703, "z_loss": 41.76205015182495, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16145707666873932, "tokens_per_second": 1399990.014625571, "eta_seconds": 7043.859519696235, "data_wait_seconds": 2.676248550415039e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 53680, "total_steps": 72479, "tokens": 28143779840, "loss": 4.356222689151764, "l_mix": 2.6995849013328552, "l_a": 2.783751606941223, "l_c": 2.7383742332458496, "z_loss": 41.85185670852661, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15779954195022583, "tokens_per_second": 1381806.2936382098, "eta_seconds": 7132.758156752586, "data_wait_seconds": 2.5779008865356445e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 53690, "total_steps": 72479, "tokens": 28149022720, "loss": 4.260755300521851, "l_mix": 2.640173524618149, "l_a": 2.7259185910224915, "l_c": 2.6760202050209045, "z_loss": 41.53413009643555, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11995073407888412, "tokens_per_second": 1434851.0178767235, "eta_seconds": 6865.41467320919, "data_wait_seconds": 2.3037195205688477e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 53700, "total_steps": 72479, "tokens": 28154265600, "loss": 4.332931995391846, "l_mix": 2.6849185824394226, "l_a": 2.7709881961345673, "l_c": 2.7223894894123077, "z_loss": 41.67727470397949, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12087248265743256, "tokens_per_second": 1430303.6813845246, "eta_seconds": 6883.576180458069, "data_wait_seconds": 2.187490463256836e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.17216516, "gate_a2b/L1_absmax": 0.70184743, "gate_a2b/L1_rms": 0.18308374, "gate_a2b/L3_absmean": 0.16533928, "gate_a2b/L3_absmax": 0.53581995, "gate_a2b/L3_rms": 0.17128211, "gate_a2b/L5_absmean": 0.24839988, "gate_a2b/L5_absmax": 0.73748845, "gate_a2b/L5_rms": 0.25443316, "gate_c2b/L1_absmean": 0.02625016, "gate_c2b/L1_absmax": 0.62667131, "gate_c2b/L1_rms": 0.06648053, "gate_c2b/L3_absmean": 0.17791335, "gate_c2b/L3_absmax": 0.40423855, "gate_c2b/L3_rms": 0.18174256, "gate_c2b/L5_absmean": 0.50582951, "gate_c2b/L5_absmax": 0.71016586, "gate_c2b/L5_rms": 0.51625592, "gate_b2a/L7_absmean": 0.05062961, "gate_b2a/L7_absmax": 0.32373503, "gate_b2a/L7_rms": 0.05630925, "gate_b2a/L11_absmean": 0.08213898, "gate_b2a/L11_absmax": 0.60543388, "gate_b2a/L11_rms": 0.09127308, "gate_b2a/L14_absmean": 0.21178094, "gate_b2a/L14_absmax": 0.80539674, "gate_b2a/L14_rms": 0.22082508, "gate_b2c/L3_absmean": 0.03541014, "gate_b2c/L3_absmax": 0.56136262, "gate_b2c/L3_rms": 0.04631294, "gate_b2c/L5_absmean": 0.03744964, "gate_b2c/L5_absmax": 0.55574757, "gate_b2c/L5_rms": 0.04730173, "gate_b2c/L6_absmean": 0.05395938, "gate_b2c/L6_absmax": 0.35148415, "gate_b2c/L6_rms": 0.06020175}, "streams": {"cos_a2b/round0": 0.363614, "cos_c2b/round0": 0.280762, "cos_b2a/round0": 0.142488, "cos_b2c/round0": 0.138517, "rms_a/round0": 6.831624, "rms_b/round0": 11.862905, "rms_c/round0": 4.509501, "cos_a2b/round1": 0.40375, "cos_c2b/round1": 0.287082, "cos_b2a/round1": 0.242243, "cos_b2c/round1": 0.105713, "rms_a/round1": 13.155808, "rms_b/round1": 27.545639, "rms_c/round1": 9.477941, "cos_a2b/round2": 0.527382, "cos_c2b/round2": 0.524327, "cos_b2a/round2": 0.094532, "cos_b2c/round2": 0.177484, "rms_a/round2": 36.603275, "rms_b/round2": 74.96064, "rms_c/round2": 18.092155}, "alpha_mean": 0.42017788558618463, "alpha_hist": [0.00879, 0.07529, 0.19205, 0.2191, 0.18655, 0.14587, 0.10796, 0.05591, 0.00826, 0.00023], "alpha_by_type": {"special": {"share": 0.00067, "mean": 0.5413, "std": 0.10705}, "whitespace": {"share": 0.03656, "mean": 0.52518, "std": 0.12697}, "digit": {"share": 0.07684, "mean": 0.42181, "std": 0.10454}, "word_start": {"share": 0.50223, "mean": 0.43503, "std": 0.17103}, "word_cont": {"share": 0.26944, "mean": 0.31935, "std": 0.1233}, "punct": {"share": 0.11426, "mean": 0.55727, "std": 0.1591}}, "alpha_p05": 0.15637, "alpha_p50": 0.39654, "alpha_p95": 0.71661}
{"step": 53710, "total_steps": 72479, "tokens": 28159508480, "loss": 2.866071730852127, "l_mix": 1.7740042954683304, "l_a": 1.8411985039710999, "l_c": 1.7990260571241379, "z_loss": 48.74942874908447, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13529141247272491, "tokens_per_second": 1457361.883154082, "eta_seconds": 6752.174312877655, "data_wait_seconds": 1.6629695892333984e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 53720, "total_steps": 72479, "tokens": 28164751360, "loss": 4.72159880399704, "l_mix": 2.929160922765732, "l_a": 3.0048877894878387, "l_c": 2.9699049293994904, "z_loss": 39.885369300842285, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16234801709651947, "tokens_per_second": 1365645.8166730239, "eta_seconds": 7201.807724905014, "data_wait_seconds": 5.21540641784668e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 53730, "total_steps": 72479, "tokens": 28169994240, "loss": 4.64322555065155, "l_mix": 2.880521982908249, "l_a": 2.9562333822250366, "l_c": 2.9194448590278625, "z_loss": 39.579792976379395, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12178055197000504, "tokens_per_second": 1425711.310291342, "eta_seconds": 6894.716792273521, "data_wait_seconds": 2.512335777282715e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 53740, "total_steps": 72479, "tokens": 28175237120, "loss": 4.617777466773987, "l_mix": 2.8640905618667603, "l_a": 2.94225737452507, "l_c": 2.9033651053905487, "z_loss": 39.10825443267822, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1544368416070938, "tokens_per_second": 1377325.9074718608, "eta_seconds": 7133.121346735955, "data_wait_seconds": 1.7076730728149414e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 53750, "total_steps": 72479, "tokens": 28180480000, "loss": 4.719828486442566, "l_mix": 2.9283851087093353, "l_a": 3.0044401586055756, "l_c": 2.967037558555603, "z_loss": 39.40659713745117, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13749217987060547, "tokens_per_second": 1274936.4440479563, "eta_seconds": 7701.8662364244465, "data_wait_seconds": 4.807114601135254e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 53760, "total_steps": 72479, "tokens": 28185722880, "loss": 2.783577084541321, "l_mix": 1.7231401652097702, "l_a": 1.7865949869155884, "l_c": 1.7481946349143982, "z_loss": 49.6829719543457, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13635388016700745, "tokens_per_second": 1332379.3635656445, "eta_seconds": 7365.880424427986, "data_wait_seconds": 2.1070241928100586e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 53770, "total_steps": 72479, "tokens": 28190965760, "loss": 4.289811491966248, "l_mix": 2.659272015094757, "l_a": 2.741103321313858, "l_c": 2.6940282583236694, "z_loss": 41.15272331237793, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1511739194393158, "tokens_per_second": 1428321.6200412384, "eta_seconds": 6867.433814883232, "data_wait_seconds": 2.5659799575805664e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 53780, "total_steps": 72479, "tokens": 28196208640, "loss": 4.286919176578522, "l_mix": 2.6557843685150146, "l_a": 2.742789000272751, "l_c": 2.694326728582382, "z_loss": 41.43326950073242, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15124347805976868, "tokens_per_second": 1379638.0226677337, "eta_seconds": 7105.966312122345, "data_wait_seconds": 1.6957521438598633e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 53790, "total_steps": 72479, "tokens": 28201451520, "loss": 4.2466104328632355, "l_mix": 2.630777597427368, "l_a": 2.7194928526878357, "l_c": 2.666616588830948, "z_loss": 41.944416999816895, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1516280323266983, "tokens_per_second": 1437646.5207504658, "eta_seconds": 6815.5963865756985, "data_wait_seconds": 1.850724220275879e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 53800, "total_steps": 72479, "tokens": 28206694400, "loss": 4.290310919284821, "l_mix": 2.6585986018180847, "l_a": 2.743289977312088, "l_c": 2.69575098156929, "z_loss": 41.42046117782593, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11797687411308289, "tokens_per_second": 1434338.9877600453, "eta_seconds": 6827.657642698288, "data_wait_seconds": 1.9937753677368164e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.17204086, "gate_a2b/L1_absmax": 0.70064908, "gate_a2b/L1_rms": 0.18284711, "gate_a2b/L3_absmean": 0.16475466, "gate_a2b/L3_absmax": 0.53833097, "gate_a2b/L3_rms": 0.17078608, "gate_a2b/L5_absmean": 0.24790387, "gate_a2b/L5_absmax": 0.73727334, "gate_a2b/L5_rms": 0.25394839, "gate_c2b/L1_absmean": 0.02619693, "gate_c2b/L1_absmax": 0.62640637, "gate_c2b/L1_rms": 0.06650496, "gate_c2b/L3_absmean": 0.177617, "gate_c2b/L3_absmax": 0.40415192, "gate_c2b/L3_rms": 0.18142377, "gate_c2b/L5_absmean": 0.50598317, "gate_c2b/L5_absmax": 0.71244353, "gate_c2b/L5_rms": 0.51632464, "gate_b2a/L7_absmean": 0.05072185, "gate_b2a/L7_absmax": 0.32715741, "gate_b2a/L7_rms": 0.05636748, "gate_b2a/L11_absmean": 0.08184896, "gate_b2a/L11_absmax": 0.60443676, "gate_b2a/L11_rms": 0.09097531, "gate_b2a/L14_absmean": 0.21238557, "gate_b2a/L14_absmax": 0.80467582, "gate_b2a/L14_rms": 0.22134778, "gate_b2c/L3_absmean": 0.03529425, "gate_b2c/L3_absmax": 0.56001252, "gate_b2c/L3_rms": 0.04614454, "gate_b2c/L5_absmean": 0.03744623, "gate_b2c/L5_absmax": 0.55714232, "gate_b2c/L5_rms": 0.04733892, "gate_b2c/L6_absmean": 0.05397832, "gate_b2c/L6_absmax": 0.35070375, "gate_b2c/L6_rms": 0.06020415}, "streams": {"cos_a2b/round0": 0.358292, "cos_c2b/round0": 0.274069, "cos_b2a/round0": 0.154535, "cos_b2c/round0": 0.137907, "rms_a/round0": 6.7981, "rms_b/round0": 12.061605, "rms_c/round0": 4.557923, "cos_a2b/round1": 0.406775, "cos_c2b/round1": 0.299192, "cos_b2a/round1": 0.251264, "cos_b2c/round1": 0.113601, "rms_a/round1": 13.334099, "rms_b/round1": 27.945292, "rms_c/round1": 9.631589, "cos_a2b/round2": 0.541743, "cos_c2b/round2": 0.513504, "cos_b2a/round2": 0.101066, "cos_b2c/round2": 0.182337, "rms_a/round2": 37.486992, "rms_b/round2": 74.822601, "rms_c/round2": 18.495209}, "alpha_mean": 0.42209595167905434, "alpha_hist": [0.01266, 0.07959, 0.18726, 0.20528, 0.18005, 0.15329, 0.11514, 0.05957, 0.00703, 0.00012], "alpha_by_type": {"special": {"share": 0.00128, "mean": 0.50847, "std": 0.12301}, "whitespace": {"share": 0.03026, "mean": 0.50572, "std": 0.12502}, "digit": {"share": 0.04858, "mean": 0.44939, "std": 0.10813}, "word_start": {"share": 0.53659, "mean": 0.43915, "std": 0.16764}, "word_cont": {"share": 0.27655, "mean": 0.31594, "std": 0.13212}, "punct": {"share": 0.10674, "mean": 0.57423, "std": 0.16133}}, "alpha_p05": 0.17051, "alpha_p50": 0.41017, "alpha_p95": 0.71263}
{"step": 53810, "total_steps": 72479, "tokens": 28211937280, "loss": 4.270405471324921, "l_mix": 2.6454760134220123, "l_a": 2.7325630486011505, "l_c": 2.6838682293891907, "z_loss": 42.392723083496094, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1436496376991272, "tokens_per_second": 1408879.2510044004, "eta_seconds": 6947.31834897995, "data_wait_seconds": 1.6301870346069336e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 53820, "total_steps": 72479, "tokens": 28217180160, "loss": 2.697069823741913, "l_mix": 1.6692813783884048, "l_a": 1.730428084731102, "l_c": 1.6955330967903137, "z_loss": 49.08711242675781, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15606719255447388, "tokens_per_second": 1380200.4373488205, "eta_seconds": 7087.876171660423, "data_wait_seconds": 1.6510486602783203e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 53830, "total_steps": 72479, "tokens": 28222423040, "loss": 2.848771810531616, "l_mix": 1.7645721584558487, "l_a": 1.8257181197404861, "l_c": 1.7882804423570633, "z_loss": 49.14162826538086, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20942196249961853, "tokens_per_second": 1453344.1833253684, "eta_seconds": 6727.550861096382, "data_wait_seconds": 4.222989082336426e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 53840, "total_steps": 72479, "tokens": 28227665920, "loss": 4.692606568336487, "l_mix": 2.911065846681595, "l_a": 2.9876638650894165, "l_c": 2.950805127620697, "z_loss": 40.87956523895264, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17686419188976288, "tokens_per_second": 1470973.9607933061, "eta_seconds": 6643.356233668327, "data_wait_seconds": 2.6345252990722656e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 53850, "total_steps": 72479, "tokens": 28232908800, "loss": 4.621268093585968, "l_mix": 2.866902083158493, "l_a": 2.9400985836982727, "l_c": 2.90778785943985, "z_loss": 39.11787128448486, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19549046456813812, "tokens_per_second": 1450062.591762637, "eta_seconds": 6735.544525790214, "data_wait_seconds": 2.1159648895263672e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 53860, "total_steps": 72479, "tokens": 28238151680, "loss": 4.515132546424866, "l_mix": 2.800923317670822, "l_a": 2.8759706914424896, "l_c": 2.838059723377228, "z_loss": 38.57429885864258, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20173689723014832, "tokens_per_second": 1452598.1451541733, "eta_seconds": 6720.1781198501585, "data_wait_seconds": 2.104043960571289e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 53870, "total_steps": 72479, "tokens": 28243394560, "loss": 4.641092360019684, "l_mix": 2.8786662817001343, "l_a": 2.9568755626678467, "l_c": 2.917878031730652, "z_loss": 40.143293380737305, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1970599889755249, "tokens_per_second": 1408824.3722321922, "eta_seconds": 6925.260227108002, "data_wait_seconds": 2.327561378479004e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 53880, "total_steps": 72479, "tokens": 28248637440, "loss": 4.297143816947937, "l_mix": 2.662305027246475, "l_a": 2.749116599559784, "l_c": 2.70034596323967, "z_loss": 41.524033069610596, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12479642033576965, "tokens_per_second": 1408265.899020654, "eta_seconds": 6924.283630514145, "data_wait_seconds": 2.7805566787719727e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 53890, "total_steps": 72479, "tokens": 28253880320, "loss": 4.323264926671982, "l_mix": 2.678384780883789, "l_a": 2.7669458985328674, "l_c": 2.715987652540207, "z_loss": 41.7613959312439, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11061068624258041, "tokens_per_second": 1228016.6297895145, "eta_seconds": 7936.366166043281, "data_wait_seconds": 1.8090009689331055e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 53900, "total_steps": 72479, "tokens": 28259123200, "loss": 4.348464071750641, "l_mix": 2.694790095090866, "l_a": 2.780991017818451, "l_c": 2.7312555611133575, "z_loss": 40.957520484924316, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12429839372634888, "tokens_per_second": 1447284.4587789248, "eta_seconds": 6730.360913443566, "data_wait_seconds": 1.52587890625e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.17117293, "gate_a2b/L1_absmax": 0.70800215, "gate_a2b/L1_rms": 0.18215473, "gate_a2b/L3_absmean": 0.16444825, "gate_a2b/L3_absmax": 0.53819782, "gate_a2b/L3_rms": 0.17063262, "gate_a2b/L5_absmean": 0.24700575, "gate_a2b/L5_absmax": 0.73737955, "gate_a2b/L5_rms": 0.25311807, "gate_c2b/L1_absmean": 0.0271817, "gate_c2b/L1_absmax": 0.63363814, "gate_c2b/L1_rms": 0.0668174, "gate_c2b/L3_absmean": 0.17696412, "gate_c2b/L3_absmax": 0.40232021, "gate_c2b/L3_rms": 0.18089312, "gate_c2b/L5_absmean": 0.50581402, "gate_c2b/L5_absmax": 0.70626616, "gate_c2b/L5_rms": 0.51607656, "gate_b2a/L7_absmean": 0.05113553, "gate_b2a/L7_absmax": 0.33047205, "gate_b2a/L7_rms": 0.05696781, "gate_b2a/L11_absmean": 0.08256932, "gate_b2a/L11_absmax": 0.60050648, "gate_b2a/L11_rms": 0.09163261, "gate_b2a/L14_absmean": 0.21220468, "gate_b2a/L14_absmax": 0.80699158, "gate_b2a/L14_rms": 0.22122446, "gate_b2c/L3_absmean": 0.03543141, "gate_b2c/L3_absmax": 0.55848134, "gate_b2c/L3_rms": 0.04623188, "gate_b2c/L5_absmean": 0.03749897, "gate_b2c/L5_absmax": 0.55429548, "gate_b2c/L5_rms": 0.04732239, "gate_b2c/L6_absmean": 0.05402353, "gate_b2c/L6_absmax": 0.34787127, "gate_b2c/L6_rms": 0.06019594}, "streams": {"cos_a2b/round0": 0.361469, "cos_c2b/round0": 0.276082, "cos_b2a/round0": 0.153447, "cos_b2c/round0": 0.139987, "rms_a/round0": 6.905726, "rms_b/round0": 12.105213, "rms_c/round0": 4.571617, "cos_a2b/round1": 0.410319, "cos_c2b/round1": 0.300267, "cos_b2a/round1": 0.251613, "cos_b2c/round1": 0.112331, "rms_a/round1": 13.435645, "rms_b/round1": 28.01919, "rms_c/round1": 9.661156, "cos_a2b/round2": 0.54141, "cos_c2b/round2": 0.515361, "cos_b2a/round2": 0.099872, "cos_b2c/round2": 0.179149, "rms_a/round2": 37.707111, "rms_b/round2": 75.069588, "rms_c/round2": 18.715857}, "alpha_mean": 0.4282522451849786, "alpha_hist": [0.01082, 0.08333, 0.18912, 0.19423, 0.16777, 0.15318, 0.12216, 0.06779, 0.01146, 0.00014], "alpha_by_type": {"special": {"share": 0.00139, "mean": 0.59177, "std": 0.13955}, "whitespace": {"share": 0.0293, "mean": 0.5039, "std": 0.1301}, "digit": {"share": 0.04691, "mean": 0.42834, "std": 0.11891}, "word_start": {"share": 0.56184, "mean": 0.44688, "std": 0.17241}, "word_cont": {"share": 0.25836, "mean": 0.31478, "std": 0.13496}, "punct": {"share": 0.1022, "mean": 0.58875, "std": 0.16399}}, "alpha_p05": 0.17696, "alpha_p50": 0.40639, "alpha_p95": 0.73866}
{"step": 53910, "total_steps": 72479, "tokens": 28264366080, "loss": 4.2773414850234985, "l_mix": 2.650541365146637, "l_a": 2.736766964197159, "l_c": 2.6858998239040375, "z_loss": 42.2461895942688, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12084496021270752, "tokens_per_second": 1390520.352982293, "eta_seconds": 7001.338636374474, "data_wait_seconds": 1.5914440155029297e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 53920, "total_steps": 72479, "tokens": 28269608960, "loss": 4.228407979011536, "l_mix": 2.619754433631897, "l_a": 2.7058254778385162, "l_c": 2.6563529670238495, "z_loss": 41.98671007156372, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12943199276924133, "tokens_per_second": 1411871.0943554582, "eta_seconds": 6891.748850798606, "data_wait_seconds": 2.1517276763916016e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 53930, "total_steps": 72479, "tokens": 28274851840, "loss": 2.866866558790207, "l_mix": 1.7754997164011002, "l_a": 1.8362740576267242, "l_c": 1.801615133881569, "z_loss": 48.66385793685913, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19237233698368073, "tokens_per_second": 1410526.4283943472, "eta_seconds": 6894.601842427253, "data_wait_seconds": 2.6106834411621094e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 53940, "total_steps": 72479, "tokens": 28280094720, "loss": 2.917659819126129, "l_mix": 1.8063580691814423, "l_a": 1.8702399730682373, "l_c": 1.834098905324936, "z_loss": 49.59869337081909, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.21328233182430267, "tokens_per_second": 1335421.274227407, "eta_seconds": 7278.433719444275, "data_wait_seconds": 2.1010637283325195e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 53950, "total_steps": 72479, "tokens": 28285337600, "loss": 4.378554105758667, "l_mix": 2.713531196117401, "l_a": 2.7977715730667114, "l_c": 2.7523047029972076, "z_loss": 40.5781455039978, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14318716526031494, "tokens_per_second": 1348430.3128294582, "eta_seconds": 7204.3265859365465, "data_wait_seconds": 1.5556812286376953e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 53960, "total_steps": 72479, "tokens": 28290580480, "loss": 4.257427930831909, "l_mix": 2.638234704732895, "l_a": 2.724501758813858, "l_c": 2.6728086173534393, "z_loss": 41.74375629425049, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.22714996337890625, "tokens_per_second": 1302192.7151123818, "eta_seconds": 7456.107962608337, "data_wait_seconds": 3.337860107421875e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 53970, "total_steps": 72479, "tokens": 28295823360, "loss": 4.344569683074951, "l_mix": 2.6914939284324646, "l_a": 2.7770081162452698, "l_c": 2.7332441806793213, "z_loss": 41.48237943649292, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.26288360357284546, "tokens_per_second": 1460539.7454569824, "eta_seconds": 6644.150987458229, "data_wait_seconds": 2.0802021026611328e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 53980, "total_steps": 72479, "tokens": 28301066240, "loss": 4.347127318382263, "l_mix": 2.6945805847644806, "l_a": 2.7799190282821655, "l_c": 2.728569984436035, "z_loss": 41.89975547790527, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2700669765472412, "tokens_per_second": 1444295.513760825, "eta_seconds": 6715.248797488212, "data_wait_seconds": 2.4646520614624023e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 53990, "total_steps": 72479, "tokens": 28306309120, "loss": 4.317612290382385, "l_mix": 2.6751469671726227, "l_a": 2.760963410139084, "l_c": 2.7139207422733307, "z_loss": 42.922648906707764, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20932795107364655, "tokens_per_second": 1432557.2691460655, "eta_seconds": 6766.613133573533, "data_wait_seconds": 2.041459083557129e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 54000, "total_steps": 72479, "tokens": 28311552000, "loss": 4.240997135639191, "l_mix": 2.6275997161865234, "l_a": 2.712088942527771, "l_c": 2.6659025847911835, "z_loss": 41.587849617004395, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12109306454658508, "tokens_per_second": 1403311.432516303, "eta_seconds": 6903.897258663177, "data_wait_seconds": 2.0444393157958984e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.1707934, "gate_a2b/L1_absmax": 0.70611703, "gate_a2b/L1_rms": 0.18181473, "gate_a2b/L3_absmean": 0.16433139, "gate_a2b/L3_absmax": 0.53548223, "gate_a2b/L3_rms": 0.17034143, "gate_a2b/L5_absmean": 0.24675898, "gate_a2b/L5_absmax": 0.73648459, "gate_a2b/L5_rms": 0.25281298, "gate_c2b/L1_absmean": 0.02653822, "gate_c2b/L1_absmax": 0.63091767, "gate_c2b/L1_rms": 0.06656272, "gate_c2b/L3_absmean": 0.17697707, "gate_c2b/L3_absmax": 0.40317768, "gate_c2b/L3_rms": 0.18075883, "gate_c2b/L5_absmean": 0.5048871, "gate_c2b/L5_absmax": 0.71799588, "gate_c2b/L5_rms": 0.51516229, "gate_b2a/L7_absmean": 0.05111291, "gate_b2a/L7_absmax": 0.32881299, "gate_b2a/L7_rms": 0.05680804, "gate_b2a/L11_absmean": 0.08228628, "gate_b2a/L11_absmax": 0.60273612, "gate_b2a/L11_rms": 0.09132598, "gate_b2a/L14_absmean": 0.21196438, "gate_b2a/L14_absmax": 0.80758953, "gate_b2a/L14_rms": 0.22098605, "gate_b2c/L3_absmean": 0.03520457, "gate_b2c/L3_absmax": 0.55915302, "gate_b2c/L3_rms": 0.0460922, "gate_b2c/L5_absmean": 0.03744105, "gate_b2c/L5_absmax": 0.5563494, "gate_b2c/L5_rms": 0.04731205, "gate_b2c/L6_absmean": 0.05361349, "gate_b2c/L6_absmax": 0.34856337, "gate_b2c/L6_rms": 0.05982821}, "streams": {"cos_a2b/round0": 0.364381, "cos_c2b/round0": 0.281495, "cos_b2a/round0": 0.145018, "cos_b2c/round0": 0.140643, "rms_a/round0": 6.961242, "rms_b/round0": 12.037427, "rms_c/round0": 4.556019, "cos_a2b/round1": 0.41146, "cos_c2b/round1": 0.287689, "cos_b2a/round1": 0.247712, "cos_b2c/round1": 0.103616, "rms_a/round1": 13.487219, "rms_b/round1": 27.944773, "rms_c/round1": 9.518933, "cos_a2b/round2": 0.527547, "cos_c2b/round2": 0.528665, "cos_b2a/round2": 0.093543, "cos_b2c/round2": 0.175168, "rms_a/round2": 37.256004, "rms_b/round2": 75.617493, "rms_c/round2": 18.514351}, "alpha_mean": 0.4212847417087602, "alpha_hist": [0.01181, 0.08371, 0.2, 0.1994, 0.16783, 0.14636, 0.1205, 0.06067, 0.00946, 0.00026], "alpha_by_type": {"special": {"share": 0.00079, "mean": 0.52464, "std": 0.14878}, "whitespace": {"share": 0.03238, "mean": 0.5104, "std": 0.13329}, "digit": {"share": 0.05083, "mean": 0.37044, "std": 0.13974}, "word_start": {"share": 0.55711, "mean": 0.4431, "std": 0.17153}, "word_cont": {"share": 0.2617, "mean": 0.31569, "std": 0.13132}, "punct": {"share": 0.09718, "mean": 0.5766, "std": 0.16251}}, "alpha_p05": 0.16479, "alpha_p50": 0.39584, "alpha_p95": 0.73297}
{"step": 54010, "total_steps": 72479, "tokens": 28316794880, "loss": 4.342301189899445, "l_mix": 2.6910870373249054, "l_a": 2.7757934629917145, "l_c": 2.7282533943653107, "z_loss": 41.08440971374512, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12637636065483093, "tokens_per_second": 1282968.3381768425, "eta_seconds": 7547.399872517585, "data_wait_seconds": 2.0056962966918945e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 54020, "total_steps": 72479, "tokens": 28322037760, "loss": 4.344517529010773, "l_mix": 2.6926089823246, "l_a": 2.7771900594234467, "l_c": 2.729171186685562, "z_loss": 41.72626829147339, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12200246006250381, "tokens_per_second": 1302852.8950975153, "eta_seconds": 7428.184892106056, "data_wait_seconds": 2.3037195205688477e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 54030, "total_steps": 72479, "tokens": 28327280640, "loss": 4.301974773406982, "l_mix": 2.6661079227924347, "l_a": 2.7481989562511444, "l_c": 2.7046902179718018, "z_loss": 41.71342945098877, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1325850635766983, "tokens_per_second": 1436338.6407533623, "eta_seconds": 6734.1983551502235, "data_wait_seconds": 1.9609928131103516e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 54040, "total_steps": 72479, "tokens": 28332523520, "loss": 2.8819068372249603, "l_mix": 1.7844960689544678, "l_a": 1.8486432284116745, "l_c": 1.8093924969434738, "z_loss": 50.424734115600586, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2029532641172409, "tokens_per_second": 1426663.8351367563, "eta_seconds": 6776.190854430198, "data_wait_seconds": 2.7120113372802734e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 54050, "total_steps": 72479, "tokens": 28337766400, "loss": 2.976233094930649, "l_mix": 1.8425575941801071, "l_a": 1.9085394591093063, "l_c": 1.8703787624835968, "z_loss": 48.66162157058716, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2216167002916336, "tokens_per_second": 1384898.7523460058, "eta_seconds": 6976.758073925971, "data_wait_seconds": 2.5659799575805664e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 54060, "total_steps": 72479, "tokens": 28343009280, "loss": 4.37309342622757, "l_mix": 2.710604667663574, "l_a": 2.7942413091659546, "l_c": 2.7473875284194946, "z_loss": 41.40671110153198, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12082900106906891, "tokens_per_second": 1306398.513960375, "eta_seconds": 7391.971568250657, "data_wait_seconds": 2.5570392608642578e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 54070, "total_steps": 72479, "tokens": 28348252160, "loss": 4.245163142681122, "l_mix": 2.6310577392578125, "l_a": 2.713980108499527, "l_c": 2.666371077299118, "z_loss": 40.93000888824463, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11640668660402298, "tokens_per_second": 1440003.661559892, "eta_seconds": 6702.495312786102, "data_wait_seconds": 1.6003847122192383e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 54080, "total_steps": 72479, "tokens": 28353495040, "loss": 4.239860534667969, "l_mix": 2.627559542655945, "l_a": 2.7094520032405853, "l_c": 2.664884328842163, "z_loss": 41.902010440826416, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12745197117328644, "tokens_per_second": 1439431.601082017, "eta_seconds": 6701.516699194908, "data_wait_seconds": 1.9818544387817383e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 54090, "total_steps": 72479, "tokens": 28358737920, "loss": 4.2890214920043945, "l_mix": 2.6575538218021393, "l_a": 2.74306583404541, "l_c": 2.695159465074539, "z_loss": 41.91127061843872, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13115666806697845, "tokens_per_second": 1447632.1199192863, "eta_seconds": 6659.932381534576, "data_wait_seconds": 2.276897430419922e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 54100, "total_steps": 72479, "tokens": 28363980800, "loss": 4.300890147686005, "l_mix": 2.6646989285945892, "l_a": 2.7535853683948517, "l_c": 2.7003851532936096, "z_loss": 41.24874258041382, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1480759084224701, "tokens_per_second": 1421892.1062040655, "eta_seconds": 6776.807543945312, "data_wait_seconds": 2.4527311325073242e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.1706754, "gate_a2b/L1_absmax": 0.70519161, "gate_a2b/L1_rms": 0.18168403, "gate_a2b/L3_absmean": 0.1644478, "gate_a2b/L3_absmax": 0.53514868, "gate_a2b/L3_rms": 0.17042486, "gate_a2b/L5_absmean": 0.24652465, "gate_a2b/L5_absmax": 0.73460269, "gate_a2b/L5_rms": 0.25259605, "gate_c2b/L1_absmean": 0.02677203, "gate_c2b/L1_absmax": 0.62913489, "gate_c2b/L1_rms": 0.06671347, "gate_c2b/L3_absmean": 0.17738234, "gate_c2b/L3_absmax": 0.40411115, "gate_c2b/L3_rms": 0.18118075, "gate_c2b/L5_absmean": 0.50429434, "gate_c2b/L5_absmax": 0.71031904, "gate_c2b/L5_rms": 0.51455951, "gate_b2a/L7_absmean": 0.05091239, "gate_b2a/L7_absmax": 0.32828736, "gate_b2a/L7_rms": 0.05664834, "gate_b2a/L11_absmean": 0.08195662, "gate_b2a/L11_absmax": 0.60403454, "gate_b2a/L11_rms": 0.09108504, "gate_b2a/L14_absmean": 0.21199211, "gate_b2a/L14_absmax": 0.80522043, "gate_b2a/L14_rms": 0.2210163, "gate_b2c/L3_absmean": 0.03517891, "gate_b2c/L3_absmax": 0.55997676, "gate_b2c/L3_rms": 0.0461083, "gate_b2c/L5_absmean": 0.03742795, "gate_b2c/L5_absmax": 0.55983025, "gate_b2c/L5_rms": 0.04740312, "gate_b2c/L6_absmean": 0.053749, "gate_b2c/L6_absmax": 0.34736603, "gate_b2c/L6_rms": 0.05990951}, "streams": {"cos_a2b/round0": 0.360352, "cos_c2b/round0": 0.28438, "cos_b2a/round0": 0.158312, "cos_b2c/round0": 0.142115, "rms_a/round0": 6.878606, "rms_b/round0": 11.989646, "rms_c/round0": 4.551322, "cos_a2b/round1": 0.414286, "cos_c2b/round1": 0.302529, "cos_b2a/round1": 0.25249, "cos_b2c/round1": 0.106528, "rms_a/round1": 13.382674, "rms_b/round1": 27.627676, "rms_c/round1": 9.59109, "cos_a2b/round2": 0.536177, "cos_c2b/round2": 0.527501, "cos_b2a/round2": 0.101332, "cos_b2c/round2": 0.177219, "rms_a/round2": 37.243633, "rms_b/round2": 74.609413, "rms_c/round2": 18.581078}, "alpha_mean": 0.43874385049806364, "alpha_hist": [0.0076, 0.06244, 0.17126, 0.19606, 0.19971, 0.1689, 0.12042, 0.06339, 0.00989, 0.00034], "alpha_by_type": {"special": {"share": 0.00114, "mean": 0.55856, "std": 0.11856}, "whitespace": {"share": 0.02531, "mean": 0.53189, "std": 0.11927}, "digit": {"share": 0.03235, "mean": 0.41392, "std": 0.13723}, "word_start": {"share": 0.5614, "mean": 0.45447, "std": 0.16308}, "word_cont": {"share": 0.24998, "mean": 0.32813, "std": 0.1295}, "punct": {"share": 0.12981, "mean": 0.57073, "std": 0.14296}}, "alpha_p05": 0.19806, "alpha_p50": 0.4323, "alpha_p95": 0.71163}
{"step": 54110, "total_steps": 72479, "tokens": 28369223680, "loss": 4.340773224830627, "l_mix": 2.6897810101509094, "l_a": 2.7752570509910583, "l_c": 2.7280503809452057, "z_loss": 41.49085760116577, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17417551577091217, "tokens_per_second": 1399605.794950818, "eta_seconds": 6880.9705609560015, "data_wait_seconds": 1.6927719116210938e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 54120, "total_steps": 72479, "tokens": 28374466560, "loss": 4.598513960838318, "l_mix": 2.8522574603557587, "l_a": 2.9304055273532867, "l_c": 2.890449345111847, "z_loss": 40.637531757354736, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12485713511705399, "tokens_per_second": 1428937.1565750916, "eta_seconds": 6736.057878899574, "data_wait_seconds": 1.7940998077392578e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 54130, "total_steps": 72479, "tokens": 28379709440, "loss": 4.725163996219635, "l_mix": 2.9324577152729034, "l_a": 3.004490315914154, "l_c": 2.971197187900543, "z_loss": 38.95599603652954, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12521277368068695, "tokens_per_second": 1432164.9484572608, "eta_seconds": 6717.215445303917, "data_wait_seconds": 2.434849739074707e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 54140, "total_steps": 72479, "tokens": 28384952320, "loss": 4.643999934196472, "l_mix": 2.881074368953705, "l_a": 2.9565197229385376, "l_c": 2.919898420572281, "z_loss": 39.03467655181885, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14609117805957794, "tokens_per_second": 1026853.1903356159, "eta_seconds": 9363.478365254403, "data_wait_seconds": 2.7000904083251953e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 54150, "total_steps": 72479, "tokens": 28390195200, "loss": 4.7767544984817505, "l_mix": 2.9636846482753754, "l_a": 3.041838675737381, "l_c": 3.0017269253730774, "z_loss": 40.84298753738403, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2133949100971222, "tokens_per_second": 1427352.4278605601, "eta_seconds": 6732.5171866655355, "data_wait_seconds": 2.568960189819336e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 54160, "total_steps": 72479, "tokens": 28395438080, "loss": 3.0590695440769196, "l_mix": 1.893530696630478, "l_a": 1.9668019264936447, "l_c": 1.9183273613452911, "z_loss": 49.69816875457764, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.22955244779586792, "tokens_per_second": 1423388.6204370812, "eta_seconds": 6747.582307529449, "data_wait_seconds": 3.108382225036621e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 54170, "total_steps": 72479, "tokens": 28400680960, "loss": 4.272826582193375, "l_mix": 2.648785799741745, "l_a": 2.7300712764263153, "l_c": 2.683397948741913, "z_loss": 40.621458530426025, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12747377157211304, "tokens_per_second": 1476535.1274345785, "eta_seconds": 6501.158566188813, "data_wait_seconds": 1.6987323760986328e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 54180, "total_steps": 72479, "tokens": 28405923840, "loss": 4.477132439613342, "l_mix": 2.775054633617401, "l_a": 2.861290216445923, "l_c": 2.8123020827770233, "z_loss": 40.36481809616089, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16938011348247528, "tokens_per_second": 1437686.654932112, "eta_seconds": 6673.182977032662, "data_wait_seconds": 2.4050474166870117e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 54190, "total_steps": 72479, "tokens": 28411166720, "loss": 4.348674416542053, "l_mix": 2.6949129700660706, "l_a": 2.7794263064861298, "l_c": 2.733111709356308, "z_loss": 41.108084201812744, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13852015137672424, "tokens_per_second": 1432441.6498244186, "eta_seconds": 6693.95729534626, "data_wait_seconds": 2.7477741241455078e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 54200, "total_steps": 72479, "tokens": 28416409600, "loss": 4.271656394004822, "l_mix": 2.646888107061386, "l_a": 2.732249438762665, "l_c": 2.6836447417736053, "z_loss": 42.11006307601929, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12258325517177582, "tokens_per_second": 1400947.6495494207, "eta_seconds": 6840.698405170441, "data_wait_seconds": 1.5735626220703125e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.17082556, "gate_a2b/L1_absmax": 0.70418608, "gate_a2b/L1_rms": 0.18171562, "gate_a2b/L3_absmean": 0.1641625, "gate_a2b/L3_absmax": 0.52825701, "gate_a2b/L3_rms": 0.17010553, "gate_a2b/L5_absmean": 0.24576436, "gate_a2b/L5_absmax": 0.73560905, "gate_a2b/L5_rms": 0.25183147, "gate_c2b/L1_absmean": 0.02685319, "gate_c2b/L1_absmax": 0.62955284, "gate_c2b/L1_rms": 0.06708409, "gate_c2b/L3_absmean": 0.17694969, "gate_c2b/L3_absmax": 0.39902171, "gate_c2b/L3_rms": 0.1807081, "gate_c2b/L5_absmean": 0.50377977, "gate_c2b/L5_absmax": 0.71236426, "gate_c2b/L5_rms": 0.51401645, "gate_b2a/L7_absmean": 0.05088307, "gate_b2a/L7_absmax": 0.32780603, "gate_b2a/L7_rms": 0.05660247, "gate_b2a/L11_absmean": 0.08213647, "gate_b2a/L11_absmax": 0.60335225, "gate_b2a/L11_rms": 0.09124158, "gate_b2a/L14_absmean": 0.2125921, "gate_b2a/L14_absmax": 0.80558974, "gate_b2a/L14_rms": 0.22154044, "gate_b2c/L3_absmean": 0.03516594, "gate_b2c/L3_absmax": 0.55940038, "gate_b2c/L3_rms": 0.04604422, "gate_b2c/L5_absmean": 0.0376506, "gate_b2c/L5_absmax": 0.5611183, "gate_b2c/L5_rms": 0.04764127, "gate_b2c/L6_absmean": 0.05393152, "gate_b2c/L6_absmax": 0.34880725, "gate_b2c/L6_rms": 0.06007887}, "streams": {"cos_a2b/round0": 0.35895, "cos_c2b/round0": 0.275634, "cos_b2a/round0": 0.153971, "cos_b2c/round0": 0.1396, "rms_a/round0": 6.852909, "rms_b/round0": 12.019647, "rms_c/round0": 4.562315, "cos_a2b/round1": 0.409227, "cos_c2b/round1": 0.29561, "cos_b2a/round1": 0.249574, "cos_b2c/round1": 0.11331, "rms_a/round1": 13.340126, "rms_b/round1": 27.967045, "rms_c/round1": 9.563354, "cos_a2b/round2": 0.538717, "cos_c2b/round2": 0.516958, "cos_b2a/round2": 0.098069, "cos_b2c/round2": 0.184049, "rms_a/round2": 37.516518, "rms_b/round2": 75.097618, "rms_c/round2": 18.394655}, "alpha_mean": 0.43099326676249916, "alpha_hist": [0.00887, 0.07404, 0.18846, 0.20207, 0.17506, 0.15024, 0.11818, 0.07216, 0.01068, 0.00024], "alpha_by_type": {"special": {"share": 0.00113, "mean": 0.55647, "std": 0.15149}, "whitespace": {"share": 0.02644, "mean": 0.5369, "std": 0.12976}, "digit": {"share": 0.04594, "mean": 0.46643, "std": 0.12472}, "word_start": {"share": 0.55324, "mean": 0.4474, "std": 0.17259}, "word_cont": {"share": 0.26193, "mean": 0.3206, "std": 0.12773}, "punct": {"share": 0.11131, "mean": 0.56815, "std": 0.16865}}, "alpha_p05": 0.18892, "alpha_p50": 0.41111, "alpha_p95": 0.74205}
{"step": 54210, "total_steps": 72479, "tokens": 28421652480, "loss": 4.30444198846817, "l_mix": 2.6671217679977417, "l_a": 2.7528411746025085, "l_c": 2.7048929035663605, "z_loss": 41.294275760650635, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1338745653629303, "tokens_per_second": 1442304.0255217557, "eta_seconds": 6640.914330482483, "data_wait_seconds": 1.582503318786621e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 54220, "total_steps": 72479, "tokens": 28426895360, "loss": 4.382495045661926, "l_mix": 2.7161123156547546, "l_a": 2.8005219995975494, "l_c": 2.754086822271347, "z_loss": 41.70494365692139, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12157399207353592, "tokens_per_second": 1439789.5451642275, "eta_seconds": 6648.8707493066795, "data_wait_seconds": 1.6391277313232422e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 54230, "total_steps": 72479, "tokens": 28432138240, "loss": 4.772104442119598, "l_mix": 2.959779590368271, "l_a": 3.0408680737018585, "l_c": 3.0002144277095795, "z_loss": 39.7601637840271, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14785601198673248, "tokens_per_second": 1211521.1164696373, "eta_seconds": 7897.288443374634, "data_wait_seconds": 1.8894672393798828e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 54240, "total_steps": 72479, "tokens": 28437381120, "loss": 4.5475404262542725, "l_mix": 2.821168154478073, "l_a": 2.896162450313568, "l_c": 2.858411341905594, "z_loss": 40.17525100708008, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12259935587644577, "tokens_per_second": 1318955.484404821, "eta_seconds": 7250.046680927277, "data_wait_seconds": 3.0487775802612305e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 54250, "total_steps": 72479, "tokens": 28442624000, "loss": 4.71357524394989, "l_mix": 2.9245907962322235, "l_a": 3.0027797520160675, "l_c": 2.960501492023468, "z_loss": 39.997690200805664, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16178086400032043, "tokens_per_second": 1410450.2518496169, "eta_seconds": 6776.024847006798, "data_wait_seconds": 3.1560659408569336e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 54260, "total_steps": 72479, "tokens": 28447866880, "loss": 4.607816278934479, "l_mix": 2.857999324798584, "l_a": 2.9343865513801575, "l_c": 2.8983360826969147, "z_loss": 40.35760450363159, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15298263728618622, "tokens_per_second": 1385705.2907306876, "eta_seconds": 6893.242838788032, "data_wait_seconds": 2.60770320892334e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 54270, "total_steps": 72479, "tokens": 28453109760, "loss": 4.626405417919159, "l_mix": 2.8703921735286713, "l_a": 2.9466339349746704, "l_c": 2.906743198633194, "z_loss": 39.38414287567139, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13259242475032806, "tokens_per_second": 1255586.2035614306, "eta_seconds": 7603.428713154793, "data_wait_seconds": 3.460049629211426e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 54280, "total_steps": 72479, "tokens": 28458352640, "loss": 4.325972259044647, "l_mix": 2.6807445287704468, "l_a": 2.767311453819275, "l_c": 2.716780811548233, "z_loss": 41.53813934326172, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12308091670274734, "tokens_per_second": 1338946.206016402, "eta_seconds": 7126.139399123192, "data_wait_seconds": 1.913309097290039e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 54290, "total_steps": 72479, "tokens": 28463595520, "loss": 4.337879836559296, "l_mix": 2.688423752784729, "l_a": 2.774047404527664, "l_c": 2.724139451980591, "z_loss": 40.452967166900635, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.10462921112775803, "tokens_per_second": 1348878.2840388657, "eta_seconds": 7069.781272959709, "data_wait_seconds": 1.7881393432617188e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 54300, "total_steps": 72479, "tokens": 28468838400, "loss": 4.350448727607727, "l_mix": 2.696057766675949, "l_a": 2.779963195323944, "l_c": 2.7346731424331665, "z_loss": 41.020862102508545, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12492010742425919, "tokens_per_second": 1325322.3730022814, "eta_seconds": 7191.481669783592, "data_wait_seconds": 1.5348196029663086e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.17095509, "gate_a2b/L1_absmax": 0.70271248, "gate_a2b/L1_rms": 0.18182932, "gate_a2b/L3_absmean": 0.1638598, "gate_a2b/L3_absmax": 0.52893078, "gate_a2b/L3_rms": 0.16981415, "gate_a2b/L5_absmean": 0.2456743, "gate_a2b/L5_absmax": 0.7355718, "gate_a2b/L5_rms": 0.25172025, "gate_c2b/L1_absmean": 0.02725926, "gate_c2b/L1_absmax": 0.62853956, "gate_c2b/L1_rms": 0.0667406, "gate_c2b/L3_absmean": 0.17685305, "gate_c2b/L3_absmax": 0.3997252, "gate_c2b/L3_rms": 0.18054523, "gate_c2b/L5_absmean": 0.50259638, "gate_c2b/L5_absmax": 0.70934886, "gate_c2b/L5_rms": 0.51277137, "gate_b2a/L7_absmean": 0.05065973, "gate_b2a/L7_absmax": 0.32774723, "gate_b2a/L7_rms": 0.05643763, "gate_b2a/L11_absmean": 0.08203779, "gate_b2a/L11_absmax": 0.60387397, "gate_b2a/L11_rms": 0.09113098, "gate_b2a/L14_absmean": 0.2122941, "gate_b2a/L14_absmax": 0.80623746, "gate_b2a/L14_rms": 0.22126065, "gate_b2c/L3_absmean": 0.03541171, "gate_b2c/L3_absmax": 0.55828959, "gate_b2c/L3_rms": 0.04619212, "gate_b2c/L5_absmean": 0.03750621, "gate_b2c/L5_absmax": 0.55853885, "gate_b2c/L5_rms": 0.04744497, "gate_b2c/L6_absmean": 0.05382447, "gate_b2c/L6_absmax": 0.34810108, "gate_b2c/L6_rms": 0.05998316}, "streams": {"cos_a2b/round0": 0.357972, "cos_c2b/round0": 0.278741, "cos_b2a/round0": 0.15432, "cos_b2c/round0": 0.139941, "rms_a/round0": 6.872041, "rms_b/round0": 12.047036, "rms_c/round0": 4.543535, "cos_a2b/round1": 0.408312, "cos_c2b/round1": 0.296702, "cos_b2a/round1": 0.24674, "cos_b2c/round1": 0.109129, "rms_a/round1": 13.452934, "rms_b/round1": 28.089127, "rms_c/round1": 9.632712, "cos_a2b/round2": 0.530852, "cos_c2b/round2": 0.523927, "cos_b2a/round2": 0.096284, "cos_b2c/round2": 0.180257, "rms_a/round2": 37.502541, "rms_b/round2": 75.382042, "rms_c/round2": 18.736481}, "alpha_mean": 0.416168491110092, "alpha_hist": [0.01385, 0.09016, 0.19785, 0.19748, 0.17282, 0.14833, 0.11467, 0.05692, 0.00786, 6e-05], "alpha_by_type": {"special": {"share": 0.00127, "mean": 0.55418, "std": 0.11428}, "whitespace": {"share": 0.02765, "mean": 0.51367, "std": 0.13486}, "digit": {"share": 0.04375, "mean": 0.43425, "std": 0.12459}, "word_start": {"share": 0.56671, "mean": 0.43259, "std": 0.17089}, "word_cont": {"share": 0.25523, "mean": 0.3047, "std": 0.12927}, "punct": {"share": 0.10539, "mean": 0.56306, "std": 0.16295}}, "alpha_p05": 0.14391, "alpha_p50": 0.37205, "alpha_p95": 0.71243}
{"step": 54310, "total_steps": 72479, "tokens": 28474081280, "loss": 4.365122497081757, "l_mix": 2.7052632868289948, "l_a": 2.7923799455165863, "l_c": 2.740483671426773, "z_loss": 41.450308322906494, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13484632968902588, "tokens_per_second": 1431630.2271378753, "eta_seconds": 6653.805215501785, "data_wait_seconds": 2.9534101486206055e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 54320, "total_steps": 72479, "tokens": 28479324160, "loss": 4.411224067211151, "l_mix": 2.734052389860153, "l_a": 2.8189836740493774, "l_c": 2.7715881764888763, "z_loss": 39.802629470825195, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13463979959487915, "tokens_per_second": 1124003.2655307725, "eta_seconds": 8470.21186144352, "data_wait_seconds": 2.709031105041504e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 54330, "total_steps": 72479, "tokens": 28484567040, "loss": 4.348591923713684, "l_mix": 2.6948831379413605, "l_a": 2.7808029651641846, "l_c": 2.7315592169761658, "z_loss": 41.29392433166504, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12294957041740417, "tokens_per_second": 1374810.9142652224, "eta_seconds": 6921.172077751159, "data_wait_seconds": 1.96993350982666e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 54340, "total_steps": 72479, "tokens": 28489809920, "loss": 3.1548358500003815, "l_mix": 1.941746011376381, "l_a": 2.055509924888611, "l_c": 1.988122582435608, "z_loss": 42.727205753326416, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16933006048202515, "tokens_per_second": 1460262.6529537567, "eta_seconds": 6512.5681416034695, "data_wait_seconds": 1.6957521438598633e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 54350, "total_steps": 72479, "tokens": 28495052800, "loss": 3.0863407850265503, "l_mix": 1.8989674896001816, "l_a": 2.0142644941806793, "l_c": 1.9436463564634323, "z_loss": 43.87707233428955, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1880427449941635, "tokens_per_second": 1399489.1094945772, "eta_seconds": 6791.633523631096, "data_wait_seconds": 2.8431415557861328e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 54360, "total_steps": 72479, "tokens": 28500295680, "loss": 4.567229509353638, "l_mix": 2.833233416080475, "l_a": 2.9088951349258423, "l_c": 2.871091604232788, "z_loss": 40.00775671005249, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19421234726905823, "tokens_per_second": 1474209.4546924047, "eta_seconds": 6443.842997860908, "data_wait_seconds": 3.141164779663086e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 54370, "total_steps": 72479, "tokens": 28505538560, "loss": 4.599892199039459, "l_mix": 2.8536119163036346, "l_a": 2.9298655688762665, "l_c": 2.8910681009292603, "z_loss": 39.40913963317871, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.145774245262146, "tokens_per_second": 1461951.7651115782, "eta_seconds": 6494.284981608391, "data_wait_seconds": 2.0891427993774414e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 54380, "total_steps": 72479, "tokens": 28510781440, "loss": 4.668091177940369, "l_mix": 2.8953919112682343, "l_a": 2.9752089381217957, "l_c": 2.9337884187698364, "z_loss": 39.318716049194336, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16176436841487885, "tokens_per_second": 1405978.623054468, "eta_seconds": 6749.098710608482, "data_wait_seconds": 2.3603439331054688e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 54390, "total_steps": 72479, "tokens": 28516024320, "loss": 4.350756466388702, "l_mix": 2.6960291862487793, "l_a": 2.781644731760025, "l_c": 2.7341125905513763, "z_loss": 40.29324817657471, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11578840017318726, "tokens_per_second": 1235441.304378506, "eta_seconds": 7676.484182929992, "data_wait_seconds": 2.1457672119140625e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 54400, "total_steps": 72479, "tokens": 28521267200, "loss": 4.390041291713715, "l_mix": 2.7207145988941193, "l_a": 2.806855261325836, "l_c": 2.757566660642624, "z_loss": 40.75646638870239, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15611068904399872, "tokens_per_second": 1306825.5899250815, "eta_seconds": 7253.150554347038, "data_wait_seconds": 1.9371509552001953e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.17065261, "gate_a2b/L1_absmax": 0.69981736, "gate_a2b/L1_rms": 0.18151402, "gate_a2b/L3_absmean": 0.16350861, "gate_a2b/L3_absmax": 0.52494186, "gate_a2b/L3_rms": 0.16940419, "gate_a2b/L5_absmean": 0.24526773, "gate_a2b/L5_absmax": 0.73037881, "gate_a2b/L5_rms": 0.25129455, "gate_c2b/L1_absmean": 0.02857653, "gate_c2b/L1_absmax": 0.62868643, "gate_c2b/L1_rms": 0.06735093, "gate_c2b/L3_absmean": 0.17675219, "gate_c2b/L3_absmax": 0.40889671, "gate_c2b/L3_rms": 0.18047301, "gate_c2b/L5_absmean": 0.50231612, "gate_c2b/L5_absmax": 0.71660918, "gate_c2b/L5_rms": 0.51244545, "gate_b2a/L7_absmean": 0.05103738, "gate_b2a/L7_absmax": 0.32678273, "gate_b2a/L7_rms": 0.05671584, "gate_b2a/L11_absmean": 0.08254746, "gate_b2a/L11_absmax": 0.60309631, "gate_b2a/L11_rms": 0.09152303, "gate_b2a/L14_absmean": 0.21200483, "gate_b2a/L14_absmax": 0.80831277, "gate_b2a/L14_rms": 0.22103649, "gate_b2c/L3_absmean": 0.03532613, "gate_b2c/L3_absmax": 0.55760831, "gate_b2c/L3_rms": 0.04611463, "gate_b2c/L5_absmean": 0.03758974, "gate_b2c/L5_absmax": 0.55769444, "gate_b2c/L5_rms": 0.04745202, "gate_b2c/L6_absmean": 0.05382056, "gate_b2c/L6_absmax": 0.35273129, "gate_b2c/L6_rms": 0.05995642}, "streams": {"cos_a2b/round0": 0.358789, "cos_c2b/round0": 0.272559, "cos_b2a/round0": 0.157441, "cos_b2c/round0": 0.138071, "rms_a/round0": 6.938362, "rms_b/round0": 12.263705, "rms_c/round0": 4.598296, "cos_a2b/round1": 0.41142, "cos_c2b/round1": 0.300823, "cos_b2a/round1": 0.253093, "cos_b2c/round1": 0.116946, "rms_a/round1": 13.608008, "rms_b/round1": 28.112278, "rms_c/round1": 9.770775, "cos_a2b/round2": 0.541433, "cos_c2b/round2": 0.518674, "cos_b2a/round2": 0.098743, "cos_b2c/round2": 0.185111, "rms_a/round2": 38.100246, "rms_b/round2": 75.565315, "rms_c/round2": 18.984461}, "alpha_mean": 0.43584631925878625, "alpha_hist": [0.00841, 0.07779, 0.17046, 0.19498, 0.18401, 0.16135, 0.12029, 0.07066, 0.01198, 9e-05], "alpha_by_type": {"special": {"share": 0.0014, "mean": 0.57758, "std": 0.10043}, "whitespace": {"share": 0.02866, "mean": 0.53935, "std": 0.11242}, "digit": {"share": 0.02917, "mean": 0.458, "std": 0.10598}, "word_start": {"share": 0.56804, "mean": 0.44861, "std": 0.17127}, "word_cont": {"share": 0.26562, "mean": 0.33148, "std": 0.1321}, "punct": {"share": 0.1071, "mean": 0.5914, "std": 0.16139}}, "alpha_p05": 0.17753, "alpha_p50": 0.43182, "alpha_p95": 0.73048}
{"step": 54410, "total_steps": 72479, "tokens": 28526510080, "loss": 4.347175121307373, "l_mix": 2.6935576796531677, "l_a": 2.7822846472263336, "l_c": 2.7297732532024384, "z_loss": 40.7820200920105, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.10337609052658081, "tokens_per_second": 1454796.1160543223, "eta_seconds": 6511.812732696533, "data_wait_seconds": 1.9818544387817383e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 54420, "total_steps": 72479, "tokens": 28531752960, "loss": 4.355430424213409, "l_mix": 2.699458599090576, "l_a": 2.7853018939495087, "l_c": 2.7346039414405823, "z_loss": 39.694597244262695, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14823278784751892, "tokens_per_second": 1445774.0881574932, "eta_seconds": 6548.821886873246, "data_wait_seconds": 1.77919864654541e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 54430, "total_steps": 72479, "tokens": 28536995840, "loss": 4.387266933917999, "l_mix": 2.719670593738556, "l_a": 2.803023248910904, "l_c": 2.755630999803543, "z_loss": 40.42436456680298, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13279075920581818, "tokens_per_second": 1383003.659450617, "eta_seconds": 6842.262525725365, "data_wait_seconds": 1.7195940017700195e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 54440, "total_steps": 72479, "tokens": 28542238720, "loss": 4.462369799613953, "l_mix": 2.7657161355018616, "l_a": 2.8539203703403473, "l_c": 2.8015914261341095, "z_loss": 41.1081280708313, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1967223584651947, "tokens_per_second": 1450443.3504979408, "eta_seconds": 6520.510593366623, "data_wait_seconds": 1.6748905181884766e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 54450, "total_steps": 72479, "tokens": 28547481600, "loss": 3.133617252111435, "l_mix": 1.9284652024507523, "l_a": 2.0423340648412704, "l_c": 1.9748392850160599, "z_loss": 43.87328577041626, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20155875384807587, "tokens_per_second": 1433056.073836733, "eta_seconds": 6595.96545073986, "data_wait_seconds": 1.665949821472168e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 54460, "total_steps": 72479, "tokens": 28552724480, "loss": 3.139121413230896, "l_mix": 1.931240826845169, "l_a": 2.045757457613945, "l_c": 1.9805108606815338, "z_loss": 43.45341873168945, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20437273383140564, "tokens_per_second": 1452175.2090307833, "eta_seconds": 6505.513531184197, "data_wait_seconds": 2.4527311325073242e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 54470, "total_steps": 72479, "tokens": 28557967360, "loss": 3.085444688796997, "l_mix": 1.8983064740896225, "l_a": 2.011385664343834, "l_c": 1.9457416534423828, "z_loss": 43.15449905395508, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18529707193374634, "tokens_per_second": 1477181.4195731252, "eta_seconds": 6391.836823081971, "data_wait_seconds": 2.0772218704223633e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 54480, "total_steps": 72479, "tokens": 28563210240, "loss": 3.0613551139831543, "l_mix": 1.8832827359437943, "l_a": 1.9985961616039276, "l_c": 1.928311511874199, "z_loss": 44.16155672073364, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16439759731292725, "tokens_per_second": 1480728.1169869232, "eta_seconds": 6372.986103081704, "data_wait_seconds": 1.7881393432617188e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 54490, "total_steps": 72479, "tokens": 28568453120, "loss": 4.399587094783783, "l_mix": 2.7269124388694763, "l_a": 2.813619703054428, "l_c": 2.7619621455669403, "z_loss": 39.929508686065674, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14006690680980682, "tokens_per_second": 1396126.26076328, "eta_seconds": 6755.4182576894755, "data_wait_seconds": 2.2679567337036133e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 54500, "total_steps": 72479, "tokens": 28573696000, "loss": 4.431240558624268, "l_mix": 2.7460362315177917, "l_a": 2.8347578644752502, "l_c": 2.7825895249843597, "z_loss": 40.31149387359619, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14649541676044464, "tokens_per_second": 1446621.5169958798, "eta_seconds": 6515.9918066024775, "data_wait_seconds": 2.396106719970703e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.170063, "gate_a2b/L1_absmax": 0.70232451, "gate_a2b/L1_rms": 0.18092164, "gate_a2b/L3_absmean": 0.1626621, "gate_a2b/L3_absmax": 0.52702981, "gate_a2b/L3_rms": 0.16870059, "gate_a2b/L5_absmean": 0.24565823, "gate_a2b/L5_absmax": 0.73689324, "gate_a2b/L5_rms": 0.25172424, "gate_c2b/L1_absmean": 0.02813043, "gate_c2b/L1_absmax": 0.63415951, "gate_c2b/L1_rms": 0.06748429, "gate_c2b/L3_absmean": 0.17635398, "gate_c2b/L3_absmax": 0.40199131, "gate_c2b/L3_rms": 0.18024267, "gate_c2b/L5_absmean": 0.50257683, "gate_c2b/L5_absmax": 0.71564704, "gate_c2b/L5_rms": 0.51269931, "gate_b2a/L7_absmean": 0.05032233, "gate_b2a/L7_absmax": 0.33201155, "gate_b2a/L7_rms": 0.05625283, "gate_b2a/L11_absmean": 0.08259073, "gate_b2a/L11_absmax": 0.60682619, "gate_b2a/L11_rms": 0.09178398, "gate_b2a/L14_absmean": 0.21275279, "gate_b2a/L14_absmax": 0.81227231, "gate_b2a/L14_rms": 0.22178262, "gate_b2c/L3_absmean": 0.03521981, "gate_b2c/L3_absmax": 0.55692053, "gate_b2c/L3_rms": 0.04612765, "gate_b2c/L5_absmean": 0.03766561, "gate_b2c/L5_absmax": 0.55774814, "gate_b2c/L5_rms": 0.04752143, "gate_b2c/L6_absmean": 0.0544441, "gate_b2c/L6_absmax": 0.3537702, "gate_b2c/L6_rms": 0.06071024}, "streams": {"cos_a2b/round0": 0.363333, "cos_c2b/round0": 0.281819, "cos_b2a/round0": 0.160056, "cos_b2c/round0": 0.14284, "rms_a/round0": 6.920784, "rms_b/round0": 12.027097, "rms_c/round0": 4.59371, "cos_a2b/round1": 0.424461, "cos_c2b/round1": 0.302565, "cos_b2a/round1": 0.258123, "cos_b2c/round1": 0.105393, "rms_a/round1": 13.371474, "rms_b/round1": 27.82003, "rms_c/round1": 9.456397, "cos_a2b/round2": 0.535597, "cos_c2b/round2": 0.523552, "cos_b2a/round2": 0.10497, "cos_b2c/round2": 0.180765, "rms_a/round2": 37.319897, "rms_b/round2": 75.494736, "rms_c/round2": 18.262564}, "alpha_mean": 0.4244555114433268, "alpha_hist": [0.00905, 0.09103, 0.19878, 0.18855, 0.16263, 0.14914, 0.11989, 0.07106, 0.00986, 2e-05], "alpha_by_type": {"special": {"share": 0.00041, "mean": 0.61668, "std": 0.09373}, "whitespace": {"share": 0.01926, "mean": 0.51937, "std": 0.14538}, "digit": {"share": 0.02028, "mean": 0.51735, "std": 0.13726}, "word_start": {"share": 0.62216, "mean": 0.43119, "std": 0.17473}, "word_cont": {"share": 0.24202, "mean": 0.32662, "std": 0.13639}, "punct": {"share": 0.09587, "mean": 0.58818, "std": 0.16601}}, "alpha_p05": 0.17667, "alpha_p50": 0.40993, "alpha_p95": 0.72428}
{"step": 54510, "total_steps": 72479, "tokens": 28578938880, "loss": 4.6417518854141235, "l_mix": 2.879069447517395, "l_a": 2.9612927734851837, "l_c": 2.9143149852752686, "z_loss": 40.32997941970825, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16936004161834717, "tokens_per_second": 1387522.6096607654, "eta_seconds": 6789.749591398239, "data_wait_seconds": 1.6868114471435547e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 54520, "total_steps": 72479, "tokens": 28584181760, "loss": 4.402293503284454, "l_mix": 2.7287722527980804, "l_a": 2.813139170408249, "l_c": 2.7652646899223328, "z_loss": 41.80275535583496, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14009389281272888, "tokens_per_second": 1313963.1611121688, "eta_seconds": 7165.869234895706, "data_wait_seconds": 3.0875205993652344e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 54530, "total_steps": 72479, "tokens": 28589424640, "loss": 4.339273750782013, "l_mix": 2.688715487718582, "l_a": 2.772606462240219, "l_c": 2.7292541563510895, "z_loss": 41.677292823791504, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14382831752300262, "tokens_per_second": 1424898.7211957567, "eta_seconds": 6604.290657305717, "data_wait_seconds": 2.0712614059448242e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 54540, "total_steps": 72479, "tokens": 28594667520, "loss": 4.414360165596008, "l_mix": 2.7356315553188324, "l_a": 2.8240223824977875, "l_c": 2.7717391550540924, "z_loss": 40.57202625274658, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11022169142961502, "tokens_per_second": 1463042.795812615, "eta_seconds": 6428.521748590469, "data_wait_seconds": 1.5497207641601562e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 54550, "total_steps": 72479, "tokens": 28599910400, "loss": 4.349337995052338, "l_mix": 2.695790082216263, "l_a": 2.78100049495697, "l_c": 2.7308254837989807, "z_loss": 41.01534128189087, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.131918802857399, "tokens_per_second": 1407004.6049654004, "eta_seconds": 6680.830694389343, "data_wait_seconds": 2.79843807220459e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 54560, "total_steps": 72479, "tokens": 28605153280, "loss": 2.8518987894058228, "l_mix": 1.7651804983615875, "l_a": 1.8307455629110336, "l_c": 1.7916486710309982, "z_loss": 49.93247985839844, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16170042753219604, "tokens_per_second": 1473743.2236689639, "eta_seconds": 6374.731039381028, "data_wait_seconds": 1.9818544387817383e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 54570, "total_steps": 72479, "tokens": 28610396160, "loss": 3.1291639506816864, "l_mix": 1.9257265627384186, "l_a": 2.040237471461296, "l_c": 1.9712203592061996, "z_loss": 44.69533586502075, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.23515735566616058, "tokens_per_second": 1425320.512596615, "eta_seconds": 6587.622719955444, "data_wait_seconds": 1.8149614334106445e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 54580, "total_steps": 72479, "tokens": 28615639040, "loss": 3.1146644055843353, "l_mix": 1.9173443168401718, "l_a": 2.026674747467041, "l_c": 1.9643919616937637, "z_loss": 43.15945863723755, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18208444118499756, "tokens_per_second": 1385617.1035086617, "eta_seconds": 6772.6003729581835, "data_wait_seconds": 3.948807716369629e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 54590, "total_steps": 72479, "tokens": 28620881920, "loss": 3.1771570444107056, "l_mix": 1.9550261795520782, "l_a": 2.069342702627182, "l_c": 2.004426658153534, "z_loss": 42.50211429595947, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18420755863189697, "tokens_per_second": 1468465.7857113713, "eta_seconds": 6386.92989871502, "data_wait_seconds": 1.5437602996826172e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 54600, "total_steps": 72479, "tokens": 28626124800, "loss": 4.429999351501465, "l_mix": 2.74577334523201, "l_a": 2.834453523159027, "l_c": 2.779633104801178, "z_loss": 40.50977182388306, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20851550996303558, "tokens_per_second": 1470756.4380541863, "eta_seconds": 6373.417725372315, "data_wait_seconds": 2.6106834411621094e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.17012791, "gate_a2b/L1_absmax": 0.70596617, "gate_a2b/L1_rms": 0.18094964, "gate_a2b/L3_absmean": 0.16294137, "gate_a2b/L3_absmax": 0.52599144, "gate_a2b/L3_rms": 0.16895494, "gate_a2b/L5_absmean": 0.24448164, "gate_a2b/L5_absmax": 0.73469877, "gate_a2b/L5_rms": 0.25052616, "gate_c2b/L1_absmean": 0.02738224, "gate_c2b/L1_absmax": 0.63689959, "gate_c2b/L1_rms": 0.06733063, "gate_c2b/L3_absmean": 0.1765883, "gate_c2b/L3_absmax": 0.40330967, "gate_c2b/L3_rms": 0.18031566, "gate_c2b/L5_absmean": 0.50131983, "gate_c2b/L5_absmax": 0.70933282, "gate_c2b/L5_rms": 0.5114401, "gate_b2a/L7_absmean": 0.05101465, "gate_b2a/L7_absmax": 0.32919821, "gate_b2a/L7_rms": 0.05673615, "gate_b2a/L11_absmean": 0.08233923, "gate_b2a/L11_absmax": 0.60511678, "gate_b2a/L11_rms": 0.09142887, "gate_b2a/L14_absmean": 0.2128191, "gate_b2a/L14_absmax": 0.80891377, "gate_b2a/L14_rms": 0.22182797, "gate_b2c/L3_absmean": 0.03521209, "gate_b2c/L3_absmax": 0.55451089, "gate_b2c/L3_rms": 0.04588404, "gate_b2c/L5_absmean": 0.0374128, "gate_b2c/L5_absmax": 0.55338913, "gate_b2c/L5_rms": 0.04717888, "gate_b2c/L6_absmean": 0.0541953, "gate_b2c/L6_absmax": 0.35039589, "gate_b2c/L6_rms": 0.06036855}, "streams": {"cos_a2b/round0": 0.360367, "cos_c2b/round0": 0.280258, "cos_b2a/round0": 0.158106, "cos_b2c/round0": 0.142689, "rms_a/round0": 6.944315, "rms_b/round0": 12.15164, "rms_c/round0": 4.586385, "cos_a2b/round1": 0.417705, "cos_c2b/round1": 0.302663, "cos_b2a/round1": 0.252549, "cos_b2c/round1": 0.111177, "rms_a/round1": 13.458104, "rms_b/round1": 27.944006, "rms_c/round1": 9.593982, "cos_a2b/round2": 0.539206, "cos_c2b/round2": 0.522855, "cos_b2a/round2": 0.097827, "cos_b2c/round2": 0.184773, "rms_a/round2": 37.843781, "rms_b/round2": 76.123596, "rms_c/round2": 18.497597}, "alpha_mean": 0.4259397037174324, "alpha_hist": [0.00645, 0.06863, 0.19632, 0.21997, 0.17897, 0.13846, 0.11717, 0.06654, 0.00748, 0.0], "alpha_by_type": {"special": {"share": 0.00069, "mean": 0.53627, "std": 0.11491}, "whitespace": {"share": 0.02203, "mean": 0.48005, "std": 0.12149}, "digit": {"share": 0.03423, "mean": 0.41139, "std": 0.11742}, "word_start": {"share": 0.59377, "mean": 0.44356, "std": 0.16682}, "word_cont": {"share": 0.25182, "mean": 0.32006, "std": 0.12035}, "punct": {"share": 0.09747, "mean": 0.58425, "std": 0.15908}}, "alpha_p05": 0.1804, "alpha_p50": 0.40052, "alpha_p95": 0.72307}
{"step": 54610, "total_steps": 72479, "tokens": 28631367680, "loss": 4.378359377384186, "l_mix": 2.713286429643631, "l_a": 2.799274653196335, "l_c": 2.750968188047409, "z_loss": 40.590370178222656, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15634401142597198, "tokens_per_second": 1425451.9866967, "eta_seconds": 6572.302932286262, "data_wait_seconds": 3.662705421447754e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 54620, "total_steps": 72479, "tokens": 28636610560, "loss": 4.3992443680763245, "l_mix": 2.726683586835861, "l_a": 2.81559956073761, "l_c": 2.759602963924408, "z_loss": 40.91349172592163, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2528182566165924, "tokens_per_second": 1417222.8819151318, "eta_seconds": 6606.765605807304, "data_wait_seconds": 1.7911195755004883e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 54630, "total_steps": 72479, "tokens": 28641853440, "loss": 3.123276859521866, "l_mix": 1.9207563996315002, "l_a": 2.0361651480197906, "l_c": 1.9722363948822021, "z_loss": 44.24562406539917, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.181013822555542, "tokens_per_second": 1260053.536840619, "eta_seconds": 7426.681675338745, "data_wait_seconds": 3.740191459655762e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 54640, "total_steps": 72479, "tokens": 28647096320, "loss": 4.641695737838745, "l_mix": 2.8791057467460632, "l_a": 2.9561122357845306, "l_c": 2.9191872477531433, "z_loss": 40.5715389251709, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1553904265165329, "tokens_per_second": 1380375.9664660697, "eta_seconds": 6775.526276326179, "data_wait_seconds": 1.71661376953125e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 54650, "total_steps": 72479, "tokens": 28652339200, "loss": 4.719768047332764, "l_mix": 2.9282091856002808, "l_a": 3.006784588098526, "l_c": 2.965077817440033, "z_loss": 40.04516553878784, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1699560284614563, "tokens_per_second": 1371166.3511180063, "eta_seconds": 6817.211306548119, "data_wait_seconds": 1.5348196029663086e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 54660, "total_steps": 72479, "tokens": 28657582080, "loss": 4.8407434821128845, "l_mix": 3.004298210144043, "l_a": 3.078293204307556, "l_c": 3.043190836906433, "z_loss": 38.725457191467285, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1520678997039795, "tokens_per_second": 1352356.6028571015, "eta_seconds": 6908.154145336151, "data_wait_seconds": 3.439188003540039e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 54670, "total_steps": 72479, "tokens": 28662824960, "loss": 4.772388458251953, "l_mix": 2.9582835733890533, "l_a": 3.025642216205597, "l_c": 3.0213738083839417, "z_loss": 39.5250358581543, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.43466898798942566, "tokens_per_second": 1452154.8790019106, "eta_seconds": 6429.785918164253, "data_wait_seconds": 2.256035804748535e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 54680, "total_steps": 72479, "tokens": 28668067840, "loss": 3.1191601753234863, "l_mix": 1.9179594069719315, "l_a": 2.0343363136053085, "l_c": 1.9696660935878754, "z_loss": 44.603562355041504, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2260834276676178, "tokens_per_second": 1450676.0550947613, "eta_seconds": 6432.726368665695, "data_wait_seconds": 2.1219253540039062e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 54690, "total_steps": 72479, "tokens": 28673310720, "loss": 3.0857068598270416, "l_mix": 1.8994902968406677, "l_a": 2.0092262029647827, "l_c": 1.9448287189006805, "z_loss": 42.65094757080078, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19536176323890686, "tokens_per_second": 1443816.725295467, "eta_seconds": 6459.655902719498, "data_wait_seconds": 2.8192996978759766e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 54700, "total_steps": 72479, "tokens": 28678553600, "loss": 3.126617670059204, "l_mix": 1.925745353102684, "l_a": 2.032667428255081, "l_c": 1.9702400416135788, "z_loss": 42.97832632064819, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.23147213459014893, "tokens_per_second": 1433182.720751634, "eta_seconds": 6503.927389740944, "data_wait_seconds": 1.895427703857422e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.17020534, "gate_a2b/L1_absmax": 0.70427871, "gate_a2b/L1_rms": 0.18108442, "gate_a2b/L3_absmean": 0.16285753, "gate_a2b/L3_absmax": 0.52776325, "gate_a2b/L3_rms": 0.16888154, "gate_a2b/L5_absmean": 0.24429527, "gate_a2b/L5_absmax": 0.7348116, "gate_a2b/L5_rms": 0.25036153, "gate_c2b/L1_absmean": 0.02754183, "gate_c2b/L1_absmax": 0.63525993, "gate_c2b/L1_rms": 0.0672009, "gate_c2b/L3_absmean": 0.17607398, "gate_c2b/L3_absmax": 0.39311272, "gate_c2b/L3_rms": 0.17974699, "gate_c2b/L5_absmean": 0.50094932, "gate_c2b/L5_absmax": 0.7107082, "gate_c2b/L5_rms": 0.51109642, "gate_b2a/L7_absmean": 0.05074783, "gate_b2a/L7_absmax": 0.3275395, "gate_b2a/L7_rms": 0.05645819, "gate_b2a/L11_absmean": 0.08224276, "gate_b2a/L11_absmax": 0.60411537, "gate_b2a/L11_rms": 0.09126531, "gate_b2a/L14_absmean": 0.21232082, "gate_b2a/L14_absmax": 0.80754966, "gate_b2a/L14_rms": 0.22132163, "gate_b2c/L3_absmean": 0.03503458, "gate_b2c/L3_absmax": 0.55359638, "gate_b2c/L3_rms": 0.04574005, "gate_b2c/L5_absmean": 0.03770394, "gate_b2c/L5_absmax": 0.55489385, "gate_b2c/L5_rms": 0.04745653, "gate_b2c/L6_absmean": 0.05413517, "gate_b2c/L6_absmax": 0.3524974, "gate_b2c/L6_rms": 0.06036494}, "streams": {"cos_a2b/round0": 0.357728, "cos_c2b/round0": 0.243672, "cos_b2a/round0": 0.147465, "cos_b2c/round0": 0.121809, "rms_a/round0": 7.300661, "rms_b/round0": 12.497084, "rms_c/round0": 4.688911, "cos_a2b/round1": 0.412387, "cos_c2b/round1": 0.304644, "cos_b2a/round1": 0.243025, "cos_b2c/round1": 0.121432, "rms_a/round1": 14.445689, "rms_b/round1": 29.461603, "rms_c/round1": 10.364837, "cos_a2b/round2": 0.52312, "cos_c2b/round2": 0.513521, "cos_b2a/round2": 0.093094, "cos_b2c/round2": 0.178361, "rms_a/round2": 39.203125, "rms_b/round2": 77.52491, "rms_c/round2": 20.252258}, "alpha_mean": 0.42556168286566276, "alpha_hist": [0.02586, 0.14354, 0.19191, 0.15465, 0.11992, 0.11026, 0.10324, 0.0952, 0.05234, 0.00308], "alpha_by_type": {"special": {"share": 0.00114, "mean": 0.6606, "std": 0.11487}, "whitespace": {"share": 0.03819, "mean": 0.63758, "std": 0.16262}, "digit": {"share": 0.00969, "mean": 0.37158, "std": 0.17375}, "word_start": {"share": 0.57039, "mean": 0.42961, "std": 0.20767}, "word_cont": {"share": 0.25633, "mean": 0.28331, "std": 0.14249}, "punct": {"share": 0.12425, "mean": 0.63729, "std": 0.18153}}, "alpha_p05": 0.1188, "alpha_p50": 0.3928, "alpha_p95": 0.80865}
{"step": 54710, "total_steps": 72479, "tokens": 28683796480, "loss": 4.437291085720062, "l_mix": 2.75075900554657, "l_a": 2.8360166549682617, "l_c": 2.7857567965984344, "z_loss": 39.33052635192871, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13260763883590698, "tokens_per_second": 1469022.7892205731, "eta_seconds": 6341.680701184272, "data_wait_seconds": 1.6003847122192383e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 54720, "total_steps": 72479, "tokens": 28689039360, "loss": 4.3353031277656555, "l_mix": 2.6868177950382233, "l_a": 2.774381071329117, "l_c": 2.7205697894096375, "z_loss": 40.44777536392212, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.166878804564476, "tokens_per_second": 1262098.7910162704, "eta_seconds": 7377.259734559058, "data_wait_seconds": 2.1398067474365234e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 54730, "total_steps": 72479, "tokens": 28694282240, "loss": 4.377024531364441, "l_mix": 2.7123310565948486, "l_a": 2.8000082075595856, "l_c": 2.7489698827266693, "z_loss": 40.46873998641968, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18304753303527832, "tokens_per_second": 1251339.1039345919, "eta_seconds": 7436.503568649292, "data_wait_seconds": 1.6391277313232422e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 54740, "total_steps": 72479, "tokens": 28699525120, "loss": 3.1391358375549316, "l_mix": 1.9320964515209198, "l_a": 2.046709269285202, "l_c": 1.9767551571130753, "z_loss": 43.909961223602295, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19710609316825867, "tokens_per_second": 1237975.958047302, "eta_seconds": 7512.540749716758, "data_wait_seconds": 1.7970800399780273e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 54750, "total_steps": 72479, "tokens": 28704768000, "loss": 3.1538472175598145, "l_mix": 1.9413280934095383, "l_a": 2.0535509139299393, "l_c": 1.9881790280342102, "z_loss": 44.017415046691895, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19491486251354218, "tokens_per_second": 1396767.2298066847, "eta_seconds": 6654.725106406212, "data_wait_seconds": 3.656744956970215e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 54760, "total_steps": 72479, "tokens": 28710010880, "loss": 3.1023247838020325, "l_mix": 1.9089364260435104, "l_a": 2.023106575012207, "l_c": 1.9548545181751251, "z_loss": 42.62765645980835, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2060735672712326, "tokens_per_second": 1471032.4107036244, "eta_seconds": 6315.196731495857, "data_wait_seconds": 1.8030405044555664e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 54770, "total_steps": 72479, "tokens": 28715253760, "loss": 3.1113827526569366, "l_mix": 1.9150721728801727, "l_a": 2.0253914147615433, "l_c": 1.9623103588819504, "z_loss": 41.68465614318848, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20608384907245636, "tokens_per_second": 1340414.0595525221, "eta_seconds": 6926.677712631226, "data_wait_seconds": 2.1070241928100586e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 54780, "total_steps": 72479, "tokens": 28720496640, "loss": 4.787805676460266, "l_mix": 2.970641106367111, "l_a": 3.046736419200897, "l_c": 3.0104787051677704, "z_loss": 37.5931453704834, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2656387686729431, "tokens_per_second": 1337795.5586216848, "eta_seconds": 6936.316429066658, "data_wait_seconds": 3.248453140258789e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 54790, "total_steps": 72479, "tokens": 28725739520, "loss": 3.138520985841751, "l_mix": 1.931627780199051, "l_a": 2.04608690738678, "l_c": 1.9768902957439423, "z_loss": 42.30153942108154, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1756807565689087, "tokens_per_second": 1446678.8089597614, "eta_seconds": 6410.635432386399, "data_wait_seconds": 1.558661460876465e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 54800, "total_steps": 72479, "tokens": 28730982400, "loss": 3.11347234249115, "l_mix": 1.914489045739174, "l_a": 2.032179072499275, "l_c": 1.9644315987825394, "z_loss": 43.908881187438965, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14990904927253723, "tokens_per_second": 1444155.324553566, "eta_seconds": 6418.206819176674, "data_wait_seconds": 2.396106719970703e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16922759, "gate_a2b/L1_absmax": 0.69948345, "gate_a2b/L1_rms": 0.18013717, "gate_a2b/L3_absmean": 0.16241276, "gate_a2b/L3_absmax": 0.52954185, "gate_a2b/L3_rms": 0.16838035, "gate_a2b/L5_absmean": 0.24354303, "gate_a2b/L5_absmax": 0.734972, "gate_a2b/L5_rms": 0.24967414, "gate_c2b/L1_absmean": 0.02871812, "gate_c2b/L1_absmax": 0.63263148, "gate_c2b/L1_rms": 0.06757581, "gate_c2b/L3_absmean": 0.17634358, "gate_c2b/L3_absmax": 0.40385258, "gate_c2b/L3_rms": 0.18018983, "gate_c2b/L5_absmean": 0.50082034, "gate_c2b/L5_absmax": 0.70125639, "gate_c2b/L5_rms": 0.51080924, "gate_b2a/L7_absmean": 0.05119073, "gate_b2a/L7_absmax": 0.33124435, "gate_b2a/L7_rms": 0.05707618, "gate_b2a/L11_absmean": 0.08264472, "gate_b2a/L11_absmax": 0.61045372, "gate_b2a/L11_rms": 0.09189791, "gate_b2a/L14_absmean": 0.21137711, "gate_b2a/L14_absmax": 0.81303072, "gate_b2a/L14_rms": 0.22068809, "gate_b2c/L3_absmean": 0.03468806, "gate_b2c/L3_absmax": 0.54992002, "gate_b2c/L3_rms": 0.0454956, "gate_b2c/L5_absmean": 0.03711342, "gate_b2c/L5_absmax": 0.54768729, "gate_b2c/L5_rms": 0.04688277, "gate_b2c/L6_absmean": 0.05401021, "gate_b2c/L6_absmax": 0.35612351, "gate_b2c/L6_rms": 0.06045618}, "streams": {"cos_a2b/round0": 0.353095, "cos_c2b/round0": 0.237367, "cos_b2a/round0": 0.143501, "cos_b2c/round0": 0.119081, "rms_a/round0": 7.240412, "rms_b/round0": 12.312197, "rms_c/round0": 4.711444, "cos_a2b/round1": 0.409786, "cos_c2b/round1": 0.303403, "cos_b2a/round1": 0.238214, "cos_b2c/round1": 0.119315, "rms_a/round1": 14.393669, "rms_b/round1": 29.707764, "rms_c/round1": 10.25876, "cos_a2b/round2": 0.517554, "cos_c2b/round2": 0.509684, "cos_b2a/round2": 0.088744, "cos_b2c/round2": 0.171324, "rms_a/round2": 38.989372, "rms_b/round2": 77.507545, "rms_c/round2": 19.881981}, "alpha_mean": 0.4134373870764705, "alpha_hist": [0.04416, 0.16953, 0.1835, 0.13661, 0.10905, 0.10393, 0.10287, 0.09343, 0.05209, 0.00482], "alpha_by_type": {"special": {"share": 0.00113, "mean": 0.66939, "std": 0.11152}, "whitespace": {"share": 0.0354, "mean": 0.63817, "std": 0.15972}, "digit": {"share": 0.01004, "mean": 0.44047, "std": 0.19272}, "word_start": {"share": 0.57024, "mean": 0.4221, "std": 0.21678}, "word_cont": {"share": 0.26208, "mean": 0.26432, "std": 0.15206}, "punct": {"share": 0.12111, "mean": 0.62502, "std": 0.19488}}, "alpha_p05": 0.09401, "alpha_p50": 0.36886, "alpha_p95": 0.81168}
{"step": 54810, "total_steps": 72479, "tokens": 28736225280, "loss": 3.1306367218494415, "l_mix": 1.9264275133609772, "l_a": 2.0380584001541138, "l_c": 1.9759721606969833, "z_loss": 43.14407444000244, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1563020795583725, "tokens_per_second": 1340710.5505134643, "eta_seconds": 6909.503821277618, "data_wait_seconds": 1.9490718841552734e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 54820, "total_steps": 72479, "tokens": 28741468160, "loss": 4.4257601499557495, "l_mix": 2.7425723373889923, "l_a": 2.832068979740143, "l_c": 2.778556764125824, "z_loss": 40.728429317474365, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11933455616235733, "tokens_per_second": 1472436.9977735567, "eta_seconds": 6287.80844681263, "data_wait_seconds": 1.806020736694336e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 54830, "total_steps": 72479, "tokens": 28746711040, "loss": 4.444555282592773, "l_mix": 2.7544430792331696, "l_a": 2.8408908247947693, "l_c": 2.7928161323070526, "z_loss": 39.969101428985596, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14980368316173553, "tokens_per_second": 1444940.279095904, "eta_seconds": 6403.8348476171495, "data_wait_seconds": 2.1129846572875977e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 54840, "total_steps": 72479, "tokens": 28751953920, "loss": 4.3775336146354675, "l_mix": 2.7138079702854156, "l_a": 2.797652304172516, "l_c": 2.74809929728508, "z_loss": 40.42133665084839, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18147453665733337, "tokens_per_second": 1412388.7052694515, "eta_seconds": 6547.713102984429, "data_wait_seconds": 1.9937753677368164e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 54850, "total_steps": 72479, "tokens": 28757196800, "loss": 3.119281768798828, "l_mix": 1.919520154595375, "l_a": 2.03469917178154, "l_c": 1.9645060449838638, "z_loss": 43.82912540435791, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1909477263689041, "tokens_per_second": 1384577.951576677, "eta_seconds": 6675.44441356659, "data_wait_seconds": 1.6301870346069336e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 54860, "total_steps": 72479, "tokens": 28762439680, "loss": 3.1297850906848907, "l_mix": 1.925767868757248, "l_a": 2.037759095430374, "l_c": 1.9756314754486084, "z_loss": 43.23440885543823, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17012165486812592, "tokens_per_second": 1454618.4710064332, "eta_seconds": 6350.414528703689, "data_wait_seconds": 3.471970558166504e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 54870, "total_steps": 72479, "tokens": 28767682560, "loss": 3.1910606920719147, "l_mix": 1.9646452367305756, "l_a": 2.075805813074112, "l_c": 2.012245461344719, "z_loss": 41.57875633239746, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2316933274269104, "tokens_per_second": 1346154.4499404184, "eta_seconds": 6858.193272256852, "data_wait_seconds": 2.7120113372802734e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 54880, "total_steps": 72479, "tokens": 28772925440, "loss": 3.1520502269268036, "l_mix": 1.939894661307335, "l_a": 2.0533725023269653, "l_c": 1.9871457666158676, "z_loss": 45.27119588851929, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2785584330558777, "tokens_per_second": 1454850.0147878416, "eta_seconds": 6342.1963901519775, "data_wait_seconds": 2.4974346160888672e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 54890, "total_steps": 72479, "tokens": 28778168320, "loss": 3.0865074396133423, "l_mix": 1.8996950387954712, "l_a": 2.0092491507530212, "l_c": 1.9467919617891312, "z_loss": 42.49980688095093, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1927158534526825, "tokens_per_second": 1424666.55157504, "eta_seconds": 6472.88421406746, "data_wait_seconds": 2.3484230041503906e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 54900, "total_steps": 72479, "tokens": 28783411200, "loss": 4.704557478427887, "l_mix": 2.9193248748779297, "l_a": 2.9960569739341736, "l_c": 2.9547181129455566, "z_loss": 39.196834087371826, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1459989845752716, "tokens_per_second": 1433016.851456113, "eta_seconds": 6431.507586693763, "data_wait_seconds": 1.6808509826660156e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.16915007, "gate_a2b/L1_absmax": 0.70036733, "gate_a2b/L1_rms": 0.18000028, "gate_a2b/L3_absmean": 0.16247992, "gate_a2b/L3_absmax": 0.52633381, "gate_a2b/L3_rms": 0.16840388, "gate_a2b/L5_absmean": 0.24291091, "gate_a2b/L5_absmax": 0.73375136, "gate_a2b/L5_rms": 0.24901678, "gate_c2b/L1_absmean": 0.02840955, "gate_c2b/L1_absmax": 0.63324755, "gate_c2b/L1_rms": 0.06744007, "gate_c2b/L3_absmean": 0.17606513, "gate_c2b/L3_absmax": 0.40249172, "gate_c2b/L3_rms": 0.17985618, "gate_c2b/L5_absmean": 0.50058794, "gate_c2b/L5_absmax": 0.70457864, "gate_c2b/L5_rms": 0.51066738, "gate_b2a/L7_absmean": 0.05082397, "gate_b2a/L7_absmax": 0.34004286, "gate_b2a/L7_rms": 0.05680653, "gate_b2a/L11_absmean": 0.08234663, "gate_b2a/L11_absmax": 0.61686367, "gate_b2a/L11_rms": 0.09172546, "gate_b2a/L14_absmean": 0.21212356, "gate_b2a/L14_absmax": 0.81844407, "gate_b2a/L14_rms": 0.22127722, "gate_b2c/L3_absmean": 0.03495802, "gate_b2c/L3_absmax": 0.54910022, "gate_b2c/L3_rms": 0.0455595, "gate_b2c/L5_absmean": 0.03734945, "gate_b2c/L5_absmax": 0.54457533, "gate_b2c/L5_rms": 0.04691182, "gate_b2c/L6_absmean": 0.05412772, "gate_b2c/L6_absmax": 0.35631731, "gate_b2c/L6_rms": 0.0605314}, "streams": {"cos_a2b/round0": 0.356075, "cos_c2b/round0": 0.272841, "cos_b2a/round0": 0.165047, "cos_b2c/round0": 0.143306, "rms_a/round0": 6.9423, "rms_b/round0": 12.206107, "rms_c/round0": 4.601737, "cos_a2b/round1": 0.411932, "cos_c2b/round1": 0.312348, "cos_b2a/round1": 0.260121, "cos_b2c/round1": 0.116547, "rms_a/round1": 13.348722, "rms_b/round1": 27.594549, "rms_c/round1": 9.604053, "cos_a2b/round2": 0.553433, "cos_c2b/round2": 0.519793, "cos_b2a/round2": 0.104401, "cos_b2c/round2": 0.193313, "rms_a/round2": 37.749519, "rms_b/round2": 74.376068, "rms_c/round2": 18.191553}, "alpha_mean": 0.4508893047949627, "alpha_hist": [0.00444, 0.04155, 0.14575, 0.19482, 0.22357, 0.2021, 0.13039, 0.0542, 0.00314, 3e-05], "alpha_by_type": {"special": {"share": 0.00052, "mean": 0.5162, "std": 0.1471}, "whitespace": {"share": 0.04619, "mean": 0.5288, "std": 0.10981}, "digit": {"share": 0.0296, "mean": 0.4535, "std": 0.12557}, "word_start": {"share": 0.57956, "mean": 0.46676, "std": 0.14523}, "word_cont": {"share": 0.23944, "mean": 0.33899, "std": 0.11947}, "punct": {"share": 0.10469, "mean": 0.58353, "std": 0.14052}}, "alpha_p05": 0.2015, "alpha_p50": 0.45786, "alpha_p95": 0.71303}
{"step": 54910, "total_steps": 72479, "tokens": 28788654080, "loss": 4.72194904088974, "l_mix": 2.9290598034858704, "l_a": 3.004081606864929, "l_c": 2.9722158014774323, "z_loss": 40.103941440582275, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18595613539218903, "tokens_per_second": 1425786.1860428355, "eta_seconds": 6460.446848320961, "data_wait_seconds": 2.0384788513183594e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 54920, "total_steps": 72479, "tokens": 28793896960, "loss": 3.1557597517967224, "l_mix": 1.9426337033510208, "l_a": 2.054537370800972, "l_c": 1.9892160445451736, "z_loss": 43.611618518829346, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.22991952300071716, "tokens_per_second": 1421144.5692991945, "eta_seconds": 6477.858193230629, "data_wait_seconds": 2.1159648895263672e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 54930, "total_steps": 72479, "tokens": 28799139840, "loss": 4.464245319366455, "l_mix": 2.7669710218906403, "l_a": 2.852569878101349, "l_c": 2.8050109446048737, "z_loss": 39.8663330078125, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.160091370344162, "tokens_per_second": 1475427.650128879, "eta_seconds": 6235.975116229057, "data_wait_seconds": 1.8030405044555664e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 54940, "total_steps": 72479, "tokens": 28804382720, "loss": 4.344465374946594, "l_mix": 2.692290812730789, "l_a": 2.778386026620865, "l_c": 2.7288620471954346, "z_loss": 40.0452561378479, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17963051795959473, "tokens_per_second": 1460820.1455476978, "eta_seconds": 6294.74289495945, "data_wait_seconds": 1.5497207641601562e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 54950, "total_steps": 72479, "tokens": 28809625600, "loss": 4.210554838180542, "l_mix": 2.609209716320038, "l_a": 2.694922238588333, "l_c": 2.642894685268402, "z_loss": 40.23096323013306, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20720969140529633, "tokens_per_second": 1445497.6285294436, "eta_seconds": 6357.841182589531, "data_wait_seconds": 3.4332275390625e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 54960, "total_steps": 72479, "tokens": 28814868480, "loss": 3.1634840071201324, "l_mix": 1.9466910362243652, "l_a": 2.060461461544037, "l_c": 1.9955150038003922, "z_loss": 43.60824728012085, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19490133225917816, "tokens_per_second": 1453884.9660244759, "eta_seconds": 6317.557225394249, "data_wait_seconds": 1.9848346710205078e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 54970, "total_steps": 72479, "tokens": 28820111360, "loss": 3.0960873067379, "l_mix": 1.9052098095417023, "l_a": 2.017096996307373, "l_c": 1.9524946063756943, "z_loss": 41.53073072433472, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.26630041003227234, "tokens_per_second": 1438105.236637093, "eta_seconds": 6383.231461882591, "data_wait_seconds": 3.30805778503418e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 54980, "total_steps": 72479, "tokens": 28825354240, "loss": 3.102856010198593, "l_mix": 1.9089691191911697, "l_a": 2.0269514471292496, "l_c": 1.95267154276371, "z_loss": 42.89479207992554, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20781122148036957, "tokens_per_second": 1363212.8085054653, "eta_seconds": 6730.068595862389, "data_wait_seconds": 1.7315149307250977e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 54990, "total_steps": 72479, "tokens": 28830597120, "loss": 3.1359600126743317, "l_mix": 1.930175557732582, "l_a": 2.0409127175807953, "l_c": 1.978368416428566, "z_loss": 43.33625793457031, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15006011724472046, "tokens_per_second": 1413635.1384913863, "eta_seconds": 6486.307946324348, "data_wait_seconds": 2.22623348236084e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 55000, "total_steps": 72479, "tokens": 28835840000, "loss": 3.1021075546741486, "l_mix": 1.9093715101480484, "l_a": 2.0196783244609833, "l_c": 1.9561083912849426, "z_loss": 42.25188159942627, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1712132692337036, "tokens_per_second": 1449917.5524226357, "eta_seconds": 6320.380035877227, "data_wait_seconds": 2.1398067474365234e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.1692052, "gate_a2b/L1_absmax": 0.70074975, "gate_a2b/L1_rms": 0.18002561, "gate_a2b/L3_absmean": 0.16226152, "gate_a2b/L3_absmax": 0.52489793, "gate_a2b/L3_rms": 0.16819067, "gate_a2b/L5_absmean": 0.24267088, "gate_a2b/L5_absmax": 0.73645037, "gate_a2b/L5_rms": 0.24877138, "gate_c2b/L1_absmean": 0.02783241, "gate_c2b/L1_absmax": 0.63311613, "gate_c2b/L1_rms": 0.06725922, "gate_c2b/L3_absmean": 0.17598477, "gate_c2b/L3_absmax": 0.40295729, "gate_c2b/L3_rms": 0.17968765, "gate_c2b/L5_absmean": 0.500099, "gate_c2b/L5_absmax": 0.71028912, "gate_c2b/L5_rms": 0.5102722, "gate_b2a/L7_absmean": 0.05082967, "gate_b2a/L7_absmax": 0.33715054, "gate_b2a/L7_rms": 0.05677648, "gate_b2a/L11_absmean": 0.08205009, "gate_b2a/L11_absmax": 0.61530876, "gate_b2a/L11_rms": 0.09138869, "gate_b2a/L14_absmean": 0.21225885, "gate_b2a/L14_absmax": 0.81665057, "gate_b2a/L14_rms": 0.22152175, "gate_b2c/L3_absmean": 0.03493069, "gate_b2c/L3_absmax": 0.55000114, "gate_b2c/L3_rms": 0.0455544, "gate_b2c/L5_absmean": 0.03723491, "gate_b2c/L5_absmax": 0.54518241, "gate_b2c/L5_rms": 0.04683775, "gate_b2c/L6_absmean": 0.05375792, "gate_b2c/L6_absmax": 0.35540789, "gate_b2c/L6_rms": 0.06021039}, "streams": {"cos_a2b/round0": 0.35497, "cos_c2b/round0": 0.240567, "cos_b2a/round0": 0.146983, "cos_b2c/round0": 0.120371, "rms_a/round0": 7.269146, "rms_b/round0": 12.364411, "rms_c/round0": 4.718742, "cos_a2b/round1": 0.406538, "cos_c2b/round1": 0.307001, "cos_b2a/round1": 0.239874, "cos_b2c/round1": 0.120616, "rms_a/round1": 14.314834, "rms_b/round1": 29.518501, "rms_c/round1": 10.338234, "cos_a2b/round2": 0.517664, "cos_c2b/round2": 0.514017, "cos_b2a/round2": 0.090316, "cos_b2c/round2": 0.175822, "rms_a/round2": 38.886147, "rms_b/round2": 77.206879, "rms_c/round2": 20.031158}, "alpha_mean": 0.41748553184424253, "alpha_hist": [0.02893, 0.1498, 0.19437, 0.15276, 0.12114, 0.10973, 0.10538, 0.09074, 0.0451, 0.00206], "alpha_by_type": {"special": {"share": 0.00113, "mean": 0.61147, "std": 0.13494}, "whitespace": {"share": 0.04047, "mean": 0.61482, "std": 0.15407}, "digit": {"share": 0.01186, "mean": 0.38242, "std": 0.15916}, "word_start": {"share": 0.56325, "mean": 0.41845, "std": 0.20672}, "word_cont": {"share": 0.25415, "mean": 0.27703, "std": 0.13841}, "punct": {"share": 0.12915, "mean": 0.62937, "std": 0.17796}}, "alpha_p05": 0.13274, "alpha_p50": 0.37616, "alpha_p95": 0.801}
{"step": 55010, "total_steps": 72479, "tokens": 28841082880, "loss": 2.92521932721138, "l_mix": 1.8124568611383438, "l_a": 1.8686514496803284, "l_c": 1.840556561946869, "z_loss": 48.17343235015869, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.3032888174057007, "tokens_per_second": 1359182.6593317317, "eta_seconds": 6738.451972675323, "data_wait_seconds": 1.817941665649414e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 55020, "total_steps": 72479, "tokens": 28846325760, "loss": 4.353239417076111, "l_mix": 2.6984751522541046, "l_a": 2.7807253003120422, "l_c": 2.7351555228233337, "z_loss": 40.47450542449951, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16123725473880768, "tokens_per_second": 1465053.64714686, "eta_seconds": 6247.924238014221, "data_wait_seconds": 1.6391277313232422e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 55030, "total_steps": 72479, "tokens": 28851568640, "loss": 4.436312198638916, "l_mix": 2.749020278453827, "l_a": 2.837729126214981, "l_c": 2.786577194929123, "z_loss": 40.03691864013672, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1944132298231125, "tokens_per_second": 1359248.7776328987, "eta_seconds": 6730.409813523292, "data_wait_seconds": 2.014636993408203e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 55040, "total_steps": 72479, "tokens": 28856811520, "loss": 4.2856733202934265, "l_mix": 2.6567104160785675, "l_a": 2.7399792969226837, "l_c": 2.689896821975708, "z_loss": 40.197365283966064, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1665632426738739, "tokens_per_second": 1413409.9858911166, "eta_seconds": 6468.79427998066, "data_wait_seconds": 1.9252300262451172e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 55050, "total_steps": 72479, "tokens": 28862054400, "loss": 4.398499608039856, "l_mix": 2.726043164730072, "l_a": 2.8116763532161713, "l_c": 2.7631782591342926, "z_loss": 41.145158767700195, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13277283310890198, "tokens_per_second": 1301159.0817618063, "eta_seconds": 7022.827323794366, "data_wait_seconds": 2.2113323211669922e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 55060, "total_steps": 72479, "tokens": 28867297280, "loss": 4.3589067459106445, "l_mix": 2.701533555984497, "l_a": 2.7852659225463867, "l_c": 2.739311158657074, "z_loss": 40.80089473724365, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1383877396583557, "tokens_per_second": 1446279.762894181, "eta_seconds": 6314.527041244507, "data_wait_seconds": 1.952052116394043e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 55070, "total_steps": 72479, "tokens": 28872540160, "loss": 3.1207345724105835, "l_mix": 1.922206699848175, "l_a": 2.0301826894283295, "l_c": 1.9649102240800858, "z_loss": 42.618950843811035, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.24941371381282806, "tokens_per_second": 1393423.967381424, "eta_seconds": 6550.289076161385, "data_wait_seconds": 2.753734588623047e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 55080, "total_steps": 72479, "tokens": 28877783040, "loss": 3.11920428276062, "l_mix": 1.9199943840503693, "l_a": 2.0323646515607834, "l_c": 1.9650014787912369, "z_loss": 43.0152792930603, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20597213506698608, "tokens_per_second": 1430057.37828448, "eta_seconds": 6378.825808334351, "data_wait_seconds": 2.4497509002685547e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 55090, "total_steps": 72479, "tokens": 28883025920, "loss": 3.119045376777649, "l_mix": 1.9194632768630981, "l_a": 2.032745197415352, "l_c": 1.9658617079257965, "z_loss": 42.946468353271484, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16695795953273773, "tokens_per_second": 1372718.764393887, "eta_seconds": 6641.45072426796, "data_wait_seconds": 3.0547380447387695e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 55100, "total_steps": 72479, "tokens": 28888268800, "loss": 3.105322688817978, "l_mix": 1.9110171645879745, "l_a": 2.0242941975593567, "l_c": 1.9567240178585052, "z_loss": 44.755637645721436, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19146829843521118, "tokens_per_second": 1422272.9308052456, "eta_seconds": 6406.366144394875, "data_wait_seconds": 1.7017126083374023e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16881433, "gate_a2b/L1_absmax": 0.69891137, "gate_a2b/L1_rms": 0.17955779, "gate_a2b/L3_absmean": 0.16196282, "gate_a2b/L3_absmax": 0.5242762, "gate_a2b/L3_rms": 0.16790166, "gate_a2b/L5_absmean": 0.24280372, "gate_a2b/L5_absmax": 0.73426634, "gate_a2b/L5_rms": 0.24890511, "gate_c2b/L1_absmean": 0.02817395, "gate_c2b/L1_absmax": 0.63260883, "gate_c2b/L1_rms": 0.06729101, "gate_c2b/L3_absmean": 0.17576408, "gate_c2b/L3_absmax": 0.39798906, "gate_c2b/L3_rms": 0.1795335, "gate_c2b/L5_absmean": 0.49963856, "gate_c2b/L5_absmax": 0.70570505, "gate_c2b/L5_rms": 0.5097056, "gate_b2a/L7_absmean": 0.0510162, "gate_b2a/L7_absmax": 0.33575901, "gate_b2a/L7_rms": 0.05682682, "gate_b2a/L11_absmean": 0.08195361, "gate_b2a/L11_absmax": 0.61394852, "gate_b2a/L11_rms": 0.09131695, "gate_b2a/L14_absmean": 0.2126395, "gate_b2a/L14_absmax": 0.81472665, "gate_b2a/L14_rms": 0.22184409, "gate_b2c/L3_absmean": 0.03495824, "gate_b2c/L3_absmax": 0.5493108, "gate_b2c/L3_rms": 0.04556194, "gate_b2c/L5_absmean": 0.03721529, "gate_b2c/L5_absmax": 0.54876179, "gate_b2c/L5_rms": 0.0469353, "gate_b2c/L6_absmean": 0.0540549, "gate_b2c/L6_absmax": 0.35441449, "gate_b2c/L6_rms": 0.06040889}, "streams": {"cos_a2b/round0": 0.354715, "cos_c2b/round0": 0.236183, "cos_b2a/round0": 0.14384, "cos_b2c/round0": 0.119913, "rms_a/round0": 7.298846, "rms_b/round0": 12.50406, "rms_c/round0": 4.739925, "cos_a2b/round1": 0.409255, "cos_c2b/round1": 0.304747, "cos_b2a/round1": 0.241949, "cos_b2c/round1": 0.1204, "rms_a/round1": 14.496676, "rms_b/round1": 29.805046, "rms_c/round1": 10.415113, "cos_a2b/round2": 0.518276, "cos_c2b/round2": 0.509653, "cos_b2a/round2": 0.09151, "cos_b2c/round2": 0.173505, "rms_a/round2": 39.26981, "rms_b/round2": 77.649963, "rms_c/round2": 20.151827}, "alpha_mean": 0.41327168350614585, "alpha_hist": [0.03645, 0.15616, 0.19156, 0.14793, 0.11963, 0.1088, 0.10017, 0.08911, 0.04839, 0.0018], "alpha_by_type": {"special": {"share": 0.00114, "mean": 0.64258, "std": 0.09884}, "whitespace": {"share": 0.03397, "mean": 0.61401, "std": 0.17346}, "digit": {"share": 0.01082, "mean": 0.38988, "std": 0.16639}, "word_start": {"share": 0.56903, "mean": 0.42261, "std": 0.20996}, "word_cont": {"share": 0.26587, "mean": 0.27315, "std": 0.14297}, "punct": {"share": 0.11917, "mean": 0.624, "std": 0.18896}}, "alpha_p05": 0.11757, "alpha_p50": 0.38353, "alpha_p95": 0.795}
{"step": 55110, "total_steps": 72479, "tokens": 28893511680, "loss": 3.1206043362617493, "l_mix": 1.919108510017395, "l_a": 2.0371863543987274, "l_c": 1.9677994549274445, "z_loss": 44.02778100967407, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14824871718883514, "tokens_per_second": 1446269.775295198, "eta_seconds": 6296.4451221704485, "data_wait_seconds": 2.294778823852539e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 55120, "total_steps": 72479, "tokens": 28898754560, "loss": 2.97044837474823, "l_mix": 1.8397517651319504, "l_a": 1.903343915939331, "l_c": 1.865644559264183, "z_loss": 48.99169588088989, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.22009402513504028, "tokens_per_second": 1322782.2596214549, "eta_seconds": 6880.282318425178, "data_wait_seconds": 3.1501054763793945e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 55130, "total_steps": 72479, "tokens": 28903997440, "loss": 4.365934193134308, "l_mix": 2.706262469291687, "l_a": 2.79137259721756, "l_c": 2.740866035223007, "z_loss": 41.37026023864746, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1509556919336319, "tokens_per_second": 1378052.3979938678, "eta_seconds": 6600.527327728271, "data_wait_seconds": 2.2917985916137695e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 55140, "total_steps": 72479, "tokens": 28909240320, "loss": 4.4047388434410095, "l_mix": 2.7300983369350433, "l_a": 2.8156591951847076, "l_c": 2.7664755284786224, "z_loss": 41.1343560218811, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16077016294002533, "tokens_per_second": 1385596.7609211153, "eta_seconds": 6560.804621076584, "data_wait_seconds": 2.3156404495239258e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 55150, "total_steps": 72479, "tokens": 28914483200, "loss": 4.328157186508179, "l_mix": 2.682057350873947, "l_a": 2.768761456012726, "l_c": 2.718237966299057, "z_loss": 40.43020677566528, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1508343517780304, "tokens_per_second": 1444514.673198598, "eta_seconds": 6289.57733733654, "data_wait_seconds": 1.6629695892333984e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 55160, "total_steps": 72479, "tokens": 28919726080, "loss": 4.385836601257324, "l_mix": 2.7188809514045715, "l_a": 2.80354106426239, "l_c": 2.7529775500297546, "z_loss": 41.22292995452881, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2529294192790985, "tokens_per_second": 1436899.1375115574, "eta_seconds": 6319.263221025467, "data_wait_seconds": 1.6987323760986328e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 55170, "total_steps": 72479, "tokens": 28924968960, "loss": 4.367822051048279, "l_mix": 2.7063629031181335, "l_a": 2.7932377755641937, "l_c": 2.7449593245983124, "z_loss": 41.326212882995605, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17979690432548523, "tokens_per_second": 1452752.7428111136, "eta_seconds": 6246.6934149026865, "data_wait_seconds": 1.6361474990844727e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 55180, "total_steps": 72479, "tokens": 28930211840, "loss": 3.082792967557907, "l_mix": 1.8970799446105957, "l_a": 2.0088274031877518, "l_c": 1.94354909658432, "z_loss": 44.28355932235718, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19065454602241516, "tokens_per_second": 1380248.8636774186, "eta_seconds": 6571.031029748917, "data_wait_seconds": 4.836916923522949e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 55190, "total_steps": 72479, "tokens": 28935454720, "loss": 3.1150715053081512, "l_mix": 1.9176075458526611, "l_a": 2.029692158102989, "l_c": 1.9618540853261948, "z_loss": 44.74238729476929, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.23651999235153198, "tokens_per_second": 1374874.7795875208, "eta_seconds": 6592.902398514747, "data_wait_seconds": 1.749396324157715e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 55200, "total_steps": 72479, "tokens": 28940697600, "loss": 3.1760711669921875, "l_mix": 1.9550132304430008, "l_a": 2.0644364207983017, "l_c": 2.005756437778473, "z_loss": 42.77748727798462, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16699184477329254, "tokens_per_second": 1331080.0177959702, "eta_seconds": 6805.8811122417455, "data_wait_seconds": 4.988908767700195e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16896412, "gate_a2b/L1_absmax": 0.69825059, "gate_a2b/L1_rms": 0.1797573, "gate_a2b/L3_absmean": 0.1620995, "gate_a2b/L3_absmax": 0.52430671, "gate_a2b/L3_rms": 0.16802932, "gate_a2b/L5_absmean": 0.24229994, "gate_a2b/L5_absmax": 0.7356208, "gate_a2b/L5_rms": 0.24835734, "gate_c2b/L1_absmean": 0.02822358, "gate_c2b/L1_absmax": 0.63114011, "gate_c2b/L1_rms": 0.06745254, "gate_c2b/L3_absmean": 0.17597951, "gate_c2b/L3_absmax": 0.39993641, "gate_c2b/L3_rms": 0.17970212, "gate_c2b/L5_absmean": 0.49934348, "gate_c2b/L5_absmax": 0.7048161, "gate_c2b/L5_rms": 0.50942552, "gate_b2a/L7_absmean": 0.05098677, "gate_b2a/L7_absmax": 0.33483806, "gate_b2a/L7_rms": 0.0568535, "gate_b2a/L11_absmean": 0.08216225, "gate_b2a/L11_absmax": 0.61105901, "gate_b2a/L11_rms": 0.09140769, "gate_b2a/L14_absmean": 0.21253642, "gate_b2a/L14_absmax": 0.81393629, "gate_b2a/L14_rms": 0.22174014, "gate_b2c/L3_absmean": 0.0350467, "gate_b2c/L3_absmax": 0.54935592, "gate_b2c/L3_rms": 0.04569615, "gate_b2c/L5_absmean": 0.03731516, "gate_b2c/L5_absmax": 0.5477668, "gate_b2c/L5_rms": 0.04698149, "gate_b2c/L6_absmean": 0.05399203, "gate_b2c/L6_absmax": 0.35486639, "gate_b2c/L6_rms": 0.06033999}, "streams": {"cos_a2b/round0": 0.353784, "cos_c2b/round0": 0.237184, "cos_b2a/round0": 0.144509, "cos_b2c/round0": 0.118845, "rms_a/round0": 7.240913, "rms_b/round0": 12.489121, "rms_c/round0": 4.751535, "cos_a2b/round1": 0.407121, "cos_c2b/round1": 0.305433, "cos_b2a/round1": 0.23872, "cos_b2c/round1": 0.119237, "rms_a/round1": 14.481478, "rms_b/round1": 29.838644, "rms_c/round1": 10.44599, "cos_a2b/round2": 0.51917, "cos_c2b/round2": 0.511058, "cos_b2a/round2": 0.086653, "cos_b2c/round2": 0.171193, "rms_a/round2": 39.324028, "rms_b/round2": 77.417183, "rms_c/round2": 20.210154}, "alpha_mean": 0.4132062704986126, "alpha_hist": [0.03468, 0.15671, 0.18755, 0.14944, 0.12318, 0.11154, 0.10522, 0.08569, 0.04323, 0.00275], "alpha_by_type": {"special": {"share": 0.00114, "mean": 0.62088, "std": 0.10858}, "whitespace": {"share": 0.0352, "mean": 0.61182, "std": 0.17794}, "digit": {"share": 0.01155, "mean": 0.38381, "std": 0.17775}, "word_start": {"share": 0.57617, "mean": 0.41701, "std": 0.2078}, "word_cont": {"share": 0.2552, "mean": 0.27988, "std": 0.14464}, "punct": {"share": 0.12073, "mean": 0.61984, "std": 0.1866}}, "alpha_p05": 0.13073, "alpha_p50": 0.37296, "alpha_p95": 0.80958}
{"step": 55210, "total_steps": 72479, "tokens": 28945940480, "loss": 3.109123855829239, "l_mix": 1.9135068207979202, "l_a": 2.0271931290626526, "l_c": 1.9581970423460007, "z_loss": 43.62001609802246, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1493775099515915, "tokens_per_second": 1353916.2061883598, "eta_seconds": 6687.215523838997, "data_wait_seconds": 2.6017427444458008e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 55220, "total_steps": 72479, "tokens": 28951183360, "loss": 3.1389079988002777, "l_mix": 1.9312137067317963, "l_a": 2.0474389791488647, "l_c": 1.97820845246315, "z_loss": 44.48948001861572, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17659519612789154, "tokens_per_second": 1399076.6799491656, "eta_seconds": 6467.613049149513, "data_wait_seconds": 2.9385089874267578e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 55230, "total_steps": 72479, "tokens": 28956426240, "loss": 3.142561048269272, "l_mix": 1.9345649182796478, "l_a": 2.044078692793846, "l_c": 1.98257477581501, "z_loss": 42.66093301773071, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2196897715330124, "tokens_per_second": 1404389.1297931138, "eta_seconds": 6439.414490008354, "data_wait_seconds": 1.8537044525146484e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 55240, "total_steps": 72479, "tokens": 28961669120, "loss": 4.298311710357666, "l_mix": 2.6630097031593323, "l_a": 2.7510543763637543, "l_c": 2.6999522149562836, "z_loss": 41.2270712852478, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12330213934183121, "tokens_per_second": 1398974.3225412841, "eta_seconds": 6460.590938925743, "data_wait_seconds": 2.7418136596679688e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 55250, "total_steps": 72479, "tokens": 28966912000, "loss": 4.325123727321625, "l_mix": 2.6801384687423706, "l_a": 2.7676315903663635, "l_c": 2.7156524062156677, "z_loss": 40.57793140411377, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1360502541065216, "tokens_per_second": 1457830.852540278, "eta_seconds": 6196.16324915886, "data_wait_seconds": 1.7374753952026367e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 55260, "total_steps": 72479, "tokens": 28972154880, "loss": 4.332006990909576, "l_mix": 2.684447854757309, "l_a": 2.7721145153045654, "l_c": 2.7197490334510803, "z_loss": 41.15612745285034, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16766691207885742, "tokens_per_second": 1432618.0257611836, "eta_seconds": 6301.55066435337, "data_wait_seconds": 1.6927719116210938e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 55270, "total_steps": 72479, "tokens": 28977397760, "loss": 4.4095219373703, "l_mix": 2.733038067817688, "l_a": 2.81937575340271, "l_c": 2.7689034938812256, "z_loss": 41.1820650100708, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2054043710231781, "tokens_per_second": 1433324.524608957, "eta_seconds": 6294.786726307869, "data_wait_seconds": 1.6629695892333984e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 55280, "total_steps": 72479, "tokens": 28982640640, "loss": 4.350915968418121, "l_mix": 2.6967667639255524, "l_a": 2.7806695997714996, "l_c": 2.733160674571991, "z_loss": 39.5839147567749, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15873362123966217, "tokens_per_second": 1347286.7564778903, "eta_seconds": 6692.880538344383, "data_wait_seconds": 1.576542854309082e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 55290, "total_steps": 72479, "tokens": 28987883520, "loss": 3.1438028812408447, "l_mix": 1.9353116601705551, "l_a": 2.043748438358307, "l_c": 1.9845552444458008, "z_loss": 42.4314751625061, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20118990540504456, "tokens_per_second": 1418835.7163383174, "eta_seconds": 6351.67717320919, "data_wait_seconds": 1.96993350982666e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 55300, "total_steps": 72479, "tokens": 28993126400, "loss": 4.6919214725494385, "l_mix": 2.9111012518405914, "l_a": 2.990215480327606, "l_c": 2.9458517134189606, "z_loss": 39.589173316955566, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1515430063009262, "tokens_per_second": 1116812.1650243928, "eta_seconds": 8064.689689159394, "data_wait_seconds": 3.921985626220703e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.16895692, "gate_a2b/L1_absmax": 0.69877517, "gate_a2b/L1_rms": 0.17972179, "gate_a2b/L3_absmean": 0.16223584, "gate_a2b/L3_absmax": 0.52626455, "gate_a2b/L3_rms": 0.16812944, "gate_a2b/L5_absmean": 0.24190859, "gate_a2b/L5_absmax": 0.73474193, "gate_a2b/L5_rms": 0.24799626, "gate_c2b/L1_absmean": 0.0279363, "gate_c2b/L1_absmax": 0.63098282, "gate_c2b/L1_rms": 0.06727549, "gate_c2b/L3_absmean": 0.17569743, "gate_c2b/L3_absmax": 0.39773491, "gate_c2b/L3_rms": 0.17937057, "gate_c2b/L5_absmean": 0.49923539, "gate_c2b/L5_absmax": 0.70270127, "gate_c2b/L5_rms": 0.50937849, "gate_b2a/L7_absmean": 0.05094613, "gate_b2a/L7_absmax": 0.33344734, "gate_b2a/L7_rms": 0.05674754, "gate_b2a/L11_absmean": 0.0820969, "gate_b2a/L11_absmax": 0.61186677, "gate_b2a/L11_rms": 0.09141124, "gate_b2a/L14_absmean": 0.21262707, "gate_b2a/L14_absmax": 0.8131569, "gate_b2a/L14_rms": 0.2218131, "gate_b2c/L3_absmean": 0.03491968, "gate_b2c/L3_absmax": 0.55108565, "gate_b2c/L3_rms": 0.04559204, "gate_b2c/L5_absmean": 0.03753275, "gate_b2c/L5_absmax": 0.55083871, "gate_b2c/L5_rms": 0.04718977, "gate_b2c/L6_absmean": 0.0539683, "gate_b2c/L6_absmax": 0.35375106, "gate_b2c/L6_rms": 0.06027284}, "streams": {"cos_a2b/round0": 0.358643, "cos_c2b/round0": 0.279507, "cos_b2a/round0": 0.164067, "cos_b2c/round0": 0.146101, "rms_a/round0": 6.918146, "rms_b/round0": 12.179474, "rms_c/round0": 4.634257, "cos_a2b/round1": 0.419817, "cos_c2b/round1": 0.309074, "cos_b2a/round1": 0.255532, "cos_b2c/round1": 0.119475, "rms_a/round1": 13.438051, "rms_b/round1": 27.853069, "rms_c/round1": 9.688162, "cos_a2b/round2": 0.552228, "cos_c2b/round2": 0.523078, "cos_b2a/round2": 0.104446, "cos_b2c/round2": 0.188721, "rms_a/round2": 37.940964, "rms_b/round2": 74.947609, "rms_c/round2": 18.719517}, "alpha_mean": 0.4489790090664485, "alpha_hist": [0.00417, 0.06508, 0.17397, 0.18306, 0.17722, 0.1675, 0.13416, 0.08269, 0.01212, 6e-05], "alpha_by_type": {"special": {"share": 0.00021, "mean": 0.60791, "std": 0.09726}, "whitespace": {"share": 0.02954, "mean": 0.5536, "std": 0.11585}, "digit": {"share": 0.02591, "mean": 0.4761, "std": 0.09379}, "word_start": {"share": 0.65382, "mean": 0.46126, "std": 0.17074}, "word_cont": {"share": 0.20996, "mean": 0.3276, "std": 0.12667}, "punct": {"share": 0.08055, "mean": 0.61818, "std": 0.15829}}, "alpha_p05": 0.19163, "alpha_p50": 0.44806, "alpha_p95": 0.7364}
{"step": 55310, "total_steps": 72479, "tokens": 28998369280, "loss": 4.826362669467926, "l_mix": 2.995112270116806, "l_a": 3.066576510667801, "l_c": 3.0375912189483643, "z_loss": 39.13366508483887, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18861770629882812, "tokens_per_second": 1401105.1956062978, "eta_seconds": 6424.571616911888, "data_wait_seconds": 1.722574234008789e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 55320, "total_steps": 72479, "tokens": 29003612160, "loss": 4.773607492446899, "l_mix": 2.962075501680374, "l_a": 3.0406825840473175, "l_c": 2.997757226228714, "z_loss": 39.385647773742676, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15299880504608154, "tokens_per_second": 1373273.4925185489, "eta_seconds": 6550.958597111702, "data_wait_seconds": 1.710653305053711e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 55330, "total_steps": 72479, "tokens": 29008855040, "loss": 3.1365368962287903, "l_mix": 1.9292883276939392, "l_a": 2.044372543692589, "l_c": 1.9797891825437546, "z_loss": 44.446428298950195, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19705328345298767, "tokens_per_second": 1448031.0545393582, "eta_seconds": 6209.131277823448, "data_wait_seconds": 2.1278858184814453e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 55340, "total_steps": 72479, "tokens": 29014097920, "loss": 3.124063491821289, "l_mix": 1.9228862076997757, "l_a": 2.0286001712083817, "l_c": 1.9753240942955017, "z_loss": 43.0893874168396, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.24336127936840057, "tokens_per_second": 1397000.6893779265, "eta_seconds": 6432.188688468933, "data_wait_seconds": 4.029273986816406e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 55350, "total_steps": 72479, "tokens": 29019340800, "loss": 4.348028481006622, "l_mix": 2.69426429271698, "l_a": 2.783037930727005, "l_c": 2.729509115219116, "z_loss": 41.088003158569336, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1609668731689453, "tokens_per_second": 1377189.9644383125, "eta_seconds": 6520.908069252968, "data_wait_seconds": 2.244114875793457e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 55360, "total_steps": 72479, "tokens": 29024583680, "loss": 4.362087309360504, "l_mix": 2.703667640686035, "l_a": 2.787912607192993, "l_c": 2.7401526272296906, "z_loss": 40.11026477813721, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15773214399814606, "tokens_per_second": 1413858.544756958, "eta_seconds": 6348.079378437997, "data_wait_seconds": 2.339482307434082e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 55370, "total_steps": 72479, "tokens": 29029826560, "loss": 4.317888051271439, "l_mix": 2.6763605177402496, "l_a": 2.761142075061798, "l_c": 2.7106162011623383, "z_loss": 39.94901371002197, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20846857130527496, "tokens_per_second": 1462815.5459340843, "eta_seconds": 6132.039966988564, "data_wait_seconds": 1.8924474716186523e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 55380, "total_steps": 72479, "tokens": 29035069440, "loss": 4.33462929725647, "l_mix": 2.6863194406032562, "l_a": 2.7725414633750916, "l_c": 2.721824526786804, "z_loss": 41.137330532073975, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19178429245948792, "tokens_per_second": 1446357.0049847816, "eta_seconds": 6198.193448162078, "data_wait_seconds": 1.9073486328125e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 55390, "total_steps": 72479, "tokens": 29040312320, "loss": 4.212320446968079, "l_mix": 2.609430819749832, "l_a": 2.696723908185959, "l_c": 2.6462414860725403, "z_loss": 41.765183448791504, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14593163132667542, "tokens_per_second": 1308420.2141657155, "eta_seconds": 6847.6148067712775, "data_wait_seconds": 0.04457497596740723, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 55400, "total_steps": 72479, "tokens": 29045555200, "loss": 3.1910945177078247, "l_mix": 1.963095098733902, "l_a": 2.0783961713314056, "l_c": 2.0149350613355637, "z_loss": 44.16615915298462, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19723443686962128, "tokens_per_second": 1445774.6584825772, "eta_seconds": 6193.437337875366, "data_wait_seconds": 1.7434358596801758e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.16876061, "gate_a2b/L1_absmax": 0.70176321, "gate_a2b/L1_rms": 0.17951824, "gate_a2b/L3_absmean": 0.16178849, "gate_a2b/L3_absmax": 0.52724749, "gate_a2b/L3_rms": 0.16770555, "gate_a2b/L5_absmean": 0.24177344, "gate_a2b/L5_absmax": 0.73589694, "gate_a2b/L5_rms": 0.2478731, "gate_c2b/L1_absmean": 0.0287809, "gate_c2b/L1_absmax": 0.63328344, "gate_c2b/L1_rms": 0.06763251, "gate_c2b/L3_absmean": 0.1755269, "gate_c2b/L3_absmax": 0.39625323, "gate_c2b/L3_rms": 0.17925005, "gate_c2b/L5_absmean": 0.49773598, "gate_c2b/L5_absmax": 0.70363277, "gate_c2b/L5_rms": 0.50775236, "gate_b2a/L7_absmean": 0.05067024, "gate_b2a/L7_absmax": 0.33407757, "gate_b2a/L7_rms": 0.05651959, "gate_b2a/L11_absmean": 0.08205133, "gate_b2a/L11_absmax": 0.61096734, "gate_b2a/L11_rms": 0.09137493, "gate_b2a/L14_absmean": 0.21242711, "gate_b2a/L14_absmax": 0.81412816, "gate_b2a/L14_rms": 0.22166586, "gate_b2c/L3_absmean": 0.03499155, "gate_b2c/L3_absmax": 0.55113542, "gate_b2c/L3_rms": 0.04568771, "gate_b2c/L5_absmean": 0.03714444, "gate_b2c/L5_absmax": 0.55168432, "gate_b2c/L5_rms": 0.04695319, "gate_b2c/L6_absmean": 0.05381415, "gate_b2c/L6_absmax": 0.35346437, "gate_b2c/L6_rms": 0.06013018}, "streams": {"cos_a2b/round0": 0.357998, "cos_c2b/round0": 0.233009, "cos_b2a/round0": 0.140994, "cos_b2c/round0": 0.117863, "rms_a/round0": 7.331568, "rms_b/round0": 12.530912, "rms_c/round0": 4.767869, "cos_a2b/round1": 0.410781, "cos_c2b/round1": 0.303037, "cos_b2a/round1": 0.238988, "cos_b2c/round1": 0.117459, "rms_a/round1": 14.56881, "rms_b/round1": 29.895241, "rms_c/round1": 10.410354, "cos_a2b/round2": 0.513968, "cos_c2b/round2": 0.509191, "cos_b2a/round2": 0.086681, "cos_b2c/round2": 0.169147, "rms_a/round2": 39.090919, "rms_b/round2": 77.513397, "rms_c/round2": 20.096415}, "alpha_mean": 0.40370738106736326, "alpha_hist": [0.04445, 0.18578, 0.18974, 0.1394, 0.10544, 0.09392, 0.09332, 0.0881, 0.05458, 0.00526], "alpha_by_type": {"special": {"share": 0.00107, "mean": 0.69383, "std": 0.12243}, "whitespace": {"share": 0.03122, "mean": 0.6643, "std": 0.17995}, "digit": {"share": 0.00964, "mean": 0.3923, "std": 0.18635}, "word_start": {"share": 0.56828, "mean": 0.40588, "std": 0.22045}, "word_cont": {"share": 0.27144, "mean": 0.26991, "std": 0.14346}, "punct": {"share": 0.11835, "mean": 0.62971, "std": 0.20005}}, "alpha_p05": 0.09637, "alpha_p50": 0.37114, "alpha_p95": 0.80958}
{"step": 55410, "total_steps": 72479, "tokens": 29050798080, "loss": 3.0275925993919373, "l_mix": 1.8750390708446503, "l_a": 1.938913956284523, "l_c": 1.902930960059166, "z_loss": 49.15001392364502, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20167826116085052, "tokens_per_second": 1302000.4267437414, "eta_seconds": 6873.324837827683, "data_wait_seconds": 1.7523765563964844e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 55420, "total_steps": 72479, "tokens": 29056040960, "loss": 4.681169629096985, "l_mix": 2.9041962325572968, "l_a": 2.9817921221256256, "l_c": 2.941452234983444, "z_loss": 39.324296951293945, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12921373546123505, "tokens_per_second": 1219868.556005437, "eta_seconds": 7331.797305512429, "data_wait_seconds": 1.7523765563964844e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 55430, "total_steps": 72479, "tokens": 29061283840, "loss": 4.73167222738266, "l_mix": 2.9367635250091553, "l_a": 3.011947363615036, "l_c": 2.9710811972618103, "z_loss": 38.46495485305786, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12796810269355774, "tokens_per_second": 1338016.8837520834, "eta_seconds": 6680.473333740234, "data_wait_seconds": 2.3543834686279297e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 55440, "total_steps": 72479, "tokens": 29066526720, "loss": 4.732954621315002, "l_mix": 2.9361617863178253, "l_a": 3.0141758620738983, "l_c": 2.975133389234543, "z_loss": 39.36962890625, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15399041771888733, "tokens_per_second": 1298350.3482258073, "eta_seconds": 6880.533628082276, "data_wait_seconds": 1.7970800399780273e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 55450, "total_steps": 72479, "tokens": 29071769600, "loss": 4.733279883861542, "l_mix": 2.9375662207603455, "l_a": 3.0123057067394257, "l_c": 2.973406106233597, "z_loss": 39.773221015930176, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1263904720544815, "tokens_per_second": 1234559.0551742692, "eta_seconds": 7231.813103294372, "data_wait_seconds": 2.6911497116088867e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 55460, "total_steps": 72479, "tokens": 29077012480, "loss": 4.674395680427551, "l_mix": 2.900480628013611, "l_a": 2.973263293504715, "l_c": 2.939786344766617, "z_loss": 39.393436908721924, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19270282983779907, "tokens_per_second": 1376349.6286348796, "eta_seconds": 6482.987524652482, "data_wait_seconds": 3.1054019927978516e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 55470, "total_steps": 72479, "tokens": 29082255360, "loss": 4.2943960428237915, "l_mix": 2.6615751683712006, "l_a": 2.749886631965637, "l_c": 2.692849338054657, "z_loss": 40.97316026687622, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.3385144770145416, "tokens_per_second": 1478350.962690756, "eta_seconds": 6032.136358046531, "data_wait_seconds": 1.963973045349121e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 55480, "total_steps": 72479, "tokens": 29087498240, "loss": 4.158753156661987, "l_mix": 2.5747646391391754, "l_a": 2.666777402162552, "l_c": 2.6131843626499176, "z_loss": 42.44157838821411, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1586993932723999, "tokens_per_second": 1433233.722069917, "eta_seconds": 6218.365905547142, "data_wait_seconds": 3.361701965332031e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 55490, "total_steps": 72479, "tokens": 29092741120, "loss": 4.0981784760952, "l_mix": 2.53822261095047, "l_a": 2.6265089213848114, "l_c": 2.5733437836170197, "z_loss": 40.79884433746338, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15880635380744934, "tokens_per_second": 1226800.1137815784, "eta_seconds": 7260.456476926804, "data_wait_seconds": 2.9385089874267578e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 55500, "total_steps": 72479, "tokens": 29097984000, "loss": 4.356467843055725, "l_mix": 2.700103133916855, "l_a": 2.7848485708236694, "l_c": 2.7363670766353607, "z_loss": 40.25684118270874, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14124703407287598, "tokens_per_second": 1279044.3711748675, "eta_seconds": 6959.794478297234, "data_wait_seconds": 2.6494264602661133e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16869475, "gate_a2b/L1_absmax": 0.70449871, "gate_a2b/L1_rms": 0.17954999, "gate_a2b/L3_absmean": 0.16203529, "gate_a2b/L3_absmax": 0.52998602, "gate_a2b/L3_rms": 0.16802561, "gate_a2b/L5_absmean": 0.24136584, "gate_a2b/L5_absmax": 0.73842567, "gate_a2b/L5_rms": 0.24746744, "gate_c2b/L1_absmean": 0.02900097, "gate_c2b/L1_absmax": 0.63519078, "gate_c2b/L1_rms": 0.06785549, "gate_c2b/L3_absmean": 0.17558718, "gate_c2b/L3_absmax": 0.40885398, "gate_c2b/L3_rms": 0.17934513, "gate_c2b/L5_absmean": 0.49820289, "gate_c2b/L5_absmax": 0.70649529, "gate_c2b/L5_rms": 0.50822878, "gate_b2a/L7_absmean": 0.05050586, "gate_b2a/L7_absmax": 0.33358324, "gate_b2a/L7_rms": 0.05650514, "gate_b2a/L11_absmean": 0.08250132, "gate_b2a/L11_absmax": 0.60932487, "gate_b2a/L11_rms": 0.09170406, "gate_b2a/L14_absmean": 0.21201648, "gate_b2a/L14_absmax": 0.81767225, "gate_b2a/L14_rms": 0.22140165, "gate_b2c/L3_absmean": 0.03515533, "gate_b2c/L3_absmax": 0.54940224, "gate_b2c/L3_rms": 0.04573849, "gate_b2c/L5_absmean": 0.03715643, "gate_b2c/L5_absmax": 0.55577606, "gate_b2c/L5_rms": 0.04710093, "gate_b2c/L6_absmean": 0.05383887, "gate_b2c/L6_absmax": 0.35405618, "gate_b2c/L6_rms": 0.06027726}, "streams": {"cos_a2b/round0": 0.356617, "cos_c2b/round0": 0.268218, "cos_b2a/round0": 0.154945, "cos_b2c/round0": 0.141781, "rms_a/round0": 7.112855, "rms_b/round0": 12.399461, "rms_c/round0": 4.66771, "cos_a2b/round1": 0.414013, "cos_c2b/round1": 0.304096, "cos_b2a/round1": 0.247202, "cos_b2c/round1": 0.108927, "rms_a/round1": 13.883121, "rms_b/round1": 28.543806, "rms_c/round1": 9.867403, "cos_a2b/round2": 0.540227, "cos_c2b/round2": 0.519029, "cos_b2a/round2": 0.099382, "cos_b2c/round2": 0.178846, "rms_a/round2": 38.791798, "rms_b/round2": 76.78521, "rms_c/round2": 19.035856}, "alpha_mean": 0.44948303909625054, "alpha_hist": [0.00768, 0.07011, 0.15544, 0.18996, 0.17636, 0.16873, 0.14143, 0.0764, 0.01366, 0.00023], "alpha_by_type": {"special": {"share": 0.00098, "mean": 0.56543, "std": 0.09085}, "whitespace": {"share": 0.01935, "mean": 0.53508, "std": 0.1311}, "digit": {"share": 0.02953, "mean": 0.4668, "std": 0.12063}, "word_start": {"share": 0.58165, "mean": 0.46426, "std": 0.17128}, "word_cont": {"share": 0.25793, "mean": 0.33963, "std": 0.13207}, "punct": {"share": 0.11057, "mean": 0.60739, "std": 0.15782}}, "alpha_p05": 0.17385, "alpha_p50": 0.4347, "alpha_p95": 0.72448}
{"step": 55510, "total_steps": 72479, "tokens": 29103226880, "loss": 3.114552766084671, "l_mix": 1.9170383512973785, "l_a": 2.028719499707222, "l_c": 1.9629949182271957, "z_loss": 43.380863189697266, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13937918841838837, "tokens_per_second": 1355264.1319933184, "eta_seconds": 6564.508616423607, "data_wait_seconds": 1.7642974853515625e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 55520, "total_steps": 72479, "tokens": 29108469760, "loss": 2.708443373441696, "l_mix": 1.6757450997829437, "l_a": 1.740241825580597, "l_c": 1.702085629105568, "z_loss": 49.558051109313965, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16822576522827148, "tokens_per_second": 1450682.37130954, "eta_seconds": 6129.115764999389, "data_wait_seconds": 3.921985626220703e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 55530, "total_steps": 72479, "tokens": 29113712640, "loss": 4.084487080574036, "l_mix": 2.5296326279640198, "l_a": 2.619070529937744, "l_c": 2.5637773275375366, "z_loss": 42.28041887283325, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12408307194709778, "tokens_per_second": 1373934.590628678, "eta_seconds": 6467.671294260025, "data_wait_seconds": 1.8864870071411133e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 55540, "total_steps": 72479, "tokens": 29118955520, "loss": 4.117765665054321, "l_mix": 2.5502572655677795, "l_a": 2.6385209560394287, "l_c": 2.5865068435668945, "z_loss": 40.91791868209839, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15405188500881195, "tokens_per_second": 1391145.2708817143, "eta_seconds": 6383.887159657478, "data_wait_seconds": 2.1189451217651367e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 55550, "total_steps": 72479, "tokens": 29124198400, "loss": 3.990372121334076, "l_mix": 2.469531863927841, "l_a": 2.5647043585777283, "l_c": 2.504762977361679, "z_loss": 43.63962984085083, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1984054148197174, "tokens_per_second": 1412597.6526022933, "eta_seconds": 6283.226887464524, "data_wait_seconds": 1.665949821472168e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 55560, "total_steps": 72479, "tokens": 29129441280, "loss": 4.065090924501419, "l_mix": 2.517222821712494, "l_a": 2.6057203710079193, "l_c": 2.5538394153118134, "z_loss": 41.09742736816406, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.21105912327766418, "tokens_per_second": 1299063.2698275927, "eta_seconds": 6828.326901412011, "data_wait_seconds": 1.9431114196777344e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 55570, "total_steps": 72479, "tokens": 29134684160, "loss": 4.142639189958572, "l_mix": 2.5655817091464996, "l_a": 2.6577618420124054, "l_c": 2.599096179008484, "z_loss": 41.054105281829834, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1984839290380478, "tokens_per_second": 1431993.999332655, "eta_seconds": 6190.798143100738, "data_wait_seconds": 1.704692840576172e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 55580, "total_steps": 72479, "tokens": 29139927040, "loss": 4.00142627954483, "l_mix": 2.477045089006424, "l_a": 2.568734258413315, "l_c": 2.5125360190868378, "z_loss": 41.18896245956421, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1825544536113739, "tokens_per_second": 1188992.7232425127, "eta_seconds": 7451.637624692917, "data_wait_seconds": 1.6868114471435547e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 55590, "total_steps": 72479, "tokens": 29145169920, "loss": 4.185745716094971, "l_mix": 2.592637926340103, "l_a": 2.6797133088111877, "l_c": 2.6306456327438354, "z_loss": 39.958927631378174, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.24151703715324402, "tokens_per_second": 1442874.8689366432, "eta_seconds": 6136.845420646668, "data_wait_seconds": 2.1785497665405273e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 55600, "total_steps": 72479, "tokens": 29150412800, "loss": 4.045995831489563, "l_mix": 2.50492662191391, "l_a": 2.595117002725601, "l_c": 2.541780263185501, "z_loss": 41.92948341369629, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12112566083669662, "tokens_per_second": 1412468.7203483463, "eta_seconds": 6265.241151547431, "data_wait_seconds": 1.8805265426635742e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16860265, "gate_a2b/L1_absmax": 0.70472455, "gate_a2b/L1_rms": 0.17937566, "gate_a2b/L3_absmean": 0.16220616, "gate_a2b/L3_absmax": 0.53174305, "gate_a2b/L3_rms": 0.16814306, "gate_a2b/L5_absmean": 0.24104224, "gate_a2b/L5_absmax": 0.74038774, "gate_a2b/L5_rms": 0.24714094, "gate_c2b/L1_absmean": 0.02852162, "gate_c2b/L1_absmax": 0.63685167, "gate_c2b/L1_rms": 0.06787879, "gate_c2b/L3_absmean": 0.17539315, "gate_c2b/L3_absmax": 0.39755622, "gate_c2b/L3_rms": 0.17916627, "gate_c2b/L5_absmean": 0.49833795, "gate_c2b/L5_absmax": 0.70247763, "gate_c2b/L5_rms": 0.5084157, "gate_b2a/L7_absmean": 0.05078232, "gate_b2a/L7_absmax": 0.33486864, "gate_b2a/L7_rms": 0.05663892, "gate_b2a/L11_absmean": 0.08218241, "gate_b2a/L11_absmax": 0.61088818, "gate_b2a/L11_rms": 0.09141847, "gate_b2a/L14_absmean": 0.21234086, "gate_b2a/L14_absmax": 0.82552528, "gate_b2a/L14_rms": 0.22168161, "gate_b2c/L3_absmean": 0.03476226, "gate_b2c/L3_absmax": 0.55098188, "gate_b2c/L3_rms": 0.04549463, "gate_b2c/L5_absmean": 0.03712057, "gate_b2c/L5_absmax": 0.55290115, "gate_b2c/L5_rms": 0.04695207, "gate_b2c/L6_absmean": 0.05377394, "gate_b2c/L6_absmax": 0.35759616, "gate_b2c/L6_rms": 0.0601493}, "streams": {"cos_a2b/round0": 0.361212, "cos_c2b/round0": 0.277803, "cos_b2a/round0": 0.15975, "cos_b2c/round0": 0.146513, "rms_a/round0": 7.029847, "rms_b/round0": 12.232936, "rms_c/round0": 4.636175, "cos_a2b/round1": 0.418277, "cos_c2b/round1": 0.308672, "cos_b2a/round1": 0.251564, "cos_b2c/round1": 0.111655, "rms_a/round1": 13.67154, "rms_b/round1": 28.338179, "rms_c/round1": 9.676754, "cos_a2b/round2": 0.538281, "cos_c2b/round2": 0.523055, "cos_b2a/round2": 0.096691, "cos_b2c/round2": 0.185134, "rms_a/round2": 38.49012, "rms_b/round2": 76.896255, "rms_c/round2": 18.638666}, "alpha_mean": 0.4338533141031178, "alpha_hist": [0.00786, 0.07338, 0.18555, 0.19078, 0.17215, 0.16911, 0.12985, 0.06221, 0.00902, 9e-05], "alpha_by_type": {"special": {"share": 0.00052, "mean": 0.57297, "std": 0.11425}, "whitespace": {"share": 0.02153, "mean": 0.54319, "std": 0.11482}, "digit": {"share": 0.03746, "mean": 0.44141, "std": 0.10586}, "word_start": {"share": 0.60507, "mean": 0.44771, "std": 0.1691}, "word_cont": {"share": 0.23009, "mean": 0.31621, "std": 0.1247}, "punct": {"share": 0.10533, "mean": 0.58552, "std": 0.15343}}, "alpha_p05": 0.19103, "alpha_p50": 0.42394, "alpha_p95": 0.7235}
{"step": 55610, "total_steps": 72479, "tokens": 29155655680, "loss": 3.981151133775711, "l_mix": 2.4651070535182953, "l_a": 2.5544586181640625, "l_c": 2.4990213811397552, "z_loss": 42.629258155822754, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1929730325937271, "tokens_per_second": 1416130.8774135632, "eta_seconds": 6245.336792707443, "data_wait_seconds": 2.1457672119140625e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 55620, "total_steps": 72479, "tokens": 29160898560, "loss": 3.1032493114471436, "l_mix": 1.9100370705127716, "l_a": 2.019944503903389, "l_c": 1.9574292600154877, "z_loss": 43.74198293685913, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17331109941005707, "tokens_per_second": 1418564.5204909123, "eta_seconds": 6230.926591157913, "data_wait_seconds": 2.396106719970703e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 55630, "total_steps": 72479, "tokens": 29166141440, "loss": 2.673058971762657, "l_mix": 1.6544305384159088, "l_a": 1.7152378857135773, "l_c": 1.6801899373531342, "z_loss": 48.560612201690674, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.22461450099945068, "tokens_per_second": 1364839.4121433797, "eta_seconds": 6472.357431507111, "data_wait_seconds": 3.4868717193603516e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 55640, "total_steps": 72479, "tokens": 29171384320, "loss": 4.000354528427124, "l_mix": 2.4764319956302643, "l_a": 2.5664880573749542, "l_c": 2.5132535994052887, "z_loss": 41.638405323028564, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12239030748605728, "tokens_per_second": 1436234.135660056, "eta_seconds": 6146.968250370025, "data_wait_seconds": 2.142786979675293e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 55650, "total_steps": 72479, "tokens": 29176627200, "loss": 3.953132927417755, "l_mix": 2.4461832642555237, "l_a": 2.540332943201065, "l_c": 2.4828324615955353, "z_loss": 43.501505851745605, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16592934727668762, "tokens_per_second": 1376368.9252806692, "eta_seconds": 6410.521619558334, "data_wait_seconds": 2.014636993408203e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 55660, "total_steps": 72479, "tokens": 29181870080, "loss": 3.95560160279274, "l_mix": 2.4486698508262634, "l_a": 2.5365184247493744, "l_c": 2.486587256193161, "z_loss": 42.10755920410156, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19548636674880981, "tokens_per_second": 1442452.4656932477, "eta_seconds": 6113.199624752999, "data_wait_seconds": 1.6897916793823242e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 55670, "total_steps": 72479, "tokens": 29187112960, "loss": 3.9155942797660828, "l_mix": 2.4226675629615784, "l_a": 2.5172682404518127, "l_c": 2.4591537714004517, "z_loss": 43.66246700286865, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2063606232404709, "tokens_per_second": 1388167.9694214892, "eta_seconds": 6348.48028922081, "data_wait_seconds": 2.065300941467285e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 55680, "total_steps": 72479, "tokens": 29192355840, "loss": 3.8378138840198517, "l_mix": 2.3750952780246735, "l_a": 2.4672344028949738, "l_c": 2.4084941148757935, "z_loss": 42.12840986251831, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1536542922258377, "tokens_per_second": 1380424.405113053, "eta_seconds": 6380.294407558441, "data_wait_seconds": 2.181529998779297e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 55690, "total_steps": 72479, "tokens": 29197598720, "loss": 4.828676998615265, "l_mix": 2.9965518414974213, "l_a": 3.071451634168625, "l_c": 3.035632014274597, "z_loss": 38.13242483139038, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20419488847255707, "tokens_per_second": 1445075.2078790003, "eta_seconds": 6091.220155191421, "data_wait_seconds": 2.1457672119140625e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 55700, "total_steps": 72479, "tokens": 29202841600, "loss": 4.68384063243866, "l_mix": 2.9064840376377106, "l_a": 2.981118619441986, "l_c": 2.943402975797653, "z_loss": 40.20908498764038, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2033320665359497, "tokens_per_second": 1437250.4686386965, "eta_seconds": 6120.734377169609, "data_wait_seconds": 1.71661376953125e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.16837381, "gate_a2b/L1_absmax": 0.70099407, "gate_a2b/L1_rms": 0.17900927, "gate_a2b/L3_absmean": 0.16166435, "gate_a2b/L3_absmax": 0.52822763, "gate_a2b/L3_rms": 0.16763549, "gate_a2b/L5_absmean": 0.24100323, "gate_a2b/L5_absmax": 0.739205, "gate_a2b/L5_rms": 0.24712247, "gate_c2b/L1_absmean": 0.02885874, "gate_c2b/L1_absmax": 0.63526511, "gate_c2b/L1_rms": 0.06771345, "gate_c2b/L3_absmean": 0.17554478, "gate_c2b/L3_absmax": 0.39728338, "gate_c2b/L3_rms": 0.17928174, "gate_c2b/L5_absmean": 0.4976224, "gate_c2b/L5_absmax": 0.70139247, "gate_c2b/L5_rms": 0.50766021, "gate_b2a/L7_absmean": 0.05059032, "gate_b2a/L7_absmax": 0.33265522, "gate_b2a/L7_rms": 0.05639783, "gate_b2a/L11_absmean": 0.08218923, "gate_b2a/L11_absmax": 0.60697067, "gate_b2a/L11_rms": 0.09131891, "gate_b2a/L14_absmean": 0.2133788, "gate_b2a/L14_absmax": 0.81921273, "gate_b2a/L14_rms": 0.22253084, "gate_b2c/L3_absmean": 0.03500666, "gate_b2c/L3_absmax": 0.55140388, "gate_b2c/L3_rms": 0.04565134, "gate_b2c/L5_absmean": 0.03729193, "gate_b2c/L5_absmax": 0.54761082, "gate_b2c/L5_rms": 0.04696868, "gate_b2c/L6_absmean": 0.05387298, "gate_b2c/L6_absmax": 0.35213593, "gate_b2c/L6_rms": 0.06015046}, "streams": {"cos_a2b/round0": 0.358745, "cos_c2b/round0": 0.276968, "cos_b2a/round0": 0.160307, "cos_b2c/round0": 0.145677, "rms_a/round0": 7.021142, "rms_b/round0": 12.225623, "rms_c/round0": 4.592652, "cos_a2b/round1": 0.406404, "cos_c2b/round1": 0.303302, "cos_b2a/round1": 0.248569, "cos_b2c/round1": 0.113101, "rms_a/round1": 13.704001, "rms_b/round1": 28.141199, "rms_c/round1": 9.742796, "cos_a2b/round2": 0.541253, "cos_c2b/round2": 0.522414, "cos_b2a/round2": 0.101707, "cos_b2c/round2": 0.18368, "rms_a/round2": 38.600998, "rms_b/round2": 76.569901, "rms_c/round2": 18.878189}, "alpha_mean": 0.4429045197550252, "alpha_hist": [0.01294, 0.06081, 0.15239, 0.18669, 0.20097, 0.19362, 0.12439, 0.06123, 0.00691, 5e-05], "alpha_by_type": {"special": {"share": 0.00073, "mean": 0.54145, "std": 0.11897}, "whitespace": {"share": 0.06253, "mean": 0.51079, "std": 0.11872}, "digit": {"share": 0.05132, "mean": 0.40981, "std": 0.16596}, "word_start": {"share": 0.52011, "mean": 0.45469, "std": 0.16094}, "word_cont": {"share": 0.22794, "mean": 0.32676, "std": 0.1244}, "punct": {"share": 0.13737, "mean": 0.57191, "std": 0.14775}}, "alpha_p05": 0.18418, "alpha_p50": 0.45133, "alpha_p95": 0.72584}
{"step": 55710, "total_steps": 72479, "tokens": 29208084480, "loss": 4.807471513748169, "l_mix": 2.9829728603363037, "l_a": 3.060953199863434, "l_c": 3.020708739757538, "z_loss": 39.08092784881592, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14579899609088898, "tokens_per_second": 1376516.7693223916, "eta_seconds": 6386.980288171768, "data_wait_seconds": 1.7136335372924805e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 55720, "total_steps": 72479, "tokens": 29213327360, "loss": 4.835145533084869, "l_mix": 3.0008866488933563, "l_a": 3.0765308141708374, "l_c": 3.0376650989055634, "z_loss": 38.230618476867676, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12978416681289673, "tokens_per_second": 1444914.8344085317, "eta_seconds": 6081.010716176033, "data_wait_seconds": 3.191828727722168e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 55730, "total_steps": 72479, "tokens": 29218570240, "loss": 4.815878927707672, "l_mix": 2.9887750148773193, "l_a": 3.0648608803749084, "l_c": 3.025485396385193, "z_loss": 39.984320640563965, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.21943296492099762, "tokens_per_second": 1429274.6644616595, "eta_seconds": 6143.8853779077535, "data_wait_seconds": 1.6391277313232422e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 55740, "total_steps": 72479, "tokens": 29223813120, "loss": 4.675983130931854, "l_mix": 2.901430308818817, "l_a": 2.9743869304656982, "l_c": 2.9407889544963837, "z_loss": 40.0741229057312, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1902759075164795, "tokens_per_second": 1347419.0063922377, "eta_seconds": 6513.235148358345, "data_wait_seconds": 3.552436828613281e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 55750, "total_steps": 72479, "tokens": 29229056000, "loss": 3.946687877178192, "l_mix": 2.4430855214595795, "l_a": 2.5352217257022858, "l_c": 2.47678604722023, "z_loss": 42.31257200241089, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13324187695980072, "tokens_per_second": 1416096.6796470936, "eta_seconds": 6193.654768109322, "data_wait_seconds": 2.1070241928100586e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 55760, "total_steps": 72479, "tokens": 29234298880, "loss": 3.9929012060165405, "l_mix": 2.471604287624359, "l_a": 2.5632841289043427, "l_c": 2.5077053606510162, "z_loss": 42.42776298522949, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1354924738407135, "tokens_per_second": 1416003.2141786849, "eta_seconds": 6190.360999345779, "data_wait_seconds": 2.434849739074707e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 55770, "total_steps": 72479, "tokens": 29239541760, "loss": 4.00962895154953, "l_mix": 2.4814821779727936, "l_a": 2.574171155691147, "l_c": 2.5196512937545776, "z_loss": 41.97392559051514, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18748833239078522, "tokens_per_second": 1380077.9572953545, "eta_seconds": 6347.705320334435, "data_wait_seconds": 2.059340476989746e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 55780, "total_steps": 72479, "tokens": 29244784640, "loss": 4.000811278820038, "l_mix": 2.478008419275284, "l_a": 2.566119998693466, "l_c": 2.5098894834518433, "z_loss": 40.236247062683105, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18764851987361908, "tokens_per_second": 1379170.5155101733, "eta_seconds": 6348.080395817757, "data_wait_seconds": 3.3020973205566406e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 55790, "total_steps": 72479, "tokens": 29250027520, "loss": 4.008009761571884, "l_mix": 2.480915665626526, "l_a": 2.5735158026218414, "l_c": 2.516797512769699, "z_loss": 42.28444480895996, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14470352232456207, "tokens_per_second": 1382246.7435528834, "eta_seconds": 6330.159555673599, "data_wait_seconds": 2.0295381546020508e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 55800, "total_steps": 72479, "tokens": 29255270400, "loss": 3.05598047375679, "l_mix": 1.8919412344694138, "l_a": 1.9579740464687347, "l_c": 1.9221565425395966, "z_loss": 48.57641363143921, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18607260286808014, "tokens_per_second": 1426349.8553051804, "eta_seconds": 6130.753629255295, "data_wait_seconds": 2.1398067474365234e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.16843198, "gate_a2b/L1_absmax": 0.69795561, "gate_a2b/L1_rms": 0.17913999, "gate_a2b/L3_absmean": 0.16216104, "gate_a2b/L3_absmax": 0.52730662, "gate_a2b/L3_rms": 0.16803414, "gate_a2b/L5_absmean": 0.24074912, "gate_a2b/L5_absmax": 0.73898017, "gate_a2b/L5_rms": 0.24687414, "gate_c2b/L1_absmean": 0.02875115, "gate_c2b/L1_absmax": 0.63261145, "gate_c2b/L1_rms": 0.06754255, "gate_c2b/L3_absmean": 0.17564563, "gate_c2b/L3_absmax": 0.39931792, "gate_c2b/L3_rms": 0.17932934, "gate_c2b/L5_absmean": 0.49689198, "gate_c2b/L5_absmax": 0.70355242, "gate_c2b/L5_rms": 0.50690651, "gate_b2a/L7_absmean": 0.05076527, "gate_b2a/L7_absmax": 0.33326808, "gate_b2a/L7_rms": 0.05655245, "gate_b2a/L11_absmean": 0.08216641, "gate_b2a/L11_absmax": 0.60861009, "gate_b2a/L11_rms": 0.09133121, "gate_b2a/L14_absmean": 0.21303402, "gate_b2a/L14_absmax": 0.81649506, "gate_b2a/L14_rms": 0.22214626, "gate_b2c/L3_absmean": 0.03500756, "gate_b2c/L3_absmax": 0.55085862, "gate_b2c/L3_rms": 0.04563009, "gate_b2c/L5_absmean": 0.03724068, "gate_b2c/L5_absmax": 0.54899198, "gate_b2c/L5_rms": 0.04692039, "gate_b2c/L6_absmean": 0.05372244, "gate_b2c/L6_absmax": 0.3500078, "gate_b2c/L6_rms": 0.06004495}, "streams": {"cos_a2b/round0": 0.363454, "cos_c2b/round0": 0.275286, "cos_b2a/round0": 0.152376, "cos_b2c/round0": 0.140353, "rms_a/round0": 7.204398, "rms_b/round0": 12.161751, "rms_c/round0": 4.68642, "cos_a2b/round1": 0.405687, "cos_c2b/round1": 0.285187, "cos_b2a/round1": 0.239181, "cos_b2c/round1": 0.105049, "rms_a/round1": 14.191919, "rms_b/round1": 28.137777, "rms_c/round1": 9.864178, "cos_a2b/round2": 0.510962, "cos_c2b/round2": 0.545475, "cos_b2a/round2": 0.078608, "cos_b2c/round2": 0.174384, "rms_a/round2": 38.094509, "rms_b/round2": 76.939507, "rms_c/round2": 19.044546}, "alpha_mean": 0.4368274178701128, "alpha_hist": [0.00642, 0.0464, 0.15607, 0.22267, 0.2233, 0.181, 0.11223, 0.04543, 0.00632, 0.00017], "alpha_by_type": {"special": {"share": 0.00075, "mean": 0.53264, "std": 0.14952}, "whitespace": {"share": 0.06129, "mean": 0.52423, "std": 0.12817}, "digit": {"share": 0.05724, "mean": 0.44929, "std": 0.12778}, "word_start": {"share": 0.46727, "mean": 0.43885, "std": 0.15142}, "word_cont": {"share": 0.24016, "mean": 0.33916, "std": 0.11763}, "punct": {"share": 0.17329, "mean": 0.53128, "std": 0.14667}}, "alpha_p05": 0.21734, "alpha_p50": 0.4371, "alpha_p95": 0.69925}
{"step": 55810, "total_steps": 72479, "tokens": 29260513280, "loss": 4.709224998950958, "l_mix": 2.922149807214737, "l_a": 2.997862786054611, "l_c": 2.959054410457611, "z_loss": 38.727869510650635, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19682537019252777, "tokens_per_second": 1406178.574318821, "eta_seconds": 6214.969301629067, "data_wait_seconds": 2.0831823348999023e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 55820, "total_steps": 72479, "tokens": 29265756160, "loss": 4.709303200244904, "l_mix": 2.9220584630966187, "l_a": 2.998756021261215, "l_c": 2.958726227283478, "z_loss": 39.7741813659668, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14888499677181244, "tokens_per_second": 1431923.9714053213, "eta_seconds": 6099.565316605567, "data_wait_seconds": 2.5987625122070312e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 55830, "total_steps": 72479, "tokens": 29270999040, "loss": 4.758077621459961, "l_mix": 2.95183527469635, "l_a": 3.0283784568309784, "l_c": 2.9924290776252747, "z_loss": 40.65858840942383, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13869263231754303, "tokens_per_second": 1470521.2789347414, "eta_seconds": 5935.902483725547, "data_wait_seconds": 1.5437602996826172e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 55840, "total_steps": 72479, "tokens": 29276241920, "loss": 4.836864769458771, "l_mix": 3.001499593257904, "l_a": 3.0792604982852936, "l_c": 3.038623034954071, "z_loss": 39.14629411697388, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1267133504152298, "tokens_per_second": 1313771.4625521142, "eta_seconds": 6640.141212272644, "data_wait_seconds": 1.96993350982666e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 55850, "total_steps": 72479, "tokens": 29281484800, "loss": 4.7221996784210205, "l_mix": 2.92989981174469, "l_a": 3.0061204731464386, "l_c": 2.968212127685547, "z_loss": 39.14281940460205, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1478768289089203, "tokens_per_second": 1403003.7076497856, "eta_seconds": 6214.085611081124, "data_wait_seconds": 2.0772218704223633e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 55860, "total_steps": 72479, "tokens": 29286727680, "loss": 4.058829694986343, "l_mix": 2.5135431587696075, "l_a": 2.6002804338932037, "l_c": 2.5506748259067535, "z_loss": 41.950647830963135, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1907547414302826, "tokens_per_second": 1429532.3142082698, "eta_seconds": 6095.099904632569, "data_wait_seconds": 2.1278858184814453e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 55870, "total_steps": 72479, "tokens": 29291970560, "loss": 4.168063133955002, "l_mix": 2.582590162754059, "l_a": 2.6683789789676666, "l_c": 2.616530418395996, "z_loss": 41.338958740234375, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12011595815420151, "tokens_per_second": 1360877.888494348, "eta_seconds": 6398.736775445937, "data_wait_seconds": 1.7523765563964844e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 55880, "total_steps": 72479, "tokens": 29297213440, "loss": 4.0046098828315735, "l_mix": 2.4798313975334167, "l_a": 2.567623496055603, "l_c": 2.5149711966514587, "z_loss": 41.66449308395386, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.10995806008577347, "tokens_per_second": 1388927.4382158048, "eta_seconds": 6265.738779830933, "data_wait_seconds": 2.4646520614624023e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 55890, "total_steps": 72479, "tokens": 29302456320, "loss": 3.9210841953754425, "l_mix": 2.4278957843780518, "l_a": 2.516103446483612, "l_c": 2.461190849542618, "z_loss": 42.50539541244507, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16688893735408783, "tokens_per_second": 1443099.0901497551, "eta_seconds": 6026.899809837341, "data_wait_seconds": 1.7255544662475586e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 55900, "total_steps": 72479, "tokens": 29307699200, "loss": 3.9549486339092255, "l_mix": 2.447721093893051, "l_a": 2.5370099246501923, "l_c": 2.4870817959308624, "z_loss": 42.125988483428955, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.29608744382858276, "tokens_per_second": 1450647.2500247543, "eta_seconds": 5991.925846791268, "data_wait_seconds": 1.5527009963989258e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16789423, "gate_a2b/L1_absmax": 0.70158333, "gate_a2b/L1_rms": 0.17852205, "gate_a2b/L3_absmean": 0.16171117, "gate_a2b/L3_absmax": 0.52551031, "gate_a2b/L3_rms": 0.16755961, "gate_a2b/L5_absmean": 0.24023698, "gate_a2b/L5_absmax": 0.73771083, "gate_a2b/L5_rms": 0.24632047, "gate_c2b/L1_absmean": 0.02878057, "gate_c2b/L1_absmax": 0.63455057, "gate_c2b/L1_rms": 0.0679693, "gate_c2b/L3_absmean": 0.17493121, "gate_c2b/L3_absmax": 0.40059954, "gate_c2b/L3_rms": 0.17872883, "gate_c2b/L5_absmean": 0.49695835, "gate_c2b/L5_absmax": 0.70132583, "gate_c2b/L5_rms": 0.50699991, "gate_b2a/L7_absmean": 0.05055511, "gate_b2a/L7_absmax": 0.33252817, "gate_b2a/L7_rms": 0.05641119, "gate_b2a/L11_absmean": 0.08235845, "gate_b2a/L11_absmax": 0.60882568, "gate_b2a/L11_rms": 0.09153128, "gate_b2a/L14_absmean": 0.21298486, "gate_b2a/L14_absmax": 0.81882197, "gate_b2a/L14_rms": 0.22214201, "gate_b2c/L3_absmean": 0.03534072, "gate_b2c/L3_absmax": 0.54945242, "gate_b2c/L3_rms": 0.0458376, "gate_b2c/L5_absmean": 0.03734097, "gate_b2c/L5_absmax": 0.54991066, "gate_b2c/L5_rms": 0.0469996, "gate_b2c/L6_absmean": 0.05379073, "gate_b2c/L6_absmax": 0.35232478, "gate_b2c/L6_rms": 0.06008808}, "streams": {"cos_a2b/round0": 0.358097, "cos_c2b/round0": 0.272812, "cos_b2a/round0": 0.163055, "cos_b2c/round0": 0.142812, "rms_a/round0": 7.06559, "rms_b/round0": 12.310427, "rms_c/round0": 4.666983, "cos_a2b/round1": 0.416248, "cos_c2b/round1": 0.305805, "cos_b2a/round1": 0.24998, "cos_b2c/round1": 0.113521, "rms_a/round1": 13.940651, "rms_b/round1": 28.780371, "rms_c/round1": 9.891048, "cos_a2b/round2": 0.52922, "cos_c2b/round2": 0.528253, "cos_b2a/round2": 0.098624, "cos_b2c/round2": 0.179003, "rms_a/round2": 38.983776, "rms_b/round2": 78.062569, "rms_c/round2": 19.239717}, "alpha_mean": 0.4201463093011526, "alpha_hist": [0.01523, 0.09776, 0.19305, 0.19174, 0.16333, 0.14154, 0.11339, 0.07047, 0.0134, 9e-05], "alpha_by_type": {"special": {"share": 0.00104, "mean": 0.54896, "std": 0.09742}, "whitespace": {"share": 0.02477, "mean": 0.53518, "std": 0.11271}, "digit": {"share": 0.02191, "mean": 0.42574, "std": 0.12185}, "word_start": {"share": 0.60123, "mean": 0.43345, "std": 0.17711}, "word_cont": {"share": 0.25018, "mean": 0.30607, "std": 0.13345}, "punct": {"share": 0.10088, "mean": 0.59298, "std": 0.16961}}, "alpha_p05": 0.16133, "alpha_p50": 0.40333, "alpha_p95": 0.7364}
{"step": 55910, "total_steps": 72479, "tokens": 29312942080, "loss": 2.8116978257894516, "l_mix": 1.740682378411293, "l_a": 1.8032947927713394, "l_c": 1.7667566388845444, "z_loss": 50.124064445495605, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20552438497543335, "tokens_per_second": 1424886.903180704, "eta_seconds": 6096.573596549034, "data_wait_seconds": 3.376603126525879e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 55920, "total_steps": 72479, "tokens": 29318184960, "loss": 3.048263758420944, "l_mix": 1.8882767260074615, "l_a": 1.949280470609665, "l_c": 1.9173429608345032, "z_loss": 50.507253646850586, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2384517341852188, "tokens_per_second": 1347317.8766249744, "eta_seconds": 6443.679804611206, "data_wait_seconds": 2.199411392211914e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 55930, "total_steps": 72479, "tokens": 29323427840, "loss": 4.730469644069672, "l_mix": 2.93557271361351, "l_a": 3.0093931555747986, "l_c": 2.973596215248108, "z_loss": 38.423287868499756, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1282697170972824, "tokens_per_second": 1462767.1853716846, "eta_seconds": 5931.52635550499, "data_wait_seconds": 3.1828880310058594e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 55940, "total_steps": 72479, "tokens": 29328670720, "loss": 4.6873844265937805, "l_mix": 2.908643424510956, "l_a": 2.9859710335731506, "l_c": 2.9431653320789337, "z_loss": 38.628761768341064, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13200679421424866, "tokens_per_second": 1378781.470747382, "eta_seconds": 6289.030869626999, "data_wait_seconds": 2.92360782623291e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 55950, "total_steps": 72479, "tokens": 29333913600, "loss": 4.734693288803101, "l_mix": 2.9383743703365326, "l_a": 3.0107757449150085, "l_c": 2.976953625679016, "z_loss": 38.76486158370972, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18928496539592743, "tokens_per_second": 1442902.514069955, "eta_seconds": 6005.919504261017, "data_wait_seconds": 1.52587890625e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 55960, "total_steps": 72479, "tokens": 29339156480, "loss": 4.764964938163757, "l_mix": 2.957032859325409, "l_a": 3.0329343378543854, "l_c": 2.9935054779052734, "z_loss": 38.86460828781128, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13410022854804993, "tokens_per_second": 1430413.3730930707, "eta_seconds": 6054.692744708061, "data_wait_seconds": 1.6957521438598633e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 55970, "total_steps": 72479, "tokens": 29344399360, "loss": 4.006145238876343, "l_mix": 2.480248510837555, "l_a": 2.568194717168808, "l_c": 2.5181277096271515, "z_loss": 43.50192451477051, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12621790170669556, "tokens_per_second": 1302926.3069071784, "eta_seconds": 6643.100646686554, "data_wait_seconds": 2.244114875793457e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 55980, "total_steps": 72479, "tokens": 29349642240, "loss": 3.958275228738785, "l_mix": 2.4505776166915894, "l_a": 2.540166288614273, "l_c": 2.4854921996593475, "z_loss": 42.5265679359436, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12291587889194489, "tokens_per_second": 1404090.0765990398, "eta_seconds": 6160.735593938828, "data_wait_seconds": 1.9550323486328125e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 55990, "total_steps": 72479, "tokens": 29354885120, "loss": 4.0370913445949554, "l_mix": 2.499743700027466, "l_a": 2.588451325893402, "l_c": 2.5360403656959534, "z_loss": 42.37573766708374, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1232568621635437, "tokens_per_second": 1450271.8360728717, "eta_seconds": 5960.94098842144, "data_wait_seconds": 1.6242265701293945e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 56000, "total_steps": 72479, "tokens": 29360128000, "loss": 4.013827860355377, "l_mix": 2.4841509461402893, "l_a": 2.577042430639267, "l_c": 2.5218802988529205, "z_loss": 42.63162422180176, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15339460968971252, "tokens_per_second": 1393197.968519634, "eta_seconds": 6201.374210429191, "data_wait_seconds": 2.2113323211669922e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16813116, "gate_a2b/L1_absmax": 0.70170659, "gate_a2b/L1_rms": 0.17877579, "gate_a2b/L3_absmean": 0.16218343, "gate_a2b/L3_absmax": 0.52700233, "gate_a2b/L3_rms": 0.16805278, "gate_a2b/L5_absmean": 0.24058905, "gate_a2b/L5_absmax": 0.73832119, "gate_a2b/L5_rms": 0.24660558, "gate_c2b/L1_absmean": 0.02918924, "gate_c2b/L1_absmax": 0.63423961, "gate_c2b/L1_rms": 0.06784125, "gate_c2b/L3_absmean": 0.17465594, "gate_c2b/L3_absmax": 0.39508715, "gate_c2b/L3_rms": 0.1783787, "gate_c2b/L5_absmean": 0.49605656, "gate_c2b/L5_absmax": 0.70008045, "gate_c2b/L5_rms": 0.50603902, "gate_b2a/L7_absmean": 0.05066909, "gate_b2a/L7_absmax": 0.33169398, "gate_b2a/L7_rms": 0.05644878, "gate_b2a/L11_absmean": 0.08204193, "gate_b2a/L11_absmax": 0.60850149, "gate_b2a/L11_rms": 0.09124278, "gate_b2a/L14_absmean": 0.21326828, "gate_b2a/L14_absmax": 0.81668186, "gate_b2a/L14_rms": 0.22238705, "gate_b2c/L3_absmean": 0.03510296, "gate_b2c/L3_absmax": 0.54973733, "gate_b2c/L3_rms": 0.04570446, "gate_b2c/L5_absmean": 0.03762638, "gate_b2c/L5_absmax": 0.5496369, "gate_b2c/L5_rms": 0.04723725, "gate_b2c/L6_absmean": 0.05395605, "gate_b2c/L6_absmax": 0.35012299, "gate_b2c/L6_rms": 0.06018977}, "streams": {"cos_a2b/round0": 0.357517, "cos_c2b/round0": 0.272378, "cos_b2a/round0": 0.16025, "cos_b2c/round0": 0.141372, "rms_a/round0": 7.076478, "rms_b/round0": 12.344613, "rms_c/round0": 4.702294, "cos_a2b/round1": 0.417239, "cos_c2b/round1": 0.301677, "cos_b2a/round1": 0.247557, "cos_b2c/round1": 0.11854, "rms_a/round1": 14.103301, "rms_b/round1": 29.19549, "rms_c/round1": 9.966478, "cos_a2b/round2": 0.530784, "cos_c2b/round2": 0.522926, "cos_b2a/round2": 0.099409, "cos_b2c/round2": 0.179771, "rms_a/round2": 39.413155, "rms_b/round2": 78.605728, "rms_c/round2": 19.50355}, "alpha_mean": 0.4071133558557918, "alpha_hist": [0.02217, 0.12973, 0.19946, 0.17441, 0.14854, 0.13049, 0.11021, 0.06927, 0.01537, 0.00034], "alpha_by_type": {"special": {"share": 0.00107, "mean": 0.57437, "std": 0.11323}, "whitespace": {"share": 0.0248, "mean": 0.51966, "std": 0.12324}, "digit": {"share": 0.02426, "mean": 0.44229, "std": 0.12208}, "word_start": {"share": 0.59995, "mean": 0.41714, "std": 0.18674}, "word_cont": {"share": 0.24841, "mean": 0.29211, "std": 0.13846}, "punct": {"share": 0.10152, "mean": 0.59163, "std": 0.1753}}, "alpha_p05": 0.13296, "alpha_p50": 0.3803, "alpha_p95": 0.75851}
{"step": 56010, "total_steps": 72479, "tokens": 29365370880, "loss": 4.034745037555695, "l_mix": 2.4978953897953033, "l_a": 2.586791515350342, "l_c": 2.5360405445098877, "z_loss": 41.668156147003174, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14591173827648163, "tokens_per_second": 1312936.4761909936, "eta_seconds": 6576.478929924965, "data_wait_seconds": 2.1696090698242188e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 56020, "total_steps": 72479, "tokens": 29370613760, "loss": 3.995872676372528, "l_mix": 2.4745983481407166, "l_a": 2.5648086071014404, "l_c": 2.5061057806015015, "z_loss": 40.7113733291626, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19230365753173828, "tokens_per_second": 1462867.8022888883, "eta_seconds": 5898.862616634369, "data_wait_seconds": 1.9162893295288086e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 56030, "total_steps": 72479, "tokens": 29375856640, "loss": 2.9755138158798218, "l_mix": 1.842667281627655, "l_a": 1.9066852629184723, "l_c": 1.8694696426391602, "z_loss": 48.9225549697876, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1769724041223526, "tokens_per_second": 1376798.760450944, "eta_seconds": 6263.815424394607, "data_wait_seconds": 2.03549861907959e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 56040, "total_steps": 72479, "tokens": 29381099520, "loss": 2.9502944946289062, "l_mix": 1.8271364867687225, "l_a": 1.8925524204969406, "l_c": 1.8513074666261673, "z_loss": 49.104498863220215, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16328367590904236, "tokens_per_second": 1309532.4831200726, "eta_seconds": 6581.5629189014435, "data_wait_seconds": 3.129243850708008e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 56050, "total_steps": 72479, "tokens": 29386342400, "loss": 4.017424076795578, "l_mix": 2.4876788854599, "l_a": 2.575909346342087, "l_c": 2.523241102695465, "z_loss": 41.54303455352783, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1570999175310135, "tokens_per_second": 1461362.7196503293, "eta_seconds": 5894.174961614609, "data_wait_seconds": 2.1845102310180664e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 56060, "total_steps": 72479, "tokens": 29391585280, "loss": 3.841087222099304, "l_mix": 2.3766476213932037, "l_a": 2.467761814594269, "l_c": 2.4137034118175507, "z_loss": 43.11329364776611, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16531875729560852, "tokens_per_second": 1466780.1851045012, "eta_seconds": 5868.830762386322, "data_wait_seconds": 1.609325408935547e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 56070, "total_steps": 72479, "tokens": 29396828160, "loss": 3.946597099304199, "l_mix": 2.442268669605255, "l_a": 2.5385242998600006, "l_c": 2.4759037792682648, "z_loss": 44.24205923080444, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.24316762387752533, "tokens_per_second": 1393602.3463680565, "eta_seconds": 6173.240031075478, "data_wait_seconds": 3.1501054763793945e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 56080, "total_steps": 72479, "tokens": 29402071040, "loss": 3.9982213377952576, "l_mix": 2.476207435131073, "l_a": 2.5601821541786194, "l_c": 2.5131973922252655, "z_loss": 40.603272914886475, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.30250993371009827, "tokens_per_second": 1449614.7556380122, "eta_seconds": 5931.092297840119, "data_wait_seconds": 2.2530555725097656e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 56090, "total_steps": 72479, "tokens": 29407313920, "loss": 3.975902110338211, "l_mix": 2.4606827795505524, "l_a": 2.5546637177467346, "l_c": 2.4960670471191406, "z_loss": 42.79519462585449, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13763262331485748, "tokens_per_second": 1390159.5041349065, "eta_seconds": 6180.985711669921, "data_wait_seconds": 1.9073486328125e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 56100, "total_steps": 72479, "tokens": 29412556800, "loss": 3.9777325987815857, "l_mix": 2.461726635694504, "l_a": 2.5551369190216064, "l_c": 2.4982159435749054, "z_loss": 42.26242208480835, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11512427031993866, "tokens_per_second": 1402400.289247262, "eta_seconds": 6123.296763300896, "data_wait_seconds": 2.3633241653442383e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16768217, "gate_a2b/L1_absmax": 0.69904554, "gate_a2b/L1_rms": 0.17841204, "gate_a2b/L3_absmean": 0.16170882, "gate_a2b/L3_absmax": 0.52285409, "gate_a2b/L3_rms": 0.16756456, "gate_a2b/L5_absmean": 0.24012008, "gate_a2b/L5_absmax": 0.73468316, "gate_a2b/L5_rms": 0.24609888, "gate_c2b/L1_absmean": 0.02934743, "gate_c2b/L1_absmax": 0.6324501, "gate_c2b/L1_rms": 0.06771562, "gate_c2b/L3_absmean": 0.17469582, "gate_c2b/L3_absmax": 0.39380893, "gate_c2b/L3_rms": 0.17845076, "gate_c2b/L5_absmean": 0.49527693, "gate_c2b/L5_absmax": 0.69499749, "gate_c2b/L5_rms": 0.50529796, "gate_b2a/L7_absmean": 0.05079767, "gate_b2a/L7_absmax": 0.33116555, "gate_b2a/L7_rms": 0.05655859, "gate_b2a/L11_absmean": 0.08242506, "gate_b2a/L11_absmax": 0.60584807, "gate_b2a/L11_rms": 0.0914969, "gate_b2a/L14_absmean": 0.21276048, "gate_b2a/L14_absmax": 0.8160764, "gate_b2a/L14_rms": 0.22192061, "gate_b2c/L3_absmean": 0.03501632, "gate_b2c/L3_absmax": 0.54964811, "gate_b2c/L3_rms": 0.04562059, "gate_b2c/L5_absmean": 0.03742782, "gate_b2c/L5_absmax": 0.5483911, "gate_b2c/L5_rms": 0.04709997, "gate_b2c/L6_absmean": 0.05379551, "gate_b2c/L6_absmax": 0.34903839, "gate_b2c/L6_rms": 0.06004803}, "streams": {"cos_a2b/round0": 0.358588, "cos_c2b/round0": 0.271529, "cos_b2a/round0": 0.163549, "cos_b2c/round0": 0.142221, "rms_a/round0": 7.116642, "rms_b/round0": 12.297268, "rms_c/round0": 4.695889, "cos_a2b/round1": 0.416232, "cos_c2b/round1": 0.303384, "cos_b2a/round1": 0.253443, "cos_b2c/round1": 0.116717, "rms_a/round1": 14.042288, "rms_b/round1": 28.886642, "rms_c/round1": 9.912781, "cos_a2b/round2": 0.532272, "cos_c2b/round2": 0.530903, "cos_b2a/round2": 0.096814, "cos_b2c/round2": 0.18412, "rms_a/round2": 39.135468, "rms_b/round2": 78.341606, "rms_c/round2": 19.448601}, "alpha_mean": 0.41442611522721506, "alpha_hist": [0.01283, 0.10033, 0.20502, 0.20305, 0.1627, 0.12675, 0.10289, 0.06871, 0.01747, 0.00024], "alpha_by_type": {"special": {"share": 0.00125, "mean": 0.55112, "std": 0.09821}, "whitespace": {"share": 0.02435, "mean": 0.52103, "std": 0.12038}, "digit": {"share": 0.02765, "mean": 0.38642, "std": 0.10537}, "word_start": {"share": 0.59456, "mean": 0.42762, "std": 0.18219}, "word_cont": {"share": 0.2518, "mean": 0.30848, "std": 0.12546}, "punct": {"share": 0.10039, "mean": 0.58218, "std": 0.17423}}, "alpha_p05": 0.15688, "alpha_p50": 0.40521, "alpha_p95": 0.74354}
{"step": 56110, "total_steps": 72479, "tokens": 29417799680, "loss": 3.990531414747238, "l_mix": 2.4695676267147064, "l_a": 2.5631551146507263, "l_c": 2.506724238395691, "z_loss": 43.13737964630127, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16353249549865723, "tokens_per_second": 1419879.6385808534, "eta_seconds": 6044.2237770080565, "data_wait_seconds": 1.9073486328125e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 56120, "total_steps": 72479, "tokens": 29423042560, "loss": 3.9138955175876617, "l_mix": 2.422870635986328, "l_a": 2.513267308473587, "l_c": 2.456815391778946, "z_loss": 42.749799728393555, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15219978988170624, "tokens_per_second": 1292423.2447720298, "eta_seconds": 6636.237336874008, "data_wait_seconds": 1.7881393432617188e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 56130, "total_steps": 72479, "tokens": 29428285440, "loss": 3.924620747566223, "l_mix": 2.428911715745926, "l_a": 2.5197875797748566, "l_c": 2.4659091532230377, "z_loss": 42.138906955718994, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2021770179271698, "tokens_per_second": 1322851.090863569, "eta_seconds": 6479.629167032242, "data_wait_seconds": 2.396106719970703e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 56140, "total_steps": 72479, "tokens": 29433528320, "loss": 3.0339758098125458, "l_mix": 1.8779383152723312, "l_a": 1.9465472102165222, "l_c": 1.906911090016365, "z_loss": 51.03302812576294, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1654427945613861, "tokens_per_second": 1410326.0528732636, "eta_seconds": 6074.015022659302, "data_wait_seconds": 1.6480684280395508e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 56150, "total_steps": 72479, "tokens": 29438771200, "loss": 3.0194490253925323, "l_mix": 1.8706195205450058, "l_a": 1.9325389862060547, "l_c": 1.8968925774097443, "z_loss": 46.713536739349365, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20909304916858673, "tokens_per_second": 1415191.5411903586, "eta_seconds": 6049.427588295936, "data_wait_seconds": 2.0444393157958984e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 56160, "total_steps": 72479, "tokens": 29444014080, "loss": 4.062684744596481, "l_mix": 2.515705406665802, "l_a": 2.6055657863616943, "l_c": 2.5510318279266357, "z_loss": 42.544376850128174, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12751679122447968, "tokens_per_second": 1454273.9863900177, "eta_seconds": 5883.248928380012, "data_wait_seconds": 1.862645149230957e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 56170, "total_steps": 72479, "tokens": 29449256960, "loss": 3.866942882537842, "l_mix": 2.3924046754837036, "l_a": 2.48514923453331, "l_c": 2.42997807264328, "z_loss": 43.36293935775757, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16954320669174194, "tokens_per_second": 1390995.3238331275, "eta_seconds": 6147.11843059063, "data_wait_seconds": 1.633167266845703e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 56180, "total_steps": 72479, "tokens": 29454499840, "loss": 3.9785163700580597, "l_mix": 2.462869942188263, "l_a": 2.553054839372635, "l_c": 2.4990999698638916, "z_loss": 41.02357625961304, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.26247695088386536, "tokens_per_second": 1414070.9279621756, "eta_seconds": 6043.098647332192, "data_wait_seconds": 1.8715858459472656e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 56190, "total_steps": 72479, "tokens": 29459742720, "loss": 3.8852302730083466, "l_mix": 2.404819369316101, "l_a": 2.49527245759964, "l_c": 2.439430147409439, "z_loss": 42.547306060791016, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17209985852241516, "tokens_per_second": 1420682.9320233988, "eta_seconds": 6011.283052325249, "data_wait_seconds": 2.1457672119140625e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 56200, "total_steps": 72479, "tokens": 29464985600, "loss": 3.923228442668915, "l_mix": 2.427539885044098, "l_a": 2.52131986618042, "l_c": 2.464308500289917, "z_loss": 43.704084396362305, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19431351125240326, "tokens_per_second": 1301562.4002616592, "eta_seconds": 6557.4146504879, "data_wait_seconds": 2.1249055862426758e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.16734627, "gate_a2b/L1_absmax": 0.69911128, "gate_a2b/L1_rms": 0.17798677, "gate_a2b/L3_absmean": 0.16167107, "gate_a2b/L3_absmax": 0.51601279, "gate_a2b/L3_rms": 0.16748796, "gate_a2b/L5_absmean": 0.23980515, "gate_a2b/L5_absmax": 0.73255414, "gate_a2b/L5_rms": 0.24581964, "gate_c2b/L1_absmean": 0.02976713, "gate_c2b/L1_absmax": 0.63089007, "gate_c2b/L1_rms": 0.06793354, "gate_c2b/L3_absmean": 0.17488526, "gate_c2b/L3_absmax": 0.39891836, "gate_c2b/L3_rms": 0.1786359, "gate_c2b/L5_absmean": 0.49548513, "gate_c2b/L5_absmax": 0.70348012, "gate_c2b/L5_rms": 0.50554228, "gate_b2a/L7_absmean": 0.05062301, "gate_b2a/L7_absmax": 0.33553267, "gate_b2a/L7_rms": 0.05641832, "gate_b2a/L11_absmean": 0.08247286, "gate_b2a/L11_absmax": 0.61101431, "gate_b2a/L11_rms": 0.0915864, "gate_b2a/L14_absmean": 0.21303709, "gate_b2a/L14_absmax": 0.81664288, "gate_b2a/L14_rms": 0.22222164, "gate_b2c/L3_absmean": 0.03511023, "gate_b2c/L3_absmax": 0.55551213, "gate_b2c/L3_rms": 0.04589, "gate_b2c/L5_absmean": 0.03713412, "gate_b2c/L5_absmax": 0.56207043, "gate_b2c/L5_rms": 0.04724444, "gate_b2c/L6_absmean": 0.05365008, "gate_b2c/L6_absmax": 0.34950835, "gate_b2c/L6_rms": 0.05976675}, "streams": {"cos_a2b/round0": 0.359173, "cos_c2b/round0": 0.270035, "cos_b2a/round0": 0.159361, "cos_b2c/round0": 0.14215, "rms_a/round0": 7.162536, "rms_b/round0": 12.191968, "rms_c/round0": 4.666691, "cos_a2b/round1": 0.412826, "cos_c2b/round1": 0.299878, "cos_b2a/round1": 0.249818, "cos_b2c/round1": 0.114546, "rms_a/round1": 14.004894, "rms_b/round1": 29.056089, "rms_c/round1": 9.857086, "cos_a2b/round2": 0.529597, "cos_c2b/round2": 0.526591, "cos_b2a/round2": 0.098979, "cos_b2c/round2": 0.179969, "rms_a/round2": 39.024834, "rms_b/round2": 78.594757, "rms_c/round2": 19.16135}, "alpha_mean": 0.4171951511906755, "alpha_hist": [0.01689, 0.10286, 0.19489, 0.19429, 0.16124, 0.13553, 0.10666, 0.0688, 0.01833, 0.00052], "alpha_by_type": {"special": {"share": 0.00117, "mean": 0.56838, "std": 0.10457}, "whitespace": {"share": 0.02803, "mean": 0.51689, "std": 0.12764}, "digit": {"share": 0.03368, "mean": 0.4228, "std": 0.1168}, "word_start": {"share": 0.58267, "mean": 0.42827, "std": 0.18203}, "word_cont": {"share": 0.24232, "mean": 0.30165, "std": 0.13298}, "punct": {"share": 0.11212, "mean": 0.58119, "std": 0.17655}}, "alpha_p05": 0.14953, "alpha_p50": 0.40005, "alpha_p95": 0.73163}
{"step": 56210, "total_steps": 72479, "tokens": 29470228480, "loss": 4.63045346736908, "l_mix": 2.873367577791214, "l_a": 2.947666734457016, "l_c": 2.9092861711978912, "z_loss": 38.532555103302, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.128043994307518, "tokens_per_second": 1440123.4285708298, "eta_seconds": 5922.85446009636, "data_wait_seconds": 1.65402889251709e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 56220, "total_steps": 72479, "tokens": 29475471360, "loss": 4.772383272647858, "l_mix": 2.961855798959732, "l_a": 3.0376651287078857, "l_c": 2.9974265098571777, "z_loss": 39.563722133636475, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13974539935588837, "tokens_per_second": 1366316.482990689, "eta_seconds": 6238.963445234299, "data_wait_seconds": 1.7374753952026367e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 56230, "total_steps": 72479, "tokens": 29480714240, "loss": 4.71706885099411, "l_mix": 2.926403522491455, "l_a": 3.007305920124054, "l_c": 2.9615782499313354, "z_loss": 40.1077299118042, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2445342242717743, "tokens_per_second": 1350247.5308212691, "eta_seconds": 6309.32885825634, "data_wait_seconds": 2.720952033996582e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 56240, "total_steps": 72479, "tokens": 29485957120, "loss": 4.70563679933548, "l_mix": 2.9194678962230682, "l_a": 2.995242118835449, "l_c": 2.9586537778377533, "z_loss": 40.33475399017334, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16107742488384247, "tokens_per_second": 1291815.3264965557, "eta_seconds": 6590.657857489586, "data_wait_seconds": 1.9937753677368164e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 56250, "total_steps": 72479, "tokens": 29491200000, "loss": 4.685423016548157, "l_mix": 2.9071576297283173, "l_a": 2.984721153974533, "l_c": 2.9428299963474274, "z_loss": 39.455928802490234, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15689945220947266, "tokens_per_second": 1358747.212204597, "eta_seconds": 6262.143447709083, "data_wait_seconds": 2.9742717742919922e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 56260, "total_steps": 72479, "tokens": 29496442880, "loss": 2.948234438896179, "l_mix": 1.8263592422008514, "l_a": 1.887984111905098, "l_c": 1.8515996485948563, "z_loss": 48.57801103591919, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16249549388885498, "tokens_per_second": 1442320.8642342447, "eta_seconds": 5895.655594301224, "data_wait_seconds": 2.5480985641479492e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 56270, "total_steps": 72479, "tokens": 29501685760, "loss": 3.946451812982559, "l_mix": 2.4427742063999176, "l_a": 2.534720540046692, "l_c": 2.4775378108024597, "z_loss": 42.69976997375488, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12315016239881516, "tokens_per_second": 1424935.7460901511, "eta_seconds": 5963.906944799423, "data_wait_seconds": 1.5735626220703125e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 56280, "total_steps": 72479, "tokens": 29506928640, "loss": 3.9746224880218506, "l_mix": 2.459884762763977, "l_a": 2.5531218349933624, "l_c": 2.4960035383701324, "z_loss": 43.63626956939697, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16997317969799042, "tokens_per_second": 1174209.1795766014, "eta_seconds": 7232.903182601929, "data_wait_seconds": 1.9043684005737305e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 56290, "total_steps": 72479, "tokens": 29512171520, "loss": 3.8658120334148407, "l_mix": 2.392462342977524, "l_a": 2.481685131788254, "l_c": 2.429480254650116, "z_loss": 42.7937970161438, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17348681390285492, "tokens_per_second": 1358396.1184207438, "eta_seconds": 6248.323531627655, "data_wait_seconds": 1.6421079635620117e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 56300, "total_steps": 72479, "tokens": 29517414400, "loss": 3.9463073313236237, "l_mix": 2.4433418810367584, "l_a": 2.5352515280246735, "l_c": 2.4746331572532654, "z_loss": 41.288156509399414, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18548095226287842, "tokens_per_second": 1291627.6836244604, "eta_seconds": 6567.260565519333, "data_wait_seconds": 1.576542854309082e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16801606, "gate_a2b/L1_absmax": 0.6986866, "gate_a2b/L1_rms": 0.17870501, "gate_a2b/L3_absmean": 0.16171086, "gate_a2b/L3_absmax": 0.52423388, "gate_a2b/L3_rms": 0.16757968, "gate_a2b/L5_absmean": 0.2393931, "gate_a2b/L5_absmax": 0.73434073, "gate_a2b/L5_rms": 0.24541202, "gate_c2b/L1_absmean": 0.02986134, "gate_c2b/L1_absmax": 0.63145024, "gate_c2b/L1_rms": 0.06789399, "gate_c2b/L3_absmean": 0.17419377, "gate_c2b/L3_absmax": 0.39788765, "gate_c2b/L3_rms": 0.17788714, "gate_c2b/L5_absmean": 0.49458423, "gate_c2b/L5_absmax": 0.6961416, "gate_c2b/L5_rms": 0.50453132, "gate_b2a/L7_absmean": 0.05046758, "gate_b2a/L7_absmax": 0.33362117, "gate_b2a/L7_rms": 0.05641153, "gate_b2a/L11_absmean": 0.08227171, "gate_b2a/L11_absmax": 0.61058319, "gate_b2a/L11_rms": 0.09144129, "gate_b2a/L14_absmean": 0.21286742, "gate_b2a/L14_absmax": 0.81516296, "gate_b2a/L14_rms": 0.22201775, "gate_b2c/L3_absmean": 0.03497734, "gate_b2c/L3_absmax": 0.55412078, "gate_b2c/L3_rms": 0.0457783, "gate_b2c/L5_absmean": 0.0373657, "gate_b2c/L5_absmax": 0.54694128, "gate_b2c/L5_rms": 0.04698909, "gate_b2c/L6_absmean": 0.0538173, "gate_b2c/L6_absmax": 0.34869963, "gate_b2c/L6_rms": 0.05997989}, "streams": {"cos_a2b/round0": 0.360275, "cos_c2b/round0": 0.273034, "cos_b2a/round0": 0.164223, "cos_b2c/round0": 0.145122, "rms_a/round0": 7.113377, "rms_b/round0": 12.265234, "rms_c/round0": 4.690082, "cos_a2b/round1": 0.419756, "cos_c2b/round1": 0.30216, "cos_b2a/round1": 0.252848, "cos_b2c/round1": 0.117719, "rms_a/round1": 13.994308, "rms_b/round1": 28.843134, "rms_c/round1": 9.869723, "cos_a2b/round2": 0.536992, "cos_c2b/round2": 0.520083, "cos_b2a/round2": 0.097693, "cos_b2c/round2": 0.185442, "rms_a/round2": 39.218281, "rms_b/round2": 77.934586, "rms_c/round2": 19.148054}, "alpha_mean": 0.43477349582875036, "alpha_hist": [0.01106, 0.08455, 0.19125, 0.19147, 0.1581, 0.14003, 0.11685, 0.08148, 0.0246, 0.00061], "alpha_by_type": {"special": {"share": 0.00113, "mean": 0.60196, "std": 0.09473}, "whitespace": {"share": 0.02637, "mean": 0.56298, "std": 0.12423}, "digit": {"share": 0.03052, "mean": 0.46662, "std": 0.10035}, "word_start": {"share": 0.59634, "mean": 0.4448, "std": 0.18473}, "word_cont": {"share": 0.24443, "mean": 0.31836, "std": 0.13241}, "punct": {"share": 0.10121, "mean": 0.61195, "std": 0.16868}}, "alpha_p05": 0.15534, "alpha_p50": 0.42776, "alpha_p95": 0.7773}
{"step": 56310, "total_steps": 72479, "tokens": 29522657280, "loss": 3.87270125746727, "l_mix": 2.395803213119507, "l_a": 2.486871510744095, "l_c": 2.4361216723918915, "z_loss": 42.23641872406006, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.28496041893959045, "tokens_per_second": 1463327.9555781186, "eta_seconds": 5793.105120205879, "data_wait_seconds": 2.5480985641479492e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 56320, "total_steps": 72479, "tokens": 29527900160, "loss": 4.654128730297089, "l_mix": 2.8881252110004425, "l_a": 2.9639109075069427, "l_c": 2.9227670431137085, "z_loss": 39.20790910720825, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.21874774992465973, "tokens_per_second": 1178571.6899526352, "eta_seconds": 7188.336411118507, "data_wait_seconds": 2.8818845748901367e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 56330, "total_steps": 72479, "tokens": 29533143040, "loss": 4.7107914686203, "l_mix": 2.9218639135360718, "l_a": 3.0018061101436615, "l_c": 2.961285263299942, "z_loss": 39.99158191680908, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18542002141475677, "tokens_per_second": 1382901.9009398664, "eta_seconds": 6122.434936451912, "data_wait_seconds": 2.148747444152832e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 56340, "total_steps": 72479, "tokens": 29538385920, "loss": 4.639753639698029, "l_mix": 2.8783960342407227, "l_a": 2.9544527530670166, "l_c": 2.9167385697364807, "z_loss": 38.94302272796631, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20219393074512482, "tokens_per_second": 1435015.5328197437, "eta_seconds": 5896.440727281571, "data_wait_seconds": 1.6927719116210938e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 56350, "total_steps": 72479, "tokens": 29543628800, "loss": 4.675265312194824, "l_mix": 2.900994747877121, "l_a": 2.976629912853241, "l_c": 2.937605172395706, "z_loss": 39.84497785568237, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12254912406206131, "tokens_per_second": 1371377.30447073, "eta_seconds": 6166.239680671692, "data_wait_seconds": 3.224611282348633e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 56360, "total_steps": 72479, "tokens": 29548871680, "loss": 4.627179563045502, "l_mix": 2.8702125549316406, "l_a": 2.94871985912323, "l_c": 2.9078369438648224, "z_loss": 39.09896469116211, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15803909301757812, "tokens_per_second": 1311001.4666797507, "eta_seconds": 6446.215726518631, "data_wait_seconds": 1.7464160919189453e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 56370, "total_steps": 72479, "tokens": 29554114560, "loss": 2.8131945729255676, "l_mix": 1.7422369718551636, "l_a": 1.803725242614746, "l_c": 1.7661333084106445, "z_loss": 48.528531551361084, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.22732147574424744, "tokens_per_second": 1351907.0667179062, "eta_seconds": 6247.289920973778, "data_wait_seconds": 1.65402889251709e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 56380, "total_steps": 72479, "tokens": 29559357440, "loss": 3.8691030740737915, "l_mix": 2.3954456448554993, "l_a": 2.481855869293213, "l_c": 2.430335223674774, "z_loss": 41.4016809463501, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.29361191391944885, "tokens_per_second": 1447277.5053543437, "eta_seconds": 5831.993160104751, "data_wait_seconds": 1.7911195755004883e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 56390, "total_steps": 72479, "tokens": 29564600320, "loss": 3.9367348849773407, "l_mix": 2.436250686645508, "l_a": 2.5287085473537445, "l_c": 2.472905158996582, "z_loss": 42.47201347351074, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14864975214004517, "tokens_per_second": 1428474.3409369069, "eta_seconds": 5905.090060257911, "data_wait_seconds": 1.9729137420654297e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 56400, "total_steps": 72479, "tokens": 29569843200, "loss": 3.99139404296875, "l_mix": 2.4710049629211426, "l_a": 2.5632034838199615, "l_c": 2.504759669303894, "z_loss": 42.636226177215576, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1896296590566635, "tokens_per_second": 1341847.5511034704, "eta_seconds": 6282.402755117416, "data_wait_seconds": 1.5735626220703125e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16795848, "gate_a2b/L1_absmax": 0.6982885, "gate_a2b/L1_rms": 0.17861927, "gate_a2b/L3_absmean": 0.1612422, "gate_a2b/L3_absmax": 0.52676034, "gate_a2b/L3_rms": 0.16716947, "gate_a2b/L5_absmean": 0.23866844, "gate_a2b/L5_absmax": 0.73501152, "gate_a2b/L5_rms": 0.24471827, "gate_c2b/L1_absmean": 0.02975003, "gate_c2b/L1_absmax": 0.63101465, "gate_c2b/L1_rms": 0.06770363, "gate_c2b/L3_absmean": 0.17460865, "gate_c2b/L3_absmax": 0.39232132, "gate_c2b/L3_rms": 0.17822234, "gate_c2b/L5_absmean": 0.49459442, "gate_c2b/L5_absmax": 0.69381344, "gate_c2b/L5_rms": 0.50446987, "gate_b2a/L7_absmean": 0.05062998, "gate_b2a/L7_absmax": 0.33526587, "gate_b2a/L7_rms": 0.05656107, "gate_b2a/L11_absmean": 0.08221063, "gate_b2a/L11_absmax": 0.61275339, "gate_b2a/L11_rms": 0.09148201, "gate_b2a/L14_absmean": 0.21253592, "gate_b2a/L14_absmax": 0.81611782, "gate_b2a/L14_rms": 0.22176093, "gate_b2c/L3_absmean": 0.03505666, "gate_b2c/L3_absmax": 0.55254936, "gate_b2c/L3_rms": 0.04568203, "gate_b2c/L5_absmean": 0.03738064, "gate_b2c/L5_absmax": 0.54884458, "gate_b2c/L5_rms": 0.04710025, "gate_b2c/L6_absmean": 0.05355919, "gate_b2c/L6_absmax": 0.34985647, "gate_b2c/L6_rms": 0.05981167}, "streams": {"cos_a2b/round0": 0.356672, "cos_c2b/round0": 0.27383, "cos_b2a/round0": 0.161769, "cos_b2c/round0": 0.141919, "rms_a/round0": 7.145926, "rms_b/round0": 12.286235, "rms_c/round0": 4.701458, "cos_a2b/round1": 0.41996, "cos_c2b/round1": 0.297104, "cos_b2a/round1": 0.251327, "cos_b2c/round1": 0.113228, "rms_a/round1": 14.157519, "rms_b/round1": 29.212704, "rms_c/round1": 9.91296, "cos_a2b/round2": 0.527285, "cos_c2b/round2": 0.529511, "cos_b2a/round2": 0.097017, "cos_b2c/round2": 0.180869, "rms_a/round2": 39.524845, "rms_b/round2": 79.905525, "rms_c/round2": 19.511278}, "alpha_mean": 0.41922072637586894, "alpha_hist": [0.01028, 0.09946, 0.20531, 0.19485, 0.15791, 0.13655, 0.1073, 0.06924, 0.01868, 0.00041], "alpha_by_type": {"special": {"share": 0.00093, "mean": 0.57979, "std": 0.10729}, "whitespace": {"share": 0.02568, "mean": 0.53959, "std": 0.11481}, "digit": {"share": 0.02394, "mean": 0.43658, "std": 0.11344}, "word_start": {"share": 0.60487, "mean": 0.43285, "std": 0.18133}, "word_cont": {"share": 0.25182, "mean": 0.30614, "std": 0.12476}, "punct": {"share": 0.09276, "mean": 0.59793, "std": 0.17168}}, "alpha_p05": 0.15688, "alpha_p50": 0.37938, "alpha_p95": 0.76454}
{"step": 56410, "total_steps": 72479, "tokens": 29575086080, "loss": 3.9789203703403473, "l_mix": 2.4633156657218933, "l_a": 2.553475856781006, "l_c": 2.4985393583774567, "z_loss": 41.79691553115845, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11791574209928513, "tokens_per_second": 1454698.1460845484, "eta_seconds": 5791.430954027175, "data_wait_seconds": 1.621246337890625e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 56420, "total_steps": 72479, "tokens": 29580328960, "loss": 4.062383711338043, "l_mix": 2.5164034366607666, "l_a": 2.602282553911209, "l_c": 2.5509851276874542, "z_loss": 42.46042776107788, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13987822830677032, "tokens_per_second": 1350026.7826470044, "eta_seconds": 6236.573303747177, "data_wait_seconds": 1.919269561767578e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 56430, "total_steps": 72479, "tokens": 29585571840, "loss": 2.912146210670471, "l_mix": 1.8032905459403992, "l_a": 1.8668094873428345, "l_c": 1.8293759375810623, "z_loss": 51.94932460784912, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17982864379882812, "tokens_per_second": 1426385.752867498, "eta_seconds": 5899.0340411663055, "data_wait_seconds": 3.585219383239746e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 56440, "total_steps": 72479, "tokens": 29590814720, "loss": 4.698482155799866, "l_mix": 2.9153289198875427, "l_a": 2.990322232246399, "l_c": 2.9535219073295593, "z_loss": 39.78501033782959, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12602932751178741, "tokens_per_second": 1408419.6830762583, "eta_seconds": 5970.560716414452, "data_wait_seconds": 2.059340476989746e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 56450, "total_steps": 72479, "tokens": 29596057600, "loss": 4.6575945019721985, "l_mix": 2.889934539794922, "l_a": 2.966239184141159, "l_c": 2.9259607791900635, "z_loss": 39.78886032104492, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1932039111852646, "tokens_per_second": 1405961.543523576, "eta_seconds": 5977.270424437523, "data_wait_seconds": 2.390146255493164e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 56460, "total_steps": 72479, "tokens": 29601300480, "loss": 4.653599917888641, "l_mix": 2.886473149061203, "l_a": 2.963608741760254, "l_c": 2.92681348323822, "z_loss": 40.618836879730225, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15909981727600098, "tokens_per_second": 1408290.790818323, "eta_seconds": 5963.661430406571, "data_wait_seconds": 2.9742717742919922e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 56470, "total_steps": 72479, "tokens": 29606543360, "loss": 4.607165098190308, "l_mix": 2.8580856919288635, "l_a": 2.9355336129665375, "l_c": 2.894730657339096, "z_loss": 40.19758081436157, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11818751692771912, "tokens_per_second": 1234547.5498740147, "eta_seconds": 6798.7066134929655, "data_wait_seconds": 4.690885543823242e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 56480, "total_steps": 72479, "tokens": 29611786240, "loss": 4.635433912277222, "l_mix": 2.8758209943771362, "l_a": 2.9529568552970886, "l_c": 2.9124194383621216, "z_loss": 39.741751194000244, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11764664947986603, "tokens_per_second": 1395280.1044691447, "eta_seconds": 6011.75612347126, "data_wait_seconds": 2.512335777282715e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 56490, "total_steps": 72479, "tokens": 29617029120, "loss": 4.073464006185532, "l_mix": 2.522320508956909, "l_a": 2.6126769483089447, "l_c": 2.5578008592128754, "z_loss": 42.51576089859009, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12256461381912231, "tokens_per_second": 1435893.8959104817, "eta_seconds": 5838.0642580032345, "data_wait_seconds": 1.9788742065429688e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 56500, "total_steps": 72479, "tokens": 29622272000, "loss": 4.098358869552612, "l_mix": 2.537826806306839, "l_a": 2.627790093421936, "l_c": 2.5739831030368805, "z_loss": 42.512739181518555, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13012222945690155, "tokens_per_second": 1372780.5501164792, "eta_seconds": 6102.649073290825, "data_wait_seconds": 2.0384788513183594e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16789424, "gate_a2b/L1_absmax": 0.69625771, "gate_a2b/L1_rms": 0.1786007, "gate_a2b/L3_absmean": 0.16143054, "gate_a2b/L3_absmax": 0.52559406, "gate_a2b/L3_rms": 0.16729654, "gate_a2b/L5_absmean": 0.23896471, "gate_a2b/L5_absmax": 0.73407191, "gate_a2b/L5_rms": 0.2449847, "gate_c2b/L1_absmean": 0.0295461, "gate_c2b/L1_absmax": 0.62950301, "gate_c2b/L1_rms": 0.06736478, "gate_c2b/L3_absmean": 0.17443709, "gate_c2b/L3_absmax": 0.39434651, "gate_c2b/L3_rms": 0.17811428, "gate_c2b/L5_absmean": 0.49362585, "gate_c2b/L5_absmax": 0.70230228, "gate_c2b/L5_rms": 0.50354904, "gate_b2a/L7_absmean": 0.05089193, "gate_b2a/L7_absmax": 0.33443302, "gate_b2a/L7_rms": 0.05676742, "gate_b2a/L11_absmean": 0.08213971, "gate_b2a/L11_absmax": 0.6103397, "gate_b2a/L11_rms": 0.09136856, "gate_b2a/L14_absmean": 0.21253294, "gate_b2a/L14_absmax": 0.81456268, "gate_b2a/L14_rms": 0.22172585, "gate_b2c/L3_absmean": 0.03501924, "gate_b2c/L3_absmax": 0.55215627, "gate_b2c/L3_rms": 0.0457464, "gate_b2c/L5_absmean": 0.03733391, "gate_b2c/L5_absmax": 0.54725933, "gate_b2c/L5_rms": 0.04697477, "gate_b2c/L6_absmean": 0.05360718, "gate_b2c/L6_absmax": 0.34951663, "gate_b2c/L6_rms": 0.05976063}, "streams": {"cos_a2b/round0": 0.357468, "cos_c2b/round0": 0.268125, "cos_b2a/round0": 0.161496, "cos_b2c/round0": 0.143859, "rms_a/round0": 7.092915, "rms_b/round0": 12.303589, "rms_c/round0": 4.690921, "cos_a2b/round1": 0.419675, "cos_c2b/round1": 0.305993, "cos_b2a/round1": 0.250306, "cos_b2c/round1": 0.115182, "rms_a/round1": 14.080319, "rms_b/round1": 28.739748, "rms_c/round1": 9.846049, "cos_a2b/round2": 0.540569, "cos_c2b/round2": 0.520262, "cos_b2a/round2": 0.10284, "cos_b2c/round2": 0.182583, "rms_a/round2": 39.385868, "rms_b/round2": 77.895416, "rms_c/round2": 19.213299}, "alpha_mean": 0.44147703055229215, "alpha_hist": [0.00885, 0.0777, 0.17583, 0.19368, 0.16615, 0.15189, 0.12286, 0.08205, 0.02069, 0.00031], "alpha_by_type": {"special": {"share": 0.00102, "mean": 0.55571, "std": 0.11301}, "whitespace": {"share": 0.02724, "mean": 0.54654, "std": 0.11511}, "digit": {"share": 0.02148, "mean": 0.46981, "std": 0.12497}, "word_start": {"share": 0.60904, "mean": 0.45198, "std": 0.17923}, "word_cont": {"share": 0.23912, "mean": 0.329, "std": 0.13237}, "punct": {"share": 0.1021, "mean": 0.60711, "std": 0.16909}}, "alpha_p05": 0.16, "alpha_p50": 0.41585, "alpha_p95": 0.73716}
{"step": 56510, "total_steps": 72479, "tokens": 29627514880, "loss": 4.183434545993805, "l_mix": 2.5910930037498474, "l_a": 2.6797671020030975, "l_c": 2.6280376613140106, "z_loss": 42.39399480819702, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12284816801548004, "tokens_per_second": 1392209.4678561923, "eta_seconds": 6013.717953586579, "data_wait_seconds": 2.0235776901245117e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 56520, "total_steps": 72479, "tokens": 29632757760, "loss": 4.1505798399448395, "l_mix": 2.570515811443329, "l_a": 2.658928543329239, "l_c": 2.60795134305954, "z_loss": 41.95326089859009, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1651948094367981, "tokens_per_second": 1441802.2610105255, "eta_seconds": 5803.2314265728, "data_wait_seconds": 2.1219253540039062e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 56530, "total_steps": 72479, "tokens": 29638000640, "loss": 4.15963152050972, "l_mix": 2.576270043849945, "l_a": 2.6660071909427643, "l_c": 2.611864358186722, "z_loss": 42.19951581954956, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15686467289924622, "tokens_per_second": 1433458.7877147559, "eta_seconds": 5833.3517389297485, "data_wait_seconds": 2.0444393157958984e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 56540, "total_steps": 72479, "tokens": 29643243520, "loss": 3.1021317839622498, "l_mix": 1.9197089821100235, "l_a": 1.9945231527090073, "l_c": 1.946886032819748, "z_loss": 48.46492958068848, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2148282825946808, "tokens_per_second": 1442376.5861478345, "eta_seconds": 5793.650917696953, "data_wait_seconds": 2.8401613235473633e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 56550, "total_steps": 72479, "tokens": 29648486400, "loss": 2.9770551323890686, "l_mix": 1.8434512615203857, "l_a": 1.9092646390199661, "l_c": 1.8694148659706116, "z_loss": 50.86275100708008, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.24036462604999542, "tokens_per_second": 1422041.432088572, "eta_seconds": 5872.813100624085, "data_wait_seconds": 2.1129846572875977e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 56560, "total_steps": 72479, "tokens": 29653729280, "loss": 3.12050661444664, "l_mix": 1.9219819456338882, "l_a": 2.027608186006546, "l_c": 1.967473953962326, "z_loss": 40.39707326889038, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.34702393412590027, "tokens_per_second": 1274198.6572046683, "eta_seconds": 6550.109454917908, "data_wait_seconds": 2.950429916381836e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 56570, "total_steps": 72479, "tokens": 29658972160, "loss": 4.344055682420731, "l_mix": 2.6920410096645355, "l_a": 2.776154726743698, "l_c": 2.730560451745987, "z_loss": 40.47867679595947, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19698022305965424, "tokens_per_second": 1453090.0737499841, "eta_seconds": 5740.110639166832, "data_wait_seconds": 2.4825334548950195e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 56580, "total_steps": 72479, "tokens": 29664215040, "loss": 4.329732775688171, "l_mix": 2.6836964190006256, "l_a": 2.76885324716568, "l_c": 2.717934489250183, "z_loss": 40.359482765197754, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1853865683078766, "tokens_per_second": 1388480.7047118563, "eta_seconds": 6003.435902070999, "data_wait_seconds": 3.376603126525879e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 56590, "total_steps": 72479, "tokens": 29669457920, "loss": 4.505843639373779, "l_mix": 2.7927367985248566, "l_a": 2.8818496465682983, "l_c": 2.8285062313079834, "z_loss": 40.30202674865723, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1314387172460556, "tokens_per_second": 1276722.4978953751, "eta_seconds": 6524.8415734291075, "data_wait_seconds": 2.0712614059448242e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 56600, "total_steps": 72479, "tokens": 29674700800, "loss": 4.278454303741455, "l_mix": 2.6509619057178497, "l_a": 2.7379395961761475, "l_c": 2.687034994363785, "z_loss": 41.29244804382324, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12844187021255493, "tokens_per_second": 1440717.7486569658, "eta_seconds": 5778.487257313728, "data_wait_seconds": 1.6808509826660156e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.16778122, "gate_a2b/L1_absmax": 0.6957413, "gate_a2b/L1_rms": 0.1783566, "gate_a2b/L3_absmean": 0.1609941, "gate_a2b/L3_absmax": 0.52522784, "gate_a2b/L3_rms": 0.16688862, "gate_a2b/L5_absmean": 0.23829852, "gate_a2b/L5_absmax": 0.73430252, "gate_a2b/L5_rms": 0.2442452, "gate_c2b/L1_absmean": 0.03004285, "gate_c2b/L1_absmax": 0.62976414, "gate_c2b/L1_rms": 0.06765042, "gate_c2b/L3_absmean": 0.17360519, "gate_c2b/L3_absmax": 0.39062363, "gate_c2b/L3_rms": 0.17726822, "gate_c2b/L5_absmean": 0.49326134, "gate_c2b/L5_absmax": 0.69512033, "gate_c2b/L5_rms": 0.50319272, "gate_b2a/L7_absmean": 0.05082762, "gate_b2a/L7_absmax": 0.33400077, "gate_b2a/L7_rms": 0.05669976, "gate_b2a/L11_absmean": 0.08201258, "gate_b2a/L11_absmax": 0.61087716, "gate_b2a/L11_rms": 0.09128253, "gate_b2a/L14_absmean": 0.21263149, "gate_b2a/L14_absmax": 0.81526959, "gate_b2a/L14_rms": 0.22178686, "gate_b2c/L3_absmean": 0.03515732, "gate_b2c/L3_absmax": 0.55173689, "gate_b2c/L3_rms": 0.04576816, "gate_b2c/L5_absmean": 0.03741426, "gate_b2c/L5_absmax": 0.54711062, "gate_b2c/L5_rms": 0.04701431, "gate_b2c/L6_absmean": 0.05361122, "gate_b2c/L6_absmax": 0.34961617, "gate_b2c/L6_rms": 0.05979793}, "streams": {"cos_a2b/round0": 0.358034, "cos_c2b/round0": 0.26392, "cos_b2a/round0": 0.157027, "cos_b2c/round0": 0.141125, "rms_a/round0": 7.115186, "rms_b/round0": 12.432489, "rms_c/round0": 4.706685, "cos_a2b/round1": 0.414511, "cos_c2b/round1": 0.299556, "cos_b2a/round1": 0.250759, "cos_b2c/round1": 0.111823, "rms_a/round1": 13.989584, "rms_b/round1": 28.928453, "rms_c/round1": 9.90286, "cos_a2b/round2": 0.52604, "cos_c2b/round2": 0.529694, "cos_b2a/round2": 0.096959, "cos_b2c/round2": 0.179807, "rms_a/round2": 39.220161, "rms_b/round2": 78.479454, "rms_c/round2": 19.399992}, "alpha_mean": 0.41883698490625676, "alpha_hist": [0.01317, 0.08495, 0.19659, 0.20605, 0.17261, 0.14389, 0.10875, 0.06281, 0.01108, 0.00011], "alpha_by_type": {"special": {"share": 0.00092, "mean": 0.56117, "std": 0.10264}, "whitespace": {"share": 0.03302, "mean": 0.50705, "std": 0.12387}, "digit": {"share": 0.03217, "mean": 0.44325, "std": 0.11386}, "word_start": {"share": 0.55101, "mean": 0.43665, "std": 0.17333}, "word_cont": {"share": 0.28342, "mean": 0.31337, "std": 0.12742}, "punct": {"share": 0.09947, "mean": 0.58219, "std": 0.16549}}, "alpha_p05": 0.16831, "alpha_p50": 0.39607, "alpha_p95": 0.71721}
{"step": 56610, "total_steps": 72479, "tokens": 29679943680, "loss": 4.318475902080536, "l_mix": 2.6756403148174286, "l_a": 2.76303893327713, "l_c": 2.7130793631076813, "z_loss": 41.33685922622681, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12888790667057037, "tokens_per_second": 1326321.5686931105, "eta_seconds": 6272.932951092721, "data_wait_seconds": 2.7298927307128906e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 56620, "total_steps": 72479, "tokens": 29685186560, "loss": 4.2320475578308105, "l_mix": 2.6221249401569366, "l_a": 2.7101563215255737, "l_c": 2.6562524139881134, "z_loss": 41.16718053817749, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14942754805088043, "tokens_per_second": 1466103.4699095278, "eta_seconds": 5671.280071735382, "data_wait_seconds": 2.041459083557129e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 56630, "total_steps": 72479, "tokens": 29690429440, "loss": 4.233749210834503, "l_mix": 2.6223217248916626, "l_a": 2.7107365429401398, "l_c": 2.660688132047653, "z_loss": 42.19350528717041, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1276640146970749, "tokens_per_second": 1351643.5698600563, "eta_seconds": 6147.656599187851, "data_wait_seconds": 1.9878149032592773e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 56640, "total_steps": 72479, "tokens": 29695672320, "loss": 4.3264080286026, "l_mix": 2.6819155514240265, "l_a": 2.7658225893974304, "l_c": 2.715818613767624, "z_loss": 40.037349224090576, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11480972170829773, "tokens_per_second": 1364086.757948063, "eta_seconds": 6087.734217500686, "data_wait_seconds": 1.728534698486328e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 56650, "total_steps": 72479, "tokens": 29700915200, "loss": 2.9388965368270874, "l_mix": 1.8194941878318787, "l_a": 1.8857428133487701, "l_c": 1.8455981761217117, "z_loss": 48.800617694854736, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1924254149198532, "tokens_per_second": 1445974.5851461252, "eta_seconds": 5739.350357365608, "data_wait_seconds": 2.1010637283325195e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 56660, "total_steps": 72479, "tokens": 29706158080, "loss": 2.9026599526405334, "l_mix": 1.7976430356502533, "l_a": 1.860270917415619, "l_c": 1.823118582367897, "z_loss": 51.17042875289917, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16649475693702698, "tokens_per_second": 1374882.8598761593, "eta_seconds": 6032.304361367225, "data_wait_seconds": 2.8014183044433594e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 56670, "total_steps": 72479, "tokens": 29711400960, "loss": 3.1017459332942963, "l_mix": 1.9078554213047028, "l_a": 2.024059548974037, "l_c": 1.9555751830339432, "z_loss": 46.01774311065674, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.21765489876270294, "tokens_per_second": 1332402.7751929464, "eta_seconds": 6220.693281579018, "data_wait_seconds": 2.3871660232543945e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 56680, "total_steps": 72479, "tokens": 29716643840, "loss": 4.379696190357208, "l_mix": 2.714775949716568, "l_a": 2.7992835342884064, "l_c": 2.7504506409168243, "z_loss": 40.420024394989014, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11963297426700592, "tokens_per_second": 1450007.9954349298, "eta_seconds": 5712.538232946396, "data_wait_seconds": 2.0712614059448242e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 56690, "total_steps": 72479, "tokens": 29721886720, "loss": 4.31677383184433, "l_mix": 2.674735814332962, "l_a": 2.760435402393341, "l_c": 2.7130246758461, "z_loss": 41.19669485092163, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19467562437057495, "tokens_per_second": 1440081.743618652, "eta_seconds": 5748.273157882691, "data_wait_seconds": 1.996755599975586e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 56700, "total_steps": 72479, "tokens": 29727129600, "loss": 4.324501156806946, "l_mix": 2.6804608404636383, "l_a": 2.7690760493278503, "l_c": 2.711058020591736, "z_loss": 41.04947757720947, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1856182962656021, "tokens_per_second": 1346326.1239742024, "eta_seconds": 6144.677879071236, "data_wait_seconds": 1.6570091247558594e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.16749425, "gate_a2b/L1_absmax": 0.69530404, "gate_a2b/L1_rms": 0.1779024, "gate_a2b/L3_absmean": 0.16046564, "gate_a2b/L3_absmax": 0.52571821, "gate_a2b/L3_rms": 0.16637337, "gate_a2b/L5_absmean": 0.23792274, "gate_a2b/L5_absmax": 0.73518074, "gate_a2b/L5_rms": 0.2439236, "gate_c2b/L1_absmean": 0.0298811, "gate_c2b/L1_absmax": 0.62867236, "gate_c2b/L1_rms": 0.06748035, "gate_c2b/L3_absmean": 0.17356233, "gate_c2b/L3_absmax": 0.39383575, "gate_c2b/L3_rms": 0.17729706, "gate_c2b/L5_absmean": 0.49297142, "gate_c2b/L5_absmax": 0.70230383, "gate_c2b/L5_rms": 0.50286961, "gate_b2a/L7_absmean": 0.05066871, "gate_b2a/L7_absmax": 0.33276752, "gate_b2a/L7_rms": 0.05653513, "gate_b2a/L11_absmean": 0.08192839, "gate_b2a/L11_absmax": 0.60927993, "gate_b2a/L11_rms": 0.09117535, "gate_b2a/L14_absmean": 0.21258837, "gate_b2a/L14_absmax": 0.81537032, "gate_b2a/L14_rms": 0.2217634, "gate_b2c/L3_absmean": 0.03480178, "gate_b2c/L3_absmax": 0.55226672, "gate_b2c/L3_rms": 0.04553893, "gate_b2c/L5_absmean": 0.03710977, "gate_b2c/L5_absmax": 0.54875964, "gate_b2c/L5_rms": 0.04685796, "gate_b2c/L6_absmean": 0.05377503, "gate_b2c/L6_absmax": 0.34915632, "gate_b2c/L6_rms": 0.05992589}, "streams": {"cos_a2b/round0": 0.357757, "cos_c2b/round0": 0.267721, "cos_b2a/round0": 0.160989, "cos_b2c/round0": 0.143836, "rms_a/round0": 7.11422, "rms_b/round0": 12.424251, "rms_c/round0": 4.72055, "cos_a2b/round1": 0.416871, "cos_c2b/round1": 0.307379, "cos_b2a/round1": 0.255045, "cos_b2c/round1": 0.111046, "rms_a/round1": 13.993834, "rms_b/round1": 28.813683, "rms_c/round1": 10.001384, "cos_a2b/round2": 0.539114, "cos_c2b/round2": 0.528074, "cos_b2a/round2": 0.100596, "cos_b2c/round2": 0.182405, "rms_a/round2": 39.41349, "rms_b/round2": 77.89006, "rms_c/round2": 19.415222}, "alpha_mean": 0.4349297743203806, "alpha_hist": [0.00394, 0.05951, 0.19051, 0.20821, 0.17851, 0.16272, 0.12746, 0.06158, 0.00749, 8e-05], "alpha_by_type": {"special": {"share": 0.00108, "mean": 0.55544, "std": 0.10127}, "whitespace": {"share": 0.02705, "mean": 0.53508, "std": 0.11093}, "digit": {"share": 0.0276, "mean": 0.44745, "std": 0.11337}, "word_start": {"share": 0.56783, "mean": 0.45328, "std": 0.16372}, "word_cont": {"share": 0.27025, "mean": 0.3279, "std": 0.11852}, "punct": {"share": 0.10619, "mean": 0.57922, "std": 0.1542}}, "alpha_p05": 0.19042, "alpha_p50": 0.4168, "alpha_p95": 0.72681}
{"step": 56710, "total_steps": 72479, "tokens": 29732372480, "loss": 4.328956842422485, "l_mix": 2.6833873987197876, "l_a": 2.767503470182419, "l_c": 2.71772763133049, "z_loss": 41.38304376602173, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14238479733467102, "tokens_per_second": 1367389.695942757, "eta_seconds": 6046.1896828174595, "data_wait_seconds": 1.665949821472168e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 56720, "total_steps": 72479, "tokens": 29737615360, "loss": 4.658026397228241, "l_mix": 2.8899628818035126, "l_a": 2.9669454991817474, "l_c": 2.9265991151332855, "z_loss": 40.03431987762451, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12276631593704224, "tokens_per_second": 1423334.908684549, "eta_seconds": 5804.856286168098, "data_wait_seconds": 1.7523765563964844e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 56730, "total_steps": 72479, "tokens": 29742858240, "loss": 4.462014436721802, "l_mix": 2.7677426040172577, "l_a": 2.8449077904224396, "l_c": 2.8026649057865143, "z_loss": 39.93634033203125, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12298734486103058, "tokens_per_second": 1458170.4494718118, "eta_seconds": 5662.583352303504, "data_wait_seconds": 1.901388168334961e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 56740, "total_steps": 72479, "tokens": 29748101120, "loss": 4.573025941848755, "l_mix": 2.8367303609848022, "l_a": 2.9154621064662933, "l_c": 2.872189462184906, "z_loss": 40.37831258773804, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13087229430675507, "tokens_per_second": 1465646.1608687944, "eta_seconds": 5630.1234583854675, "data_wait_seconds": 2.810359001159668e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 56750, "total_steps": 72479, "tokens": 29753344000, "loss": 4.616468906402588, "l_mix": 2.863939106464386, "l_a": 2.9421775937080383, "l_c": 2.8995883464813232, "z_loss": 40.03091526031494, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13109630346298218, "tokens_per_second": 1394812.2890809553, "eta_seconds": 5912.28369333744, "data_wait_seconds": 2.759695053100586e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 56760, "total_steps": 72479, "tokens": 29758586880, "loss": 4.6824159026145935, "l_mix": 2.9050684571266174, "l_a": 2.983311504125595, "l_c": 2.9411797523498535, "z_loss": 40.29530096054077, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1648024022579193, "tokens_per_second": 1454608.7527837537, "eta_seconds": 5665.635557484627, "data_wait_seconds": 3.808736801147461e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 56770, "total_steps": 72479, "tokens": 29763829760, "loss": 2.970142513513565, "l_mix": 1.8398786783218384, "l_a": 1.8916317969560623, "l_c": 1.875914067029953, "z_loss": 50.01416873931885, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.4274258613586426, "tokens_per_second": 1446585.7356129722, "eta_seconds": 5693.433848571777, "data_wait_seconds": 2.3484230041503906e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 56780, "total_steps": 72479, "tokens": 29769072640, "loss": 3.089007079601288, "l_mix": 1.902020886540413, "l_a": 2.0107810348272324, "l_c": 1.9458393156528473, "z_loss": 43.298182010650635, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17912013828754425, "tokens_per_second": 1385505.270315583, "eta_seconds": 5940.646700048447, "data_wait_seconds": 2.5153160095214844e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 56790, "total_steps": 72479, "tokens": 29774315520, "loss": 4.275088042020798, "l_mix": 2.6490370631217957, "l_a": 2.73376601934433, "l_c": 2.6864035725593567, "z_loss": 42.37021493911743, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12345781177282333, "tokens_per_second": 1369634.4208173759, "eta_seconds": 6005.656916165352, "data_wait_seconds": 1.8298625946044922e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 56800, "total_steps": 72479, "tokens": 29779558400, "loss": 4.3696741461753845, "l_mix": 2.7081838250160217, "l_a": 2.7927703857421875, "l_c": 2.7455302476882935, "z_loss": 41.42244815826416, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13383817672729492, "tokens_per_second": 1411772.8382991494, "eta_seconds": 5822.687141299248, "data_wait_seconds": 2.181529998779297e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16835885, "gate_a2b/L1_absmax": 0.69649881, "gate_a2b/L1_rms": 0.17897357, "gate_a2b/L3_absmean": 0.16046523, "gate_a2b/L3_absmax": 0.51959127, "gate_a2b/L3_rms": 0.16638929, "gate_a2b/L5_absmean": 0.23692933, "gate_a2b/L5_absmax": 0.73585582, "gate_a2b/L5_rms": 0.24304883, "gate_c2b/L1_absmean": 0.03059368, "gate_c2b/L1_absmax": 0.63213164, "gate_c2b/L1_rms": 0.06799059, "gate_c2b/L3_absmean": 0.17370288, "gate_c2b/L3_absmax": 0.39360142, "gate_c2b/L3_rms": 0.17743824, "gate_c2b/L5_absmean": 0.49215984, "gate_c2b/L5_absmax": 0.68990773, "gate_c2b/L5_rms": 0.50194842, "gate_b2a/L7_absmean": 0.05029864, "gate_b2a/L7_absmax": 0.33527321, "gate_b2a/L7_rms": 0.05639609, "gate_b2a/L11_absmean": 0.08239148, "gate_b2a/L11_absmax": 0.61010832, "gate_b2a/L11_rms": 0.09159213, "gate_b2a/L14_absmean": 0.21301106, "gate_b2a/L14_absmax": 0.81252635, "gate_b2a/L14_rms": 0.22223231, "gate_b2c/L3_absmean": 0.03508534, "gate_b2c/L3_absmax": 0.55019218, "gate_b2c/L3_rms": 0.04576772, "gate_b2c/L5_absmean": 0.03725521, "gate_b2c/L5_absmax": 0.54202133, "gate_b2c/L5_rms": 0.04677239, "gate_b2c/L6_absmean": 0.05392663, "gate_b2c/L6_absmax": 0.34933728, "gate_b2c/L6_rms": 0.05998563}, "streams": {"cos_a2b/round0": 0.360791, "cos_c2b/round0": 0.270445, "cos_b2a/round0": 0.158905, "cos_b2c/round0": 0.146557, "rms_a/round0": 7.215973, "rms_b/round0": 12.601033, "rms_c/round0": 4.67691, "cos_a2b/round1": 0.415332, "cos_c2b/round1": 0.305226, "cos_b2a/round1": 0.249911, "cos_b2c/round1": 0.112529, "rms_a/round1": 14.313573, "rms_b/round1": 28.980608, "rms_c/round1": 10.144336, "cos_a2b/round2": 0.540787, "cos_c2b/round2": 0.52077, "cos_b2a/round2": 0.097119, "cos_b2c/round2": 0.183755, "rms_a/round2": 40.340275, "rms_b/round2": 78.092735, "rms_c/round2": 19.797789}, "alpha_mean": 0.43903122026562613, "alpha_hist": [0.00873, 0.0723, 0.16521, 0.18887, 0.19009, 0.17409, 0.12506, 0.06667, 0.00882, 0.00017], "alpha_by_type": {"special": {"share": 0.00128, "mean": 0.54927, "std": 0.1164}, "whitespace": {"share": 0.02882, "mean": 0.51946, "std": 0.12528}, "digit": {"share": 0.03365, "mean": 0.48395, "std": 0.14902}, "word_start": {"share": 0.59642, "mean": 0.44586, "std": 0.16335}, "word_cont": {"share": 0.22449, "mean": 0.32372, "std": 0.13155}, "punct": {"share": 0.11534, "mean": 0.59372, "std": 0.15094}}, "alpha_p05": 0.16831, "alpha_p50": 0.44228, "alpha_p95": 0.73297}
{"step": 56810, "total_steps": 72479, "tokens": 29784801280, "loss": 4.40974360704422, "l_mix": 2.733796089887619, "l_a": 2.81892666220665, "l_c": 2.767564982175827, "z_loss": 40.45079231262207, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13628588616847992, "tokens_per_second": 1411760.6932050474, "eta_seconds": 5819.023515486718, "data_wait_seconds": 2.9593706130981445e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 56820, "total_steps": 72479, "tokens": 29790044160, "loss": 4.3178223967552185, "l_mix": 2.6757538616657257, "l_a": 2.7596468925476074, "l_c": 2.7139147222042084, "z_loss": 42.26974868774414, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1628870815038681, "tokens_per_second": 1275962.1346440886, "eta_seconds": 6434.223688220978, "data_wait_seconds": 2.47955322265625e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 56830, "total_steps": 72479, "tokens": 29795287040, "loss": 3.143233895301819, "l_mix": 1.934680536389351, "l_a": 2.0453931093215942, "l_c": 1.9831180423498154, "z_loss": 45.51404047012329, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17606276273727417, "tokens_per_second": 1426729.8319707634, "eta_seconds": 5750.621265602112, "data_wait_seconds": 2.1666288375854492e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 56840, "total_steps": 72479, "tokens": 29800529920, "loss": 3.0825347900390625, "l_mix": 1.8979507237672806, "l_a": 2.003417059779167, "l_c": 1.9451962858438492, "z_loss": 44.445666790008545, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20290039479732513, "tokens_per_second": 1419869.82891944, "eta_seconds": 5774.712487721444, "data_wait_seconds": 3.4749507904052734e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 56850, "total_steps": 72479, "tokens": 29805772800, "loss": 4.591213643550873, "l_mix": 2.8476969599723816, "l_a": 2.9274188578128815, "l_c": 2.8843033611774445, "z_loss": 39.547515869140625, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13433925807476044, "tokens_per_second": 1417939.6912921949, "eta_seconds": 5778.875647759438, "data_wait_seconds": 2.3424625396728516e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 56860, "total_steps": 72479, "tokens": 29811015680, "loss": 4.5516393184661865, "l_mix": 2.8231971859931946, "l_a": 2.9010464549064636, "l_c": 2.8604274094104767, "z_loss": 39.68742084503174, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1149982288479805, "tokens_per_second": 1434657.15028563, "eta_seconds": 5707.882381772995, "data_wait_seconds": 2.1338462829589844e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 56870, "total_steps": 72479, "tokens": 29816258560, "loss": 4.714127600193024, "l_mix": 2.9244406819343567, "l_a": 3.002930849790573, "l_c": 2.9626922011375427, "z_loss": 40.09809160232544, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12774740159511566, "tokens_per_second": 1405589.8519096677, "eta_seconds": 5822.190150904655, "data_wait_seconds": 2.041459083557129e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 56880, "total_steps": 72479, "tokens": 29821501440, "loss": 4.701514482498169, "l_mix": 2.9170114994049072, "l_a": 2.9936266243457794, "l_c": 2.954716593027115, "z_loss": 39.26709985733032, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12180669605731964, "tokens_per_second": 1451567.7594776999, "eta_seconds": 5634.162414121628, "data_wait_seconds": 2.530217170715332e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 56890, "total_steps": 72479, "tokens": 29826744320, "loss": 3.089366167783737, "l_mix": 1.9007690250873566, "l_a": 2.013857886195183, "l_c": 1.9481323659420013, "z_loss": 44.478169441223145, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14191728830337524, "tokens_per_second": 1386398.4312818616, "eta_seconds": 5895.221350216865, "data_wait_seconds": 1.919269561767578e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 56900, "total_steps": 72479, "tokens": 29831987200, "loss": 4.333870112895966, "l_mix": 2.685755044221878, "l_a": 2.7722756564617157, "l_c": 2.7214411199092865, "z_loss": 41.54658365249634, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11372295022010803, "tokens_per_second": 1376984.3757145104, "eta_seconds": 5931.717814707757, "data_wait_seconds": 1.5139579772949219e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16738576, "gate_a2b/L1_absmax": 0.6983763, "gate_a2b/L1_rms": 0.17820674, "gate_a2b/L3_absmean": 0.16004813, "gate_a2b/L3_absmax": 0.50623477, "gate_a2b/L3_rms": 0.16600034, "gate_a2b/L5_absmean": 0.23635863, "gate_a2b/L5_absmax": 0.73635954, "gate_a2b/L5_rms": 0.24258761, "gate_c2b/L1_absmean": 0.03165084, "gate_c2b/L1_absmax": 0.63412935, "gate_c2b/L1_rms": 0.06896781, "gate_c2b/L3_absmean": 0.1736697, "gate_c2b/L3_absmax": 0.3876799, "gate_c2b/L3_rms": 0.17739411, "gate_c2b/L5_absmean": 0.49133942, "gate_c2b/L5_absmax": 0.68446046, "gate_c2b/L5_rms": 0.50097334, "gate_b2a/L7_absmean": 0.05064436, "gate_b2a/L7_absmax": 0.34267157, "gate_b2a/L7_rms": 0.05645024, "gate_b2a/L11_absmean": 0.08233349, "gate_b2a/L11_absmax": 0.61392301, "gate_b2a/L11_rms": 0.09161056, "gate_b2a/L14_absmean": 0.21201231, "gate_b2a/L14_absmax": 0.81923461, "gate_b2a/L14_rms": 0.22141711, "gate_b2c/L3_absmean": 0.03495765, "gate_b2c/L3_absmax": 0.55431831, "gate_b2c/L3_rms": 0.04568364, "gate_b2c/L5_absmean": 0.03725266, "gate_b2c/L5_absmax": 0.53834766, "gate_b2c/L5_rms": 0.04668266, "gate_b2c/L6_absmean": 0.05402579, "gate_b2c/L6_absmax": 0.35197252, "gate_b2c/L6_rms": 0.06038897}, "streams": {"cos_a2b/round0": 0.356438, "cos_c2b/round0": 0.263592, "cos_b2a/round0": 0.156818, "cos_b2c/round0": 0.141797, "rms_a/round0": 7.153008, "rms_b/round0": 12.278648, "rms_c/round0": 4.785578, "cos_a2b/round1": 0.411311, "cos_c2b/round1": 0.300506, "cos_b2a/round1": 0.248952, "cos_b2c/round1": 0.11148, "rms_a/round1": 13.95281, "rms_b/round1": 28.991522, "rms_c/round1": 9.752152, "cos_a2b/round2": 0.529471, "cos_c2b/round2": 0.531747, "cos_b2a/round2": 0.091503, "cos_b2c/round2": 0.180167, "rms_a/round2": 39.274818, "rms_b/round2": 78.897186, "rms_c/round2": 18.64456}, "alpha_mean": 0.4158996601285381, "alpha_hist": [0.01303, 0.0872, 0.19847, 0.19812, 0.1763, 0.15334, 0.11371, 0.05333, 0.0065, 0.0], "alpha_by_type": {"special": {"share": 0.00085, "mean": 0.50872, "std": 0.10604}, "whitespace": {"share": 0.02943, "mean": 0.51824, "std": 0.12337}, "digit": {"share": 0.03008, "mean": 0.40294, "std": 0.12027}, "word_start": {"share": 0.56432, "mean": 0.43607, "std": 0.16949}, "word_cont": {"share": 0.26205, "mean": 0.31002, "std": 0.12925}, "punct": {"share": 0.11327, "mean": 0.53654, "std": 0.16656}}, "alpha_p05": 0.16053, "alpha_p50": 0.3977, "alpha_p95": 0.70497}
{"step": 56910, "total_steps": 72479, "tokens": 29837230080, "loss": 4.367249369621277, "l_mix": 2.7071186006069183, "l_a": 2.7922515869140625, "l_c": 2.7415173053741455, "z_loss": 40.947683334350586, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17225675284862518, "tokens_per_second": 1391207.406302476, "eta_seconds": 5867.306222653389, "data_wait_seconds": 1.722574234008789e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 56920, "total_steps": 72479, "tokens": 29842472960, "loss": 4.403707981109619, "l_mix": 2.7295129597187042, "l_a": 2.8132810294628143, "l_c": 2.767368584871292, "z_loss": 40.82733201980591, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13795842230319977, "tokens_per_second": 1375810.9193760804, "eta_seconds": 5929.155581712723, "data_wait_seconds": 1.6808509826660156e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 56930, "total_steps": 72479, "tokens": 29847715840, "loss": 4.313903987407684, "l_mix": 2.67323300242424, "l_a": 2.759178012609482, "l_c": 2.709724873304367, "z_loss": 40.22575855255127, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16024091839790344, "tokens_per_second": 1388001.4042411295, "eta_seconds": 5873.303936934471, "data_wait_seconds": 1.6003847122192383e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 56940, "total_steps": 72479, "tokens": 29852958720, "loss": 3.0737906396389008, "l_mix": 1.8917596638202667, "l_a": 2.000971406698227, "l_c": 1.9391316175460815, "z_loss": 42.564759254455566, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2298034280538559, "tokens_per_second": 1417680.993400278, "eta_seconds": 5746.646297669411, "data_wait_seconds": 1.755356788635254e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 56950, "total_steps": 72479, "tokens": 29858201600, "loss": 3.0798297822475433, "l_mix": 1.8952615410089493, "l_a": 2.0086024701595306, "l_c": 1.9399582892656326, "z_loss": 44.926509380340576, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1799372285604477, "tokens_per_second": 1467886.6677662907, "eta_seconds": 5546.523809218406, "data_wait_seconds": 2.899765968322754e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 56960, "total_steps": 72479, "tokens": 29863444480, "loss": 3.1215460896492004, "l_mix": 1.9216651320457458, "l_a": 2.0308108180761337, "l_c": 1.9687921404838562, "z_loss": 44.831928730010986, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1910901963710785, "tokens_per_second": 1435399.672695782, "eta_seconds": 5668.404157233238, "data_wait_seconds": 2.3066997528076172e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 56970, "total_steps": 72479, "tokens": 29868687360, "loss": 3.1055044531822205, "l_mix": 1.9102893620729446, "l_a": 2.0221217572689056, "l_c": 1.9619284272193909, "z_loss": 43.80264329910278, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18301059305667877, "tokens_per_second": 1435162.476459897, "eta_seconds": 5665.687840485572, "data_wait_seconds": 2.1070241928100586e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 56980, "total_steps": 72479, "tokens": 29873930240, "loss": 3.072191745042801, "l_mix": 1.8905293941497803, "l_a": 2.0042637437582016, "l_c": 1.9346106946468353, "z_loss": 44.46299171447754, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19124284386634827, "tokens_per_second": 1328677.1449295913, "eta_seconds": 6115.812063908577, "data_wait_seconds": 4.1812658309936523e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 56990, "total_steps": 72479, "tokens": 29879173120, "loss": 4.661296188831329, "l_mix": 2.8921563923358917, "l_a": 2.9685357213020325, "l_c": 2.928596794605255, "z_loss": 38.91740942001343, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1479574590921402, "tokens_per_second": 1455240.900077162, "eta_seconds": 5580.311020374298, "data_wait_seconds": 2.086162567138672e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 57000, "total_steps": 72479, "tokens": 29884416000, "loss": 3.0596634447574615, "l_mix": 1.8834088146686554, "l_a": 1.9912213236093521, "l_c": 1.929627314209938, "z_loss": 44.126473903656006, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1657632440328598, "tokens_per_second": 1382837.7225035625, "eta_seconds": 5868.695812916756, "data_wait_seconds": 1.728534698486328e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.16756348, "gate_a2b/L1_absmax": 0.70094115, "gate_a2b/L1_rms": 0.17821616, "gate_a2b/L3_absmean": 0.15994325, "gate_a2b/L3_absmax": 0.50965863, "gate_a2b/L3_rms": 0.16575697, "gate_a2b/L5_absmean": 0.23580466, "gate_a2b/L5_absmax": 0.73312497, "gate_a2b/L5_rms": 0.2418929, "gate_c2b/L1_absmean": 0.03063665, "gate_c2b/L1_absmax": 0.635903, "gate_c2b/L1_rms": 0.06841281, "gate_c2b/L3_absmean": 0.17390893, "gate_c2b/L3_absmax": 0.38879958, "gate_c2b/L3_rms": 0.17760015, "gate_c2b/L5_absmean": 0.49163046, "gate_c2b/L5_absmax": 0.69079387, "gate_c2b/L5_rms": 0.50139135, "gate_b2a/L7_absmean": 0.05088493, "gate_b2a/L7_absmax": 0.34108484, "gate_b2a/L7_rms": 0.05675852, "gate_b2a/L11_absmean": 0.08217453, "gate_b2a/L11_absmax": 0.61404032, "gate_b2a/L11_rms": 0.09141181, "gate_b2a/L14_absmean": 0.21261355, "gate_b2a/L14_absmax": 0.82458061, "gate_b2a/L14_rms": 0.22201039, "gate_b2c/L3_absmean": 0.03487469, "gate_b2c/L3_absmax": 0.55449331, "gate_b2c/L3_rms": 0.04565791, "gate_b2c/L5_absmean": 0.03735357, "gate_b2c/L5_absmax": 0.54086846, "gate_b2c/L5_rms": 0.04679137, "gate_b2c/L6_absmean": 0.05357509, "gate_b2c/L6_absmax": 0.35214838, "gate_b2c/L6_rms": 0.05984487}, "streams": {"cos_a2b/round0": 0.356553, "cos_c2b/round0": 0.229951, "cos_b2a/round0": 0.145822, "cos_b2c/round0": 0.122961, "rms_a/round0": 7.5118, "rms_b/round0": 12.644509, "rms_c/round0": 4.824487, "cos_a2b/round1": 0.41293, "cos_c2b/round1": 0.30719, "cos_b2a/round1": 0.242557, "cos_b2c/round1": 0.12239, "rms_a/round1": 14.867065, "rms_b/round1": 30.416637, "rms_c/round1": 10.596266, "cos_a2b/round2": 0.519018, "cos_c2b/round2": 0.50975, "cos_b2a/round2": 0.091074, "cos_b2c/round2": 0.174122, "rms_a/round2": 40.489902, "rms_b/round2": 80.095886, "rms_c/round2": 20.522087}, "alpha_mean": 0.41391221645785947, "alpha_hist": [0.0271, 0.15997, 0.19626, 0.15034, 0.11496, 0.10989, 0.10992, 0.08862, 0.0403, 0.00262], "alpha_by_type": {"special": {"share": 0.00113, "mean": 0.64529, "std": 0.10796}, "whitespace": {"share": 0.03299, "mean": 0.64401, "std": 0.16552}, "digit": {"share": 0.01125, "mean": 0.41151, "std": 0.16034}, "word_start": {"share": 0.5815, "mean": 0.41656, "std": 0.20665}, "word_cont": {"share": 0.25659, "mean": 0.28026, "std": 0.14016}, "punct": {"share": 0.11655, "mean": 0.62783, "std": 0.179}}, "alpha_p05": 0.12724, "alpha_p50": 0.36387, "alpha_p95": 0.782}
{"step": 57010, "total_steps": 72479, "tokens": 29889658880, "loss": 4.315258204936981, "l_mix": 2.674611121416092, "l_a": 2.7576167583465576, "l_c": 2.7112069129943848, "z_loss": 41.542051792144775, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11581048369407654, "tokens_per_second": 947488.4348828548, "eta_seconds": 8559.694000911712, "data_wait_seconds": 1.7136335372924805e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 57020, "total_steps": 72479, "tokens": 29894901760, "loss": 4.367230415344238, "l_mix": 2.706867218017578, "l_a": 2.7913233041763306, "l_c": 2.74322047829628, "z_loss": 40.94291925430298, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1208905279636383, "tokens_per_second": 1471085.0589591812, "eta_seconds": 5509.517034816742, "data_wait_seconds": 1.6629695892333984e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 57030, "total_steps": 72479, "tokens": 29900144640, "loss": 4.346389710903168, "l_mix": 2.693871349096298, "l_a": 2.780487686395645, "l_c": 2.727906435728073, "z_loss": 40.71843147277832, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11902686208486557, "tokens_per_second": 1195984.653461265, "eta_seconds": 6772.432479429244, "data_wait_seconds": 2.6673078536987305e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 57040, "total_steps": 72479, "tokens": 29905387520, "loss": 4.310661852359772, "l_mix": 2.6714287996292114, "l_a": 2.7567208111286163, "l_c": 2.7073890566825867, "z_loss": 41.39715385437012, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12285879999399185, "tokens_per_second": 1447679.3895768712, "eta_seconds": 5591.3501914024355, "data_wait_seconds": 2.333521842956543e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 57050, "total_steps": 72479, "tokens": 29910630400, "loss": 3.1310137808322906, "l_mix": 1.9271746426820755, "l_a": 2.0371688902378082, "l_c": 1.9756281077861786, "z_loss": 43.62521028518677, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15053614974021912, "tokens_per_second": 1449507.6409812542, "eta_seconds": 5580.6808624506, "data_wait_seconds": 2.154707908630371e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 57060, "total_steps": 72479, "tokens": 29915873280, "loss": 3.124274790287018, "l_mix": 1.9220920503139496, "l_a": 2.0361080169677734, "l_c": 1.9711675196886063, "z_loss": 44.72255802154541, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17523212730884552, "tokens_per_second": 1386321.0803884987, "eta_seconds": 5831.258563661575, "data_wait_seconds": 3.9637088775634766e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 57070, "total_steps": 72479, "tokens": 29921116160, "loss": 3.107885777950287, "l_mix": 1.9122492969036102, "l_a": 2.02566996216774, "l_c": 1.959784910082817, "z_loss": 44.283520221710205, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17830559611320496, "tokens_per_second": 1444904.5808326202, "eta_seconds": 5591.202283644677, "data_wait_seconds": 1.913309097290039e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 57080, "total_steps": 72479, "tokens": 29926359040, "loss": 3.1057140827178955, "l_mix": 1.9117587357759476, "l_a": 2.0216289162635803, "l_c": 1.9582218676805496, "z_loss": 43.49483585357666, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1896795928478241, "tokens_per_second": 1414377.0683933161, "eta_seconds": 5708.174356341362, "data_wait_seconds": 2.0384788513183594e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 57090, "total_steps": 72479, "tokens": 29931601920, "loss": 3.0807423889636993, "l_mix": 1.8960400372743607, "l_a": 2.0067080557346344, "l_c": 1.9422995150089264, "z_loss": 44.875783920288086, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16871237754821777, "tokens_per_second": 1453784.619856531, "eta_seconds": 5549.837246727943, "data_wait_seconds": 3.731250762939453e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 57100, "total_steps": 72479, "tokens": 29936844800, "loss": 3.1762337684631348, "l_mix": 1.954137459397316, "l_a": 2.069688528776169, "l_c": 2.0039655566215515, "z_loss": 45.89248275756836, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.278471976518631, "tokens_per_second": 1446233.9161956098, "eta_seconds": 5575.187431097031, "data_wait_seconds": 2.0444393157958984e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.16734634, "gate_a2b/L1_absmax": 0.70058811, "gate_a2b/L1_rms": 0.17800419, "gate_a2b/L3_absmean": 0.15974562, "gate_a2b/L3_absmax": 0.51277989, "gate_a2b/L3_rms": 0.16552374, "gate_a2b/L5_absmean": 0.23566274, "gate_a2b/L5_absmax": 0.73433089, "gate_a2b/L5_rms": 0.24176651, "gate_c2b/L1_absmean": 0.03040578, "gate_c2b/L1_absmax": 0.63585007, "gate_c2b/L1_rms": 0.06831758, "gate_c2b/L3_absmean": 0.17398065, "gate_c2b/L3_absmax": 0.39131793, "gate_c2b/L3_rms": 0.17764318, "gate_c2b/L5_absmean": 0.49186116, "gate_c2b/L5_absmax": 0.68786246, "gate_c2b/L5_rms": 0.50164706, "gate_b2a/L7_absmean": 0.05058852, "gate_b2a/L7_absmax": 0.33932674, "gate_b2a/L7_rms": 0.05648034, "gate_b2a/L11_absmean": 0.08224031, "gate_b2a/L11_absmax": 0.61179447, "gate_b2a/L11_rms": 0.09144135, "gate_b2a/L14_absmean": 0.21234664, "gate_b2a/L14_absmax": 0.82401139, "gate_b2a/L14_rms": 0.22170296, "gate_b2c/L3_absmean": 0.03505671, "gate_b2c/L3_absmax": 0.55462337, "gate_b2c/L3_rms": 0.04579296, "gate_b2c/L5_absmean": 0.03693599, "gate_b2c/L5_absmax": 0.5381254, "gate_b2c/L5_rms": 0.04638403, "gate_b2c/L6_absmean": 0.0534082, "gate_b2c/L6_absmax": 0.35271066, "gate_b2c/L6_rms": 0.05970984}, "streams": {"cos_a2b/round0": 0.353986, "cos_c2b/round0": 0.230446, "cos_b2a/round0": 0.143768, "cos_b2c/round0": 0.1228, "rms_a/round0": 7.472919, "rms_b/round0": 12.657091, "rms_c/round0": 4.834715, "cos_a2b/round1": 0.408537, "cos_c2b/round1": 0.30211, "cos_b2a/round1": 0.239182, "cos_b2c/round1": 0.119949, "rms_a/round1": 14.827075, "rms_b/round1": 30.401602, "rms_c/round1": 10.482, "cos_a2b/round2": 0.521509, "cos_c2b/round2": 0.507398, "cos_b2a/round2": 0.088352, "cos_b2c/round2": 0.16974, "rms_a/round2": 40.57663, "rms_b/round2": 79.539543, "rms_c/round2": 20.111975}, "alpha_mean": 0.4355811218744883, "alpha_hist": [0.03165, 0.13913, 0.18024, 0.15025, 0.11311, 0.10831, 0.10762, 0.09889, 0.06464, 0.00616], "alpha_by_type": {"special": {"share": 0.00104, "mean": 0.68827, "std": 0.10876}, "whitespace": {"share": 0.03758, "mean": 0.64115, "std": 0.16748}, "digit": {"share": 0.01334, "mean": 0.39778, "std": 0.17106}, "word_start": {"share": 0.57307, "mean": 0.44325, "std": 0.21715}, "word_cont": {"share": 0.25305, "mean": 0.28622, "std": 0.14946}, "punct": {"share": 0.12192, "mean": 0.64818, "std": 0.18858}}, "alpha_p05": 0.12065, "alpha_p50": 0.38931, "alpha_p95": 0.81197}
{"step": 57110, "total_steps": 72479, "tokens": 29942087680, "loss": 4.338677644729614, "l_mix": 2.688870996236801, "l_a": 2.7753969728946686, "l_c": 2.7239583134651184, "z_loss": 40.7930850982666, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13621366024017334, "tokens_per_second": 1224229.7249476062, "eta_seconds": 6581.920131325722, "data_wait_seconds": 1.7017126083374023e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 57120, "total_steps": 72479, "tokens": 29947330560, "loss": 4.361177146434784, "l_mix": 2.702236294746399, "l_a": 2.792694240808487, "l_c": 2.737108498811722, "z_loss": 41.16616868972778, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16304849088191986, "tokens_per_second": 1168767.9050011681, "eta_seconds": 6889.7677268028265, "data_wait_seconds": 1.817941665649414e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 57130, "total_steps": 72479, "tokens": 29952573440, "loss": 4.405910551548004, "l_mix": 2.731567621231079, "l_a": 2.813703417778015, "l_c": 2.7674397826194763, "z_loss": 40.960880279541016, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15520259737968445, "tokens_per_second": 1415675.3163698297, "eta_seconds": 5684.42242295742, "data_wait_seconds": 1.7911195755004883e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 57140, "total_steps": 72479, "tokens": 29957816320, "loss": 4.402171552181244, "l_mix": 2.728284180164337, "l_a": 2.8159467577934265, "l_c": 2.7636775076389313, "z_loss": 41.5322699546814, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1231701448559761, "tokens_per_second": 1382352.576738301, "eta_seconds": 5817.657352638244, "data_wait_seconds": 1.6987323760986328e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 57150, "total_steps": 72479, "tokens": 29963059200, "loss": 4.284278512001038, "l_mix": 2.654467284679413, "l_a": 2.741696774959564, "l_c": 2.6910073459148407, "z_loss": 41.17662954330444, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14729584753513336, "tokens_per_second": 1202388.2686377096, "eta_seconds": 6684.039558291435, "data_wait_seconds": 1.7404556274414062e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 57160, "total_steps": 72479, "tokens": 29968302080, "loss": 3.0957378447055817, "l_mix": 1.9056310206651688, "l_a": 2.01492440700531, "l_c": 1.9520979672670364, "z_loss": 43.114890575408936, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1720055192708969, "tokens_per_second": 1362183.9409268142, "eta_seconds": 5896.096430659294, "data_wait_seconds": 2.2739171981811523e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 57170, "total_steps": 72479, "tokens": 29973544960, "loss": 3.1207679510116577, "l_mix": 1.9200910180807114, "l_a": 2.0355201810598373, "l_c": 1.9667361676692963, "z_loss": 45.46016263961792, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2323785424232483, "tokens_per_second": 1295198.965284205, "eta_seconds": 6196.982245302201, "data_wait_seconds": 2.8908252716064453e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 57180, "total_steps": 72479, "tokens": 29978787840, "loss": 3.1354327499866486, "l_mix": 1.928856909275055, "l_a": 2.04341721534729, "l_c": 1.9785021841526031, "z_loss": 44.14629936218262, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1592247188091278, "tokens_per_second": 1350784.4083362727, "eta_seconds": 5938.092017126083, "data_wait_seconds": 2.1576881408691406e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 57190, "total_steps": 72479, "tokens": 29984030720, "loss": 3.1198347210884094, "l_mix": 1.9199736267328262, "l_a": 2.03052881360054, "l_c": 1.9690080136060715, "z_loss": 44.027870178222656, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1436222791671753, "tokens_per_second": 1419977.3758811483, "eta_seconds": 5645.047145223618, "data_wait_seconds": 1.8268823623657227e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 57200, "total_steps": 72479, "tokens": 29989273600, "loss": 3.123101234436035, "l_mix": 1.9216979444026947, "l_a": 2.036120265722275, "l_c": 1.9685572385787964, "z_loss": 43.52503776550293, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1216471716761589, "tokens_per_second": 1339386.5915188794, "eta_seconds": 5980.7947927236555, "data_wait_seconds": 3.457069396972656e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16667143, "gate_a2b/L1_absmax": 0.69434756, "gate_a2b/L1_rms": 0.17745791, "gate_a2b/L3_absmean": 0.15935746, "gate_a2b/L3_absmax": 0.51122397, "gate_a2b/L3_rms": 0.16516069, "gate_a2b/L5_absmean": 0.23636697, "gate_a2b/L5_absmax": 0.73564148, "gate_a2b/L5_rms": 0.24243021, "gate_c2b/L1_absmean": 0.03143764, "gate_c2b/L1_absmax": 0.63016784, "gate_c2b/L1_rms": 0.06825863, "gate_c2b/L3_absmean": 0.17319196, "gate_c2b/L3_absmax": 0.38707453, "gate_c2b/L3_rms": 0.17686339, "gate_c2b/L5_absmean": 0.4908739, "gate_c2b/L5_absmax": 0.69869906, "gate_c2b/L5_rms": 0.50066876, "gate_b2a/L7_absmean": 0.05110093, "gate_b2a/L7_absmax": 0.33286476, "gate_b2a/L7_rms": 0.05684643, "gate_b2a/L11_absmean": 0.08227479, "gate_b2a/L11_absmax": 0.61112082, "gate_b2a/L11_rms": 0.09148082, "gate_b2a/L14_absmean": 0.21260725, "gate_b2a/L14_absmax": 0.8279776, "gate_b2a/L14_rms": 0.22211063, "gate_b2c/L3_absmean": 0.03512672, "gate_b2c/L3_absmax": 0.5484187, "gate_b2c/L3_rms": 0.04565981, "gate_b2c/L5_absmean": 0.03712419, "gate_b2c/L5_absmax": 0.53246242, "gate_b2c/L5_rms": 0.04636526, "gate_b2c/L6_absmean": 0.05393153, "gate_b2c/L6_absmax": 0.35543162, "gate_b2c/L6_rms": 0.06029224}, "streams": {"cos_a2b/round0": 0.355286, "cos_c2b/round0": 0.224812, "cos_b2a/round0": 0.142004, "cos_b2c/round0": 0.120909, "rms_a/round0": 7.524072, "rms_b/round0": 12.758595, "rms_c/round0": 4.889188, "cos_a2b/round1": 0.40609, "cos_c2b/round1": 0.304438, "cos_b2a/round1": 0.242168, "cos_b2c/round1": 0.120416, "rms_a/round1": 14.783314, "rms_b/round1": 30.548424, "rms_c/round1": 10.751873, "cos_a2b/round2": 0.511602, "cos_c2b/round2": 0.515589, "cos_b2a/round2": 0.088399, "cos_b2c/round2": 0.170368, "rms_a/round2": 40.359653, "rms_b/round2": 80.10257, "rms_c/round2": 20.717363}, "alpha_mean": 0.4083692196246176, "alpha_hist": [0.03131, 0.16536, 0.20157, 0.15099, 0.11014, 0.10759, 0.10008, 0.08565, 0.04358, 0.00374], "alpha_by_type": {"special": {"share": 0.00099, "mean": 0.62232, "std": 0.13414}, "whitespace": {"share": 0.04256, "mean": 0.62298, "std": 0.16112}, "digit": {"share": 0.01318, "mean": 0.38519, "std": 0.17953}, "word_start": {"share": 0.5527, "mean": 0.4064, "std": 0.21092}, "word_cont": {"share": 0.27016, "mean": 0.28188, "std": 0.1411}, "punct": {"share": 0.12041, "mean": 0.62614, "std": 0.18252}}, "alpha_p05": 0.12963, "alpha_p50": 0.37984, "alpha_p95": 0.801}
{"step": 57210, "total_steps": 72479, "tokens": 29994516480, "loss": 3.0906962454319, "l_mix": 1.9011920392513275, "l_a": 2.0145219266414642, "l_c": 1.950492024421692, "z_loss": 44.12632608413696, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1989922672510147, "tokens_per_second": 1429432.699686841, "eta_seconds": 5600.37102394104, "data_wait_seconds": 1.9818544387817383e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 57220, "total_steps": 72479, "tokens": 29999759360, "loss": 3.056481420993805, "l_mix": 1.8819633573293686, "l_a": 1.9893324673175812, "l_c": 1.9257276356220245, "z_loss": 44.359915256500244, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.21193300187587738, "tokens_per_second": 1359499.1949438788, "eta_seconds": 5884.601198554039, "data_wait_seconds": 2.9742717742919922e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 57230, "total_steps": 72479, "tokens": 30005002240, "loss": 4.286303758621216, "l_mix": 2.656186580657959, "l_a": 2.739551156759262, "l_c": 2.6941724121570587, "z_loss": 40.733888149261475, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19645088911056519, "tokens_per_second": 1306640.238360983, "eta_seconds": 6118.644962310791, "data_wait_seconds": 1.7851591110229492e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 57240, "total_steps": 72479, "tokens": 30010245120, "loss": 4.2414897084236145, "l_mix": 2.6281204223632812, "l_a": 2.716707020998001, "l_c": 2.6611901223659515, "z_loss": 41.70487642288208, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14082269370555878, "tokens_per_second": 1368535.5384209333, "eta_seconds": 5838.083562827111, "data_wait_seconds": 1.7374753952026367e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 57250, "total_steps": 72479, "tokens": 30015488000, "loss": 4.3245896100997925, "l_mix": 2.679810106754303, "l_a": 2.767523229122162, "l_c": 2.7150751054286957, "z_loss": 42.0719051361084, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2064547836780548, "tokens_per_second": 1447835.8970436938, "eta_seconds": 5514.700919008255, "data_wait_seconds": 1.5020370483398438e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 57260, "total_steps": 72479, "tokens": 30020730880, "loss": 3.113018959760666, "l_mix": 1.9152234643697739, "l_a": 2.030769631266594, "l_c": 1.9618817865848541, "z_loss": 44.99337720870972, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16296982765197754, "tokens_per_second": 1421081.5677019944, "eta_seconds": 5614.835385489464, "data_wait_seconds": 1.8894672393798828e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 57270, "total_steps": 72479, "tokens": 30025973760, "loss": 4.617569625377655, "l_mix": 2.864148199558258, "l_a": 2.9439632296562195, "l_c": 2.9007746279239655, "z_loss": 39.90104055404663, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1214175820350647, "tokens_per_second": 1299167.0327427394, "eta_seconds": 6137.698995614052, "data_wait_seconds": 2.6434659957885742e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 57280, "total_steps": 72479, "tokens": 30031216640, "loss": 3.1216716170310974, "l_mix": 1.9211283028125763, "l_a": 2.0330109298229218, "l_c": 1.9687999039888382, "z_loss": 43.66496801376343, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14049257338047028, "tokens_per_second": 1453915.533988966, "eta_seconds": 5480.822733998299, "data_wait_seconds": 1.621246337890625e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 57290, "total_steps": 72479, "tokens": 30036459520, "loss": 3.0908666849136353, "l_mix": 1.901968002319336, "l_a": 2.014143615961075, "l_c": 1.9488519728183746, "z_loss": 44.283387184143066, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16468791663646698, "tokens_per_second": 1402358.6131374806, "eta_seconds": 5678.583464598655, "data_wait_seconds": 1.7255544662475586e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 57300, "total_steps": 72479, "tokens": 30041702400, "loss": 3.069118767976761, "l_mix": 1.8884828090667725, "l_a": 2.0008531659841537, "l_c": 1.9345997422933578, "z_loss": 45.01994800567627, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1690005213022232, "tokens_per_second": 1422720.0460441425, "eta_seconds": 5593.628608894349, "data_wait_seconds": 2.1725893020629883e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16622572, "gate_a2b/L1_absmax": 0.69463807, "gate_a2b/L1_rms": 0.1769, "gate_a2b/L3_absmean": 0.15952484, "gate_a2b/L3_absmax": 0.51135039, "gate_a2b/L3_rms": 0.1653264, "gate_a2b/L5_absmean": 0.23542424, "gate_a2b/L5_absmax": 0.73286176, "gate_a2b/L5_rms": 0.24148349, "gate_c2b/L1_absmean": 0.03117665, "gate_c2b/L1_absmax": 0.63173479, "gate_c2b/L1_rms": 0.06839853, "gate_c2b/L3_absmean": 0.17346911, "gate_c2b/L3_absmax": 0.38920048, "gate_c2b/L3_rms": 0.17713472, "gate_c2b/L5_absmean": 0.4912858, "gate_c2b/L5_absmax": 0.6937986, "gate_c2b/L5_rms": 0.50110775, "gate_b2a/L7_absmean": 0.0508759, "gate_b2a/L7_absmax": 0.33338097, "gate_b2a/L7_rms": 0.05662008, "gate_b2a/L11_absmean": 0.08218604, "gate_b2a/L11_absmax": 0.61213434, "gate_b2a/L11_rms": 0.09140604, "gate_b2a/L14_absmean": 0.21306312, "gate_b2a/L14_absmax": 0.82682258, "gate_b2a/L14_rms": 0.22249489, "gate_b2c/L3_absmean": 0.0348338, "gate_b2c/L3_absmax": 0.54850239, "gate_b2c/L3_rms": 0.04544567, "gate_b2c/L5_absmean": 0.03703983, "gate_b2c/L5_absmax": 0.53494972, "gate_b2c/L5_rms": 0.04636189, "gate_b2c/L6_absmean": 0.05348601, "gate_b2c/L6_absmax": 0.35247594, "gate_b2c/L6_rms": 0.05984243}, "streams": {"cos_a2b/round0": 0.354722, "cos_c2b/round0": 0.229156, "cos_b2a/round0": 0.146149, "cos_b2c/round0": 0.122614, "rms_a/round0": 7.532832, "rms_b/round0": 12.732112, "rms_c/round0": 4.856147, "cos_a2b/round1": 0.408626, "cos_c2b/round1": 0.308388, "cos_b2a/round1": 0.241201, "cos_b2c/round1": 0.1206, "rms_a/round1": 14.907799, "rms_b/round1": 30.56065, "rms_c/round1": 10.682279, "cos_a2b/round2": 0.520896, "cos_c2b/round2": 0.510397, "cos_b2a/round2": 0.091025, "cos_b2c/round2": 0.170977, "rms_a/round2": 40.862667, "rms_b/round2": 80.080048, "rms_c/round2": 20.685131}, "alpha_mean": 0.4223411235845802, "alpha_hist": [0.02991, 0.14384, 0.18858, 0.15187, 0.1196, 0.11702, 0.11064, 0.09348, 0.04256, 0.0025], "alpha_by_type": {"special": {"share": 0.00114, "mean": 0.68834, "std": 0.09891}, "whitespace": {"share": 0.03491, "mean": 0.62484, "std": 0.16165}, "digit": {"share": 0.01031, "mean": 0.41221, "std": 0.17334}, "word_start": {"share": 0.58456, "mean": 0.42808, "std": 0.20481}, "word_cont": {"share": 0.25467, "mean": 0.28331, "std": 0.14631}, "punct": {"share": 0.1144, "mean": 0.63898, "std": 0.17633}}, "alpha_p05": 0.11757, "alpha_p50": 0.35981, "alpha_p95": 0.79053}
{"step": 57310, "total_steps": 72479, "tokens": 30046945280, "loss": 3.0831668376922607, "l_mix": 1.897934377193451, "l_a": 2.0042355954647064, "l_c": 1.9465389251708984, "z_loss": 45.210965633392334, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17483152449131012, "tokens_per_second": 1469704.0689934217, "eta_seconds": 5411.242194795608, "data_wait_seconds": 2.0235776901245117e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 57320, "total_steps": 72479, "tokens": 30052188160, "loss": 3.1237087547779083, "l_mix": 1.9217937737703323, "l_a": 2.0356205850839615, "l_c": 1.9707625657320023, "z_loss": 45.263068199157715, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20480667054653168, "tokens_per_second": 1436788.6366130775, "eta_seconds": 5531.559471917152, "data_wait_seconds": 1.5735626220703125e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 57330, "total_steps": 72479, "tokens": 30057431040, "loss": 3.0947485864162445, "l_mix": 1.9039264023303986, "l_a": 2.0163927376270294, "l_c": 1.9530144333839417, "z_loss": 43.80420970916748, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.21435049176216125, "tokens_per_second": 1278063.1158616766, "eta_seconds": 6214.434023976326, "data_wait_seconds": 2.1696090698242188e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 57340, "total_steps": 72479, "tokens": 30062673920, "loss": 4.272420585155487, "l_mix": 2.6471171975135803, "l_a": 2.734484553337097, "l_c": 2.683193564414978, "z_loss": 42.78829002380371, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15306586027145386, "tokens_per_second": 1461132.0108857502, "eta_seconds": 5432.22376408577, "data_wait_seconds": 1.9252300262451172e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 57350, "total_steps": 72479, "tokens": 30067916800, "loss": 4.35613626241684, "l_mix": 2.6994694471359253, "l_a": 2.7864262759685516, "l_c": 2.735795944929123, "z_loss": 41.19997310638428, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16847838461399078, "tokens_per_second": 1359774.4242504516, "eta_seconds": 5833.286029314995, "data_wait_seconds": 1.806020736694336e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 57360, "total_steps": 72479, "tokens": 30073159680, "loss": 4.27900105714798, "l_mix": 2.6519429981708527, "l_a": 2.7368808686733246, "l_c": 2.686645746231079, "z_loss": 41.017996311187744, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13515698909759521, "tokens_per_second": 1394422.7709125853, "eta_seconds": 5684.581776309013, "data_wait_seconds": 1.9371509552001953e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 57370, "total_steps": 72479, "tokens": 30078402560, "loss": 3.1277793645858765, "l_mix": 1.9239700138568878, "l_a": 2.0378551185131073, "l_c": 1.9748425632715225, "z_loss": 43.38944959640503, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.23469437658786774, "tokens_per_second": 1360860.5396649211, "eta_seconds": 5820.925187492371, "data_wait_seconds": 3.466010093688965e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 57380, "total_steps": 72479, "tokens": 30083645440, "loss": 3.1439277827739716, "l_mix": 1.9359643906354904, "l_a": 2.0443723499774933, "l_c": 1.9821720719337463, "z_loss": 42.28362512588501, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.22022993862628937, "tokens_per_second": 1381514.0892227523, "eta_seconds": 5730.107693982124, "data_wait_seconds": 2.4050474166870117e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 57390, "total_steps": 72479, "tokens": 30088888320, "loss": 3.1430302262306213, "l_mix": 1.9331876039505005, "l_a": 2.048779547214508, "l_c": 1.9840290993452072, "z_loss": 46.0270562171936, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2230474203824997, "tokens_per_second": 1365045.6267740284, "eta_seconds": 5795.3972210407255, "data_wait_seconds": 2.5659799575805664e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 57400, "total_steps": 72479, "tokens": 30094131200, "loss": 4.508346676826477, "l_mix": 2.796243280172348, "l_a": 2.873360514640808, "l_c": 2.833650827407837, "z_loss": 39.59022045135498, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2357090711593628, "tokens_per_second": 1434110.3722138999, "eta_seconds": 5512.643172502518, "data_wait_seconds": 1.8388032913208008e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.16624902, "gate_a2b/L1_absmax": 0.69434291, "gate_a2b/L1_rms": 0.17692345, "gate_a2b/L3_absmean": 0.15934239, "gate_a2b/L3_absmax": 0.51064861, "gate_a2b/L3_rms": 0.16515107, "gate_a2b/L5_absmean": 0.23544586, "gate_a2b/L5_absmax": 0.73319101, "gate_a2b/L5_rms": 0.24149418, "gate_c2b/L1_absmean": 0.03101918, "gate_c2b/L1_absmax": 0.630144, "gate_c2b/L1_rms": 0.06816009, "gate_c2b/L3_absmean": 0.17333305, "gate_c2b/L3_absmax": 0.38927266, "gate_c2b/L3_rms": 0.17704342, "gate_c2b/L5_absmean": 0.49151811, "gate_c2b/L5_absmax": 0.68957067, "gate_c2b/L5_rms": 0.50128829, "gate_b2a/L7_absmean": 0.05050496, "gate_b2a/L7_absmax": 0.33257928, "gate_b2a/L7_rms": 0.05635139, "gate_b2a/L11_absmean": 0.08178301, "gate_b2a/L11_absmax": 0.612041, "gate_b2a/L11_rms": 0.09106479, "gate_b2a/L14_absmean": 0.21310022, "gate_b2a/L14_absmax": 0.82580382, "gate_b2a/L14_rms": 0.22250123, "gate_b2c/L3_absmean": 0.03474509, "gate_b2c/L3_absmax": 0.54753041, "gate_b2c/L3_rms": 0.0453444, "gate_b2c/L5_absmean": 0.03688589, "gate_b2c/L5_absmax": 0.53341913, "gate_b2c/L5_rms": 0.04621594, "gate_b2c/L6_absmean": 0.05349469, "gate_b2c/L6_absmax": 0.35216174, "gate_b2c/L6_rms": 0.05980011}, "streams": {"cos_a2b/round0": 0.35699, "cos_c2b/round0": 0.264873, "cos_b2a/round0": 0.167159, "cos_b2c/round0": 0.146532, "rms_a/round0": 7.198819, "rms_b/round0": 12.531354, "rms_c/round0": 4.731352, "cos_a2b/round1": 0.419143, "cos_c2b/round1": 0.313797, "cos_b2a/round1": 0.257081, "cos_b2c/round1": 0.119015, "rms_a/round1": 14.130808, "rms_b/round1": 28.811117, "rms_c/round1": 10.067549, "cos_a2b/round2": 0.543037, "cos_c2b/round2": 0.526485, "cos_b2a/round2": 0.100965, "cos_b2c/round2": 0.187596, "rms_a/round2": 39.680607, "rms_b/round2": 77.721863, "rms_c/round2": 19.428801}, "alpha_mean": 0.4295705132889296, "alpha_hist": [0.0081, 0.06982, 0.17896, 0.20142, 0.19196, 0.16801, 0.11891, 0.05836, 0.00432, 0.00014], "alpha_by_type": {"special": {"share": 0.00081, "mean": 0.54914, "std": 0.11112}, "whitespace": {"share": 0.04741, "mean": 0.52364, "std": 0.11594}, "digit": {"share": 0.02275, "mean": 0.45885, "std": 0.11785}, "word_start": {"share": 0.59891, "mean": 0.44, "std": 0.16083}, "word_cont": {"share": 0.23221, "mean": 0.31879, "std": 0.1242}, "punct": {"share": 0.09792, "mean": 0.57516, "std": 0.15792}}, "alpha_p05": 0.17275, "alpha_p50": 0.42561, "alpha_p95": 0.7176}
{"step": 57410, "total_steps": 72479, "tokens": 30099374080, "loss": 3.121687799692154, "l_mix": 1.9217122197151184, "l_a": 2.0309423357248306, "l_c": 1.9689760506153107, "z_loss": 45.35134410858154, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20012354850769043, "tokens_per_second": 1406704.7969098624, "eta_seconds": 5616.314019370079, "data_wait_seconds": 1.7017126083374023e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 57420, "total_steps": 72479, "tokens": 30104616960, "loss": 3.0804885029792786, "l_mix": 1.8955874741077423, "l_a": 2.0065251290798187, "l_c": 1.9431448131799698, "z_loss": 45.46060657501221, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14809638261795044, "tokens_per_second": 1455297.7210847843, "eta_seconds": 5425.180619478226, "data_wait_seconds": 2.0235776901245117e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 57430, "total_steps": 72479, "tokens": 30109859840, "loss": 3.150981068611145, "l_mix": 1.9395026862621307, "l_a": 2.0508032143115997, "l_c": 1.9874577820301056, "z_loss": 43.918766498565674, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14336878061294556, "tokens_per_second": 1401454.422527858, "eta_seconds": 5629.872784423828, "data_wait_seconds": 1.5348196029663086e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 57440, "total_steps": 72479, "tokens": 30115102720, "loss": 4.362636148929596, "l_mix": 2.7048975825309753, "l_a": 2.7876638770103455, "l_c": 2.7381310164928436, "z_loss": 41.08882236480713, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11455909162759781, "tokens_per_second": 1435985.1296150615, "eta_seconds": 5490.8418404817585, "data_wait_seconds": 1.7255544662475586e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 57450, "total_steps": 72479, "tokens": 30120345600, "loss": 4.192050874233246, "l_mix": 2.597534477710724, "l_a": 2.6828483641147614, "l_c": 2.632206082344055, "z_loss": 41.36776399612427, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12282219529151917, "tokens_per_second": 1445748.1388421606, "eta_seconds": 5450.136258387566, "data_wait_seconds": 1.9162893295288086e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 57460, "total_steps": 72479, "tokens": 30125588480, "loss": 4.326907515525818, "l_mix": 2.681857019662857, "l_a": 2.7661304473876953, "l_c": 2.717371016740799, "z_loss": 39.83342790603638, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19508680701255798, "tokens_per_second": 1406809.728694591, "eta_seconds": 5597.261172842979, "data_wait_seconds": 1.570582389831543e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 57470, "total_steps": 72479, "tokens": 30130831360, "loss": 4.3992913365364075, "l_mix": 2.726566195487976, "l_a": 2.8133911192417145, "l_c": 2.7623592019081116, "z_loss": 41.54457998275757, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15883639454841614, "tokens_per_second": 1439850.2568635575, "eta_seconds": 5465.178447890281, "data_wait_seconds": 2.059340476989746e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 57480, "total_steps": 72479, "tokens": 30136074240, "loss": 3.1560181379318237, "l_mix": 1.9418849498033524, "l_a": 2.05543515086174, "l_c": 1.9916753619909286, "z_loss": 45.04678392410278, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1714784950017929, "tokens_per_second": 1362981.7184817374, "eta_seconds": 5769.553329563141, "data_wait_seconds": 6.097555160522461e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 57490, "total_steps": 72479, "tokens": 30141317120, "loss": 3.1279380917549133, "l_mix": 1.9260384738445282, "l_a": 2.0334413647651672, "l_c": 1.9728904068470001, "z_loss": 43.93964385986328, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17983880639076233, "tokens_per_second": 1399932.5286461108, "eta_seconds": 5613.522559976578, "data_wait_seconds": 4.0411949157714844e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 57500, "total_steps": 72479, "tokens": 30146560000, "loss": 3.1166991889476776, "l_mix": 1.9175905585289001, "l_a": 2.0315748751163483, "l_c": 1.9654536843299866, "z_loss": 43.463611125946045, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1936250776052475, "tokens_per_second": 1336589.2860476263, "eta_seconds": 5875.634373235703, "data_wait_seconds": 2.6345252990722656e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.16646597, "gate_a2b/L1_absmax": 0.69383109, "gate_a2b/L1_rms": 0.17699476, "gate_a2b/L3_absmean": 0.15935011, "gate_a2b/L3_absmax": 0.51299149, "gate_a2b/L3_rms": 0.16514087, "gate_a2b/L5_absmean": 0.2352151, "gate_a2b/L5_absmax": 0.73493242, "gate_a2b/L5_rms": 0.2412826, "gate_c2b/L1_absmean": 0.03124287, "gate_c2b/L1_absmax": 0.63139921, "gate_c2b/L1_rms": 0.06825867, "gate_c2b/L3_absmean": 0.17308085, "gate_c2b/L3_absmax": 0.39002216, "gate_c2b/L3_rms": 0.17672989, "gate_c2b/L5_absmean": 0.49136057, "gate_c2b/L5_absmax": 0.68776131, "gate_c2b/L5_rms": 0.50117934, "gate_b2a/L7_absmean": 0.05053522, "gate_b2a/L7_absmax": 0.33314556, "gate_b2a/L7_rms": 0.05642503, "gate_b2a/L11_absmean": 0.08226618, "gate_b2a/L11_absmax": 0.61031508, "gate_b2a/L11_rms": 0.09145151, "gate_b2a/L14_absmean": 0.213074, "gate_b2a/L14_absmax": 0.82447344, "gate_b2a/L14_rms": 0.22242461, "gate_b2c/L3_absmean": 0.03511569, "gate_b2c/L3_absmax": 0.54901546, "gate_b2c/L3_rms": 0.04565865, "gate_b2c/L5_absmean": 0.03697365, "gate_b2c/L5_absmax": 0.53546655, "gate_b2c/L5_rms": 0.04631601, "gate_b2c/L6_absmean": 0.05350124, "gate_b2c/L6_absmax": 0.35265821, "gate_b2c/L6_rms": 0.05981429}, "streams": {"cos_a2b/round0": 0.356805, "cos_c2b/round0": 0.227626, "cos_b2a/round0": 0.143162, "cos_b2c/round0": 0.12167, "rms_a/round0": 7.552029, "rms_b/round0": 12.780764, "rms_c/round0": 4.867034, "cos_a2b/round1": 0.409081, "cos_c2b/round1": 0.300421, "cos_b2a/round1": 0.239728, "cos_b2c/round1": 0.119201, "rms_a/round1": 15.0182, "rms_b/round1": 30.761541, "rms_c/round1": 10.739199, "cos_a2b/round2": 0.516483, "cos_c2b/round2": 0.511578, "cos_b2a/round2": 0.086719, "cos_b2c/round2": 0.169342, "rms_a/round2": 40.886208, "rms_b/round2": 80.360046, "rms_c/round2": 20.818829}, "alpha_mean": 0.42505738542773486, "alpha_hist": [0.02739, 0.14519, 0.19022, 0.15498, 0.11908, 0.11055, 0.10187, 0.09088, 0.05513, 0.00471], "alpha_by_type": {"special": {"share": 0.00107, "mean": 0.6117, "std": 0.13071}, "whitespace": {"share": 0.04097, "mean": 0.622, "std": 0.15865}, "digit": {"share": 0.01689, "mean": 0.42991, "std": 0.16245}, "word_start": {"share": 0.55852, "mean": 0.42912, "std": 0.21431}, "word_cont": {"share": 0.26028, "mean": 0.28688, "std": 0.14217}, "punct": {"share": 0.12227, "mean": 0.63233, "std": 0.18542}}, "alpha_p05": 0.12554, "alpha_p50": 0.39187, "alpha_p95": 0.82104}
{"step": 57510, "total_steps": 72479, "tokens": 30151802880, "loss": 4.646031022071838, "l_mix": 2.8817813396453857, "l_a": 2.959761619567871, "l_c": 2.9210703372955322, "z_loss": 41.109838008880615, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15127667784690857, "tokens_per_second": 1442622.5140623045, "eta_seconds": 5440.139049196244, "data_wait_seconds": 2.1904706954956055e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 57520, "total_steps": 72479, "tokens": 30157045760, "loss": 4.639236688613892, "l_mix": 2.878885507583618, "l_a": 2.9546353220939636, "l_c": 2.9132016003131866, "z_loss": 39.24296236038208, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1374746561050415, "tokens_per_second": 1432388.3723029175, "eta_seconds": 5475.347568893432, "data_wait_seconds": 1.77919864654541e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 57530, "total_steps": 72479, "tokens": 30162288640, "loss": 4.682683646678925, "l_mix": 2.905658632516861, "l_a": 2.9788918495178223, "l_c": 2.9445249140262604, "z_loss": 40.49907159805298, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14067283272743225, "tokens_per_second": 1435788.8930478517, "eta_seconds": 5458.728194618226, "data_wait_seconds": 1.65402889251709e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 57540, "total_steps": 72479, "tokens": 30167531520, "loss": 3.1145918369293213, "l_mix": 1.9156460165977478, "l_a": 2.029913365840912, "l_c": 1.966572344303131, "z_loss": 45.135788917541504, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18116071820259094, "tokens_per_second": 1407870.273682221, "eta_seconds": 5563.252934885025, "data_wait_seconds": 2.3931264877319336e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 57550, "total_steps": 72479, "tokens": 30172774400, "loss": 4.236010372638702, "l_mix": 2.6240075528621674, "l_a": 2.71298685669899, "l_c": 2.6603559255599976, "z_loss": 42.62516689300537, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13164553046226501, "tokens_per_second": 1474416.6313724515, "eta_seconds": 5308.6050343275065, "data_wait_seconds": 1.5616416931152344e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 57560, "total_steps": 72479, "tokens": 30178017280, "loss": 4.234725892543793, "l_mix": 2.6233593225479126, "l_a": 2.7122679352760315, "l_c": 2.6589537858963013, "z_loss": 42.15049600601196, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1147608533501625, "tokens_per_second": 1403579.33554274, "eta_seconds": 5572.789848017693, "data_wait_seconds": 2.5957822799682617e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 57570, "total_steps": 72479, "tokens": 30183260160, "loss": 4.286876887083054, "l_mix": 2.6565078794956207, "l_a": 2.7439144551754, "l_c": 2.6906484365463257, "z_loss": 42.12279176712036, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18153613805770874, "tokens_per_second": 1378277.5696014976, "eta_seconds": 5671.288544774056, "data_wait_seconds": 3.0308961868286133e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 57580, "total_steps": 72479, "tokens": 30188503040, "loss": 4.218090236186981, "l_mix": 2.612363576889038, "l_a": 2.7025921046733856, "l_c": 2.649829775094986, "z_loss": 41.36549091339111, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16855116188526154, "tokens_per_second": 1389296.6881842166, "eta_seconds": 5622.533313751221, "data_wait_seconds": 1.58846378326416e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 57590, "total_steps": 72479, "tokens": 30193745920, "loss": 3.0584496557712555, "l_mix": 1.8819622248411179, "l_a": 1.9927242994308472, "l_c": 1.9289002567529678, "z_loss": 45.45655870437622, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1462760716676712, "tokens_per_second": 1336164.864322673, "eta_seconds": 5842.18627538681, "data_wait_seconds": 2.428889274597168e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 57600, "total_steps": 72479, "tokens": 30198988800, "loss": 3.0985636711120605, "l_mix": 1.9065884798765182, "l_a": 2.0186901837587357, "l_c": 1.9545602798461914, "z_loss": 44.82731342315674, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14756180346012115, "tokens_per_second": 1446122.73575645, "eta_seconds": 5394.34237434864, "data_wait_seconds": 2.378225326538086e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16503197, "gate_a2b/L1_absmax": 0.70201367, "gate_a2b/L1_rms": 0.17593582, "gate_a2b/L3_absmean": 0.158649, "gate_a2b/L3_absmax": 0.5112856, "gate_a2b/L3_rms": 0.16450877, "gate_a2b/L5_absmean": 0.23554759, "gate_a2b/L5_absmax": 0.73633748, "gate_a2b/L5_rms": 0.24160489, "gate_c2b/L1_absmean": 0.03194714, "gate_c2b/L1_absmax": 0.64394087, "gate_c2b/L1_rms": 0.06912155, "gate_c2b/L3_absmean": 0.17349182, "gate_c2b/L3_absmax": 0.38729933, "gate_c2b/L3_rms": 0.17716461, "gate_c2b/L5_absmean": 0.49006644, "gate_c2b/L5_absmax": 0.69364911, "gate_c2b/L5_rms": 0.4997212, "gate_b2a/L7_absmean": 0.05104557, "gate_b2a/L7_absmax": 0.33673087, "gate_b2a/L7_rms": 0.05674437, "gate_b2a/L11_absmean": 0.08263348, "gate_b2a/L11_absmax": 0.61422843, "gate_b2a/L11_rms": 0.09186156, "gate_b2a/L14_absmean": 0.21277454, "gate_b2a/L14_absmax": 0.81767482, "gate_b2a/L14_rms": 0.22211301, "gate_b2c/L3_absmean": 0.03470241, "gate_b2c/L3_absmax": 0.55975729, "gate_b2c/L3_rms": 0.04574945, "gate_b2c/L5_absmean": 0.03710433, "gate_b2c/L5_absmax": 0.53192979, "gate_b2c/L5_rms": 0.04637363, "gate_b2c/L6_absmean": 0.05343418, "gate_b2c/L6_absmax": 0.35102943, "gate_b2c/L6_rms": 0.05969015}, "streams": {"cos_a2b/round0": 0.356402, "cos_c2b/round0": 0.225318, "cos_b2a/round0": 0.141918, "cos_b2c/round0": 0.120249, "rms_a/round0": 7.638561, "rms_b/round0": 12.814422, "rms_c/round0": 4.924331, "cos_a2b/round1": 0.410572, "cos_c2b/round1": 0.303394, "cos_b2a/round1": 0.241133, "cos_b2c/round1": 0.118364, "rms_a/round1": 15.012074, "rms_b/round1": 30.961355, "rms_c/round1": 10.714314, "cos_a2b/round2": 0.514323, "cos_c2b/round2": 0.510949, "cos_b2a/round2": 0.086435, "cos_b2c/round2": 0.17011, "rms_a/round2": 40.954708, "rms_b/round2": 81.343742, "rms_c/round2": 20.860615}, "alpha_mean": 0.40541131474117975, "alpha_hist": [0.03711, 0.17091, 0.19142, 0.14485, 0.11159, 0.1097, 0.11034, 0.08717, 0.03519, 0.00172], "alpha_by_type": {"special": {"share": 0.00102, "mean": 0.63334, "std": 0.12319}, "whitespace": {"share": 0.036, "mean": 0.62836, "std": 0.16801}, "digit": {"share": 0.01019, "mean": 0.47246, "std": 0.17175}, "word_start": {"share": 0.56259, "mean": 0.40873, "std": 0.21061}, "word_cont": {"share": 0.27135, "mean": 0.27578, "std": 0.14208}, "punct": {"share": 0.11885, "mean": 0.61044, "std": 0.18087}}, "alpha_p05": 0.1101, "alpha_p50": 0.37114, "alpha_p95": 0.77764}
{"step": 57610, "total_steps": 72479, "tokens": 30204231680, "loss": 3.104738563299179, "l_mix": 1.9092064648866653, "l_a": 2.0242954939603806, "l_c": 1.9608113467693329, "z_loss": 46.29982566833496, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18828794360160828, "tokens_per_second": 1418014.5758956063, "eta_seconds": 5497.572736215591, "data_wait_seconds": 2.619624137878418e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 57620, "total_steps": 72479, "tokens": 30209474560, "loss": 3.0728743374347687, "l_mix": 1.9020822942256927, "l_a": 1.9828199595212936, "l_c": 1.9198200702667236, "z_loss": 51.456480979919434, "lr": 0.0023, "wd": 0.01, "grad_norm": 1.4646944999694824, "tokens_per_second": 1405033.4010106295, "eta_seconds": 5544.633591198921, "data_wait_seconds": 1.609325408935547e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 57630, "total_steps": 72479, "tokens": 30214717440, "loss": 4.156177878379822, "l_mix": 2.5741431415081024, "l_a": 2.6633121371269226, "l_c": 2.610136568546295, "z_loss": 42.509936809539795, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13594962656497955, "tokens_per_second": 1410752.2017522396, "eta_seconds": 5518.440802240372, "data_wait_seconds": 1.8596649169921875e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 57640, "total_steps": 72479, "tokens": 30219960320, "loss": 4.3496240973472595, "l_mix": 2.6956345438957214, "l_a": 2.7831077873706818, "l_c": 2.730190396308899, "z_loss": 42.11030197143555, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2088261842727661, "tokens_per_second": 1479090.9636380868, "eta_seconds": 5259.92641646862, "data_wait_seconds": 2.2798776626586914e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 57650, "total_steps": 72479, "tokens": 30225203200, "loss": 4.387669742107391, "l_mix": 2.7193987369537354, "l_a": 2.8032446205615997, "l_c": 2.757658898830414, "z_loss": 40.84871959686279, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17861968278884888, "tokens_per_second": 1457078.2712780708, "eta_seconds": 5335.792115807533, "data_wait_seconds": 2.0742416381835938e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 57660, "total_steps": 72479, "tokens": 30230446080, "loss": 4.21561735868454, "l_mix": 2.6128095984458923, "l_a": 2.697420984506607, "l_c": 2.645271450281143, "z_loss": 41.8127384185791, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16899000108242035, "tokens_per_second": 1455362.0593784088, "eta_seconds": 5338.481803846359, "data_wait_seconds": 1.7076730728149414e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 57670, "total_steps": 72479, "tokens": 30235688960, "loss": 4.1875141859054565, "l_mix": 2.5935965478420258, "l_a": 2.684006303548813, "l_c": 2.629052370786667, "z_loss": 43.33731555938721, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1955116242170334, "tokens_per_second": 1465888.8509046151, "eta_seconds": 5296.568690872193, "data_wait_seconds": 2.5480985641479492e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 57680, "total_steps": 72479, "tokens": 30240931840, "loss": 4.16096830368042, "l_mix": 2.577350676059723, "l_a": 2.6643716394901276, "l_c": 2.6143535673618317, "z_loss": 41.50007104873657, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1846630573272705, "tokens_per_second": 1427791.2475387256, "eta_seconds": 5434.2244535923, "data_wait_seconds": 2.9414892196655273e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 57690, "total_steps": 72479, "tokens": 30246174720, "loss": 4.352470397949219, "l_mix": 2.697413682937622, "l_a": 2.7849549055099487, "l_c": 2.73190039396286, "z_loss": 41.65984392166138, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.21387887001037598, "tokens_per_second": 1361155.0244175878, "eta_seconds": 5696.408633041382, "data_wait_seconds": 1.7523765563964844e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 57700, "total_steps": 72479, "tokens": 30251417600, "loss": 3.0559367537498474, "l_mix": 1.8801775574684143, "l_a": 1.9922139048576355, "l_c": 1.9269831329584122, "z_loss": 44.80484056472778, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1462571769952774, "tokens_per_second": 1375790.2611796635, "eta_seconds": 5632.001163721085, "data_wait_seconds": 1.9550323486328125e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16411822, "gate_a2b/L1_absmax": 0.705212, "gate_a2b/L1_rms": 0.17506917, "gate_a2b/L3_absmean": 0.15744416, "gate_a2b/L3_absmax": 0.51163733, "gate_a2b/L3_rms": 0.16333699, "gate_a2b/L5_absmean": 0.23419002, "gate_a2b/L5_absmax": 0.74144506, "gate_a2b/L5_rms": 0.24038228, "gate_c2b/L1_absmean": 0.03271782, "gate_c2b/L1_absmax": 0.64451236, "gate_c2b/L1_rms": 0.06970584, "gate_c2b/L3_absmean": 0.17293821, "gate_c2b/L3_absmax": 0.3975105, "gate_c2b/L3_rms": 0.17670795, "gate_c2b/L5_absmean": 0.49054536, "gate_c2b/L5_absmax": 0.68846172, "gate_c2b/L5_rms": 0.50042909, "gate_b2a/L7_absmean": 0.05078719, "gate_b2a/L7_absmax": 0.33592838, "gate_b2a/L7_rms": 0.05664067, "gate_b2a/L11_absmean": 0.0829818, "gate_b2a/L11_absmax": 0.6098997, "gate_b2a/L11_rms": 0.09214177, "gate_b2a/L14_absmean": 0.21329774, "gate_b2a/L14_absmax": 0.82451272, "gate_b2a/L14_rms": 0.2226411, "gate_b2c/L3_absmean": 0.03530651, "gate_b2c/L3_absmax": 0.56041974, "gate_b2c/L3_rms": 0.04623327, "gate_b2c/L5_absmean": 0.03706817, "gate_b2c/L5_absmax": 0.51524973, "gate_b2c/L5_rms": 0.0460051, "gate_b2c/L6_absmean": 0.05317619, "gate_b2c/L6_absmax": 0.3457678, "gate_b2c/L6_rms": 0.05938678}, "streams": {"cos_a2b/round0": 0.358103, "cos_c2b/round0": 0.227955, "cos_b2a/round0": 0.142636, "cos_b2c/round0": 0.122753, "rms_a/round0": 7.767136, "rms_b/round0": 12.820054, "rms_c/round0": 4.934342, "cos_a2b/round1": 0.411069, "cos_c2b/round1": 0.303995, "cos_b2a/round1": 0.239492, "cos_b2c/round1": 0.120573, "rms_a/round1": 15.163749, "rms_b/round1": 30.896263, "rms_c/round1": 10.738222, "cos_a2b/round2": 0.519205, "cos_c2b/round2": 0.511893, "cos_b2a/round2": 0.089727, "cos_b2c/round2": 0.170943, "rms_a/round2": 41.158047, "rms_b/round2": 81.43026, "rms_c/round2": 20.377302}, "alpha_mean": 0.42126828171707587, "alpha_hist": [0.03722, 0.15514, 0.1757, 0.14801, 0.11993, 0.10735, 0.11011, 0.09555, 0.04401, 0.00699], "alpha_by_type": {"special": {"share": 0.00108, "mean": 0.68847, "std": 0.10304}, "whitespace": {"share": 0.04004, "mean": 0.645, "std": 0.18059}, "digit": {"share": 0.01048, "mean": 0.43541, "std": 0.19139}, "word_start": {"share": 0.5558, "mean": 0.41616, "std": 0.21396}, "word_cont": {"share": 0.25641, "mean": 0.28755, "std": 0.14958}, "punct": {"share": 0.13618, "mean": 0.62487, "std": 0.17875}}, "alpha_p05": 0.10053, "alpha_p50": 0.37205, "alpha_p95": 0.80563}
{"step": 57710, "total_steps": 72479, "tokens": 30256660480, "loss": 3.0823866426944733, "l_mix": 1.8966428637504578, "l_a": 2.0079362988471985, "l_c": 1.9445428848266602, "z_loss": 45.055824756622314, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15389248728752136, "tokens_per_second": 1341872.688659111, "eta_seconds": 5770.450160765648, "data_wait_seconds": 3.549456596374512e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 57720, "total_steps": 72479, "tokens": 30261903360, "loss": 3.0881723165512085, "l_mix": 1.9011147618293762, "l_a": 2.0103899240493774, "l_c": 1.9464685320854187, "z_loss": 44.21863412857056, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16945861279964447, "tokens_per_second": 1363297.3216803952, "eta_seconds": 5675.9200424909595, "data_wait_seconds": 2.130866050720215e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 57730, "total_steps": 72479, "tokens": 30267146240, "loss": 2.906315356492996, "l_mix": 1.7992058247327805, "l_a": 1.8657169938087463, "l_c": 1.8246479332447052, "z_loss": 51.79408884048462, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1633206009864807, "tokens_per_second": 1423432.4772926406, "eta_seconds": 5432.448560333252, "data_wait_seconds": 2.0444393157958984e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 57740, "total_steps": 72479, "tokens": 30272389120, "loss": 4.201147437095642, "l_mix": 2.602191597223282, "l_a": 2.691406399011612, "l_c": 2.6384465098381042, "z_loss": 41.352084159851074, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19720949232578278, "tokens_per_second": 1307898.1196513688, "eta_seconds": 5908.320163393021, "data_wait_seconds": 2.2232532501220703e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 57750, "total_steps": 72479, "tokens": 30277632000, "loss": 4.289037883281708, "l_mix": 2.6578716039657593, "l_a": 2.742247760295868, "l_c": 2.6949731707572937, "z_loss": 41.413331031799316, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16528348624706268, "tokens_per_second": 1451626.7854536783, "eta_seconds": 5319.7130484104155, "data_wait_seconds": 2.804398536682129e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 57760, "total_steps": 72479, "tokens": 30282874880, "loss": 4.2529736161231995, "l_mix": 2.635103791952133, "l_a": 2.7230190336704254, "l_c": 2.669879913330078, "z_loss": 42.25667953491211, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16007758677005768, "tokens_per_second": 1461440.5128904136, "eta_seconds": 5280.403139185905, "data_wait_seconds": 1.5556812286376953e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 57770, "total_steps": 72479, "tokens": 30288117760, "loss": 4.3387985825538635, "l_mix": 2.6890217065811157, "l_a": 2.7720583379268646, "l_c": 2.7271978855133057, "z_loss": 41.33014392852783, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1598077118396759, "tokens_per_second": 1355345.9082003538, "eta_seconds": 5689.877503108978, "data_wait_seconds": 1.823902130126953e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 57780, "total_steps": 72479, "tokens": 30293360640, "loss": 4.219114542007446, "l_mix": 2.6145943701267242, "l_a": 2.698398917913437, "l_c": 2.650001436471939, "z_loss": 43.23594284057617, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18502627313137054, "tokens_per_second": 1229694.4616533988, "eta_seconds": 6267.011483192443, "data_wait_seconds": 2.104043960571289e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 57790, "total_steps": 72479, "tokens": 30298603520, "loss": 4.6048548221588135, "l_mix": 2.8566991090774536, "l_a": 2.9341390132904053, "l_c": 2.893046200275421, "z_loss": 41.80000448226929, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20441900193691254, "tokens_per_second": 1423747.2940359928, "eta_seconds": 5409.152638435364, "data_wait_seconds": 1.7404556274414062e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 57800, "total_steps": 72479, "tokens": 30303846400, "loss": 4.677735984325409, "l_mix": 2.9024141132831573, "l_a": 2.9773860573768616, "l_c": 2.940353184938431, "z_loss": 39.448856830596924, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15147314965724945, "tokens_per_second": 1394401.815272565, "eta_seconds": 5519.229441404343, "data_wait_seconds": 2.0742416381835938e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16496263, "gate_a2b/L1_absmax": 0.69964671, "gate_a2b/L1_rms": 0.17563295, "gate_a2b/L3_absmean": 0.15782242, "gate_a2b/L3_absmax": 0.51051259, "gate_a2b/L3_rms": 0.16357358, "gate_a2b/L5_absmean": 0.23350957, "gate_a2b/L5_absmax": 0.73768348, "gate_a2b/L5_rms": 0.23956169, "gate_c2b/L1_absmean": 0.03177785, "gate_c2b/L1_absmax": 0.63968503, "gate_c2b/L1_rms": 0.06898689, "gate_c2b/L3_absmean": 0.17357917, "gate_c2b/L3_absmax": 0.39522332, "gate_c2b/L3_rms": 0.17729767, "gate_c2b/L5_absmean": 0.49075174, "gate_c2b/L5_absmax": 0.69377202, "gate_c2b/L5_rms": 0.50055826, "gate_b2a/L7_absmean": 0.05075684, "gate_b2a/L7_absmax": 0.3362205, "gate_b2a/L7_rms": 0.0566191, "gate_b2a/L11_absmean": 0.0825072, "gate_b2a/L11_absmax": 0.6112206, "gate_b2a/L11_rms": 0.091666, "gate_b2a/L14_absmean": 0.21363698, "gate_b2a/L14_absmax": 0.8220858, "gate_b2a/L14_rms": 0.22301027, "gate_b2c/L3_absmean": 0.03490466, "gate_b2c/L3_absmax": 0.55763596, "gate_b2c/L3_rms": 0.04569628, "gate_b2c/L5_absmean": 0.03691987, "gate_b2c/L5_absmax": 0.51777875, "gate_b2c/L5_rms": 0.04578884, "gate_b2c/L6_absmean": 0.05335414, "gate_b2c/L6_absmax": 0.34570551, "gate_b2c/L6_rms": 0.05947446}, "streams": {"cos_a2b/round0": 0.346779, "cos_c2b/round0": 0.242962, "cos_b2a/round0": 0.155493, "cos_b2c/round0": 0.133457, "rms_a/round0": 7.379039, "rms_b/round0": 12.900148, "rms_c/round0": 4.865983, "cos_a2b/round1": 0.409787, "cos_c2b/round1": 0.319864, "cos_b2a/round1": 0.252854, "cos_b2c/round1": 0.111854, "rms_a/round1": 14.20016, "rms_b/round1": 29.053265, "rms_c/round1": 10.196589, "cos_a2b/round2": 0.54345, "cos_c2b/round2": 0.528203, "cos_b2a/round2": 0.094498, "cos_b2c/round2": 0.190555, "rms_a/round2": 40.264359, "rms_b/round2": 78.190376, "rms_c/round2": 19.2796}, "alpha_mean": 0.4523439829331437, "alpha_hist": [0.00391, 0.03961, 0.13977, 0.2043, 0.21437, 0.21117, 0.13298, 0.05019, 0.00369, 2e-05], "alpha_by_type": {"special": {"share": 0.0006, "mean": 0.55064, "std": 0.12841}, "whitespace": {"share": 0.04047, "mean": 0.54736, "std": 0.1138}, "digit": {"share": 0.02142, "mean": 0.47962, "std": 0.11348}, "word_start": {"share": 0.53883, "mean": 0.47372, "std": 0.15014}, "word_cont": {"share": 0.21432, "mean": 0.35072, "std": 0.12786}, "punct": {"share": 0.18436, "mean": 0.48367, "std": 0.14732}}, "alpha_p05": 0.20689, "alpha_p50": 0.44879, "alpha_p95": 0.7011}
{"step": 57810, "total_steps": 72479, "tokens": 30309089280, "loss": 4.675859570503235, "l_mix": 2.9016308784484863, "l_a": 2.976829081773758, "l_c": 2.937266379594803, "z_loss": 40.07838487625122, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14660970866680145, "tokens_per_second": 1424265.0721318834, "eta_seconds": 5399.82396709919, "data_wait_seconds": 2.065300941467285e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 57820, "total_steps": 72479, "tokens": 30314332160, "loss": 3.1291803121566772, "l_mix": 1.9262537211179733, "l_a": 2.036505699157715, "l_c": 1.9732494205236435, "z_loss": 45.40358114242554, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18231406807899475, "tokens_per_second": 1294211.2851747046, "eta_seconds": 5938.394974637032, "data_wait_seconds": 2.3424625396728516e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 57830, "total_steps": 72479, "tokens": 30319575040, "loss": 3.126469075679779, "l_mix": 1.9240189492702484, "l_a": 2.0366866886615753, "l_c": 1.971480056643486, "z_loss": 44.6169171333313, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16495606303215027, "tokens_per_second": 1388857.34852315, "eta_seconds": 5529.9379163503645, "data_wait_seconds": 3.090500831604004e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 57840, "total_steps": 72479, "tokens": 30324817920, "loss": 3.1953288316726685, "l_mix": 1.9803683310747147, "l_a": 2.043975353240967, "l_c": 2.0058927685022354, "z_loss": 49.60968589782715, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1945059597492218, "tokens_per_second": 1395486.1453945383, "eta_seconds": 5499.91274175644, "data_wait_seconds": 1.8149614334106445e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 57850, "total_steps": 72479, "tokens": 30330060800, "loss": 4.196898698806763, "l_mix": 2.5996978282928467, "l_a": 2.688687890768051, "l_c": 2.63531494140625, "z_loss": 41.905681133270264, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17147845029830933, "tokens_per_second": 1429411.5148065707, "eta_seconds": 5365.71104440689, "data_wait_seconds": 1.6808509826660156e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 57860, "total_steps": 72479, "tokens": 30335303680, "loss": 4.279961347579956, "l_mix": 2.652889132499695, "l_a": 2.736544966697693, "l_c": 2.6870289146900177, "z_loss": 40.507100105285645, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20875497162342072, "tokens_per_second": 1436972.0943179561, "eta_seconds": 5333.830978560448, "data_wait_seconds": 2.2977590560913086e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 57870, "total_steps": 72479, "tokens": 30340546560, "loss": 4.230544537305832, "l_mix": 2.6220203042030334, "l_a": 2.70447501540184, "l_c": 2.6572724282741547, "z_loss": 40.282206535339355, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.25546544790267944, "tokens_per_second": 1407575.9529630088, "eta_seconds": 5441.499178695679, "data_wait_seconds": 1.71661376953125e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 57880, "total_steps": 72479, "tokens": 30345789440, "loss": 4.225724577903748, "l_mix": 2.6182051599025726, "l_a": 2.7063179314136505, "l_c": 2.6520800590515137, "z_loss": 41.986764430999756, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15538230538368225, "tokens_per_second": 1469840.3221877303, "eta_seconds": 5207.423144173622, "data_wait_seconds": 1.5795230865478516e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 57890, "total_steps": 72479, "tokens": 30351032320, "loss": 4.184306859970093, "l_mix": 2.592046469449997, "l_a": 2.6789145469665527, "l_c": 2.628619998693466, "z_loss": 44.17183971405029, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.159849613904953, "tokens_per_second": 1417135.843150821, "eta_seconds": 5397.391978311538, "data_wait_seconds": 2.7000904083251953e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 57900, "total_steps": 72479, "tokens": 30356275200, "loss": 4.553982615470886, "l_mix": 2.8251563906669617, "l_a": 2.902313470840454, "l_c": 2.860440641641617, "z_loss": 40.214560985565186, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13384181261062622, "tokens_per_second": 1342524.0550122566, "eta_seconds": 5693.45087223053, "data_wait_seconds": 1.6570091247558594e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.16468576, "gate_a2b/L1_absmax": 0.69562, "gate_a2b/L1_rms": 0.17529978, "gate_a2b/L3_absmean": 0.15724717, "gate_a2b/L3_absmax": 0.51097983, "gate_a2b/L3_rms": 0.16303495, "gate_a2b/L5_absmean": 0.2332233, "gate_a2b/L5_absmax": 0.73637033, "gate_a2b/L5_rms": 0.23930238, "gate_c2b/L1_absmean": 0.03137222, "gate_c2b/L1_absmax": 0.63714546, "gate_c2b/L1_rms": 0.06886353, "gate_c2b/L3_absmean": 0.17341007, "gate_c2b/L3_absmax": 0.39368844, "gate_c2b/L3_rms": 0.17714886, "gate_c2b/L5_absmean": 0.49043724, "gate_c2b/L5_absmax": 0.69231993, "gate_c2b/L5_rms": 0.50032127, "gate_b2a/L7_absmean": 0.05112289, "gate_b2a/L7_absmax": 0.33627233, "gate_b2a/L7_rms": 0.05693231, "gate_b2a/L11_absmean": 0.08233517, "gate_b2a/L11_absmax": 0.61136425, "gate_b2a/L11_rms": 0.09154706, "gate_b2a/L14_absmean": 0.21371956, "gate_b2a/L14_absmax": 0.81981295, "gate_b2a/L14_rms": 0.22300294, "gate_b2c/L3_absmean": 0.03475906, "gate_b2c/L3_absmax": 0.55486643, "gate_b2c/L3_rms": 0.04550532, "gate_b2c/L5_absmean": 0.036731, "gate_b2c/L5_absmax": 0.52173328, "gate_b2c/L5_rms": 0.04577461, "gate_b2c/L6_absmean": 0.05325585, "gate_b2c/L6_absmax": 0.34665588, "gate_b2c/L6_rms": 0.05940945}, "streams": {"cos_a2b/round0": 0.357962, "cos_c2b/round0": 0.266771, "cos_b2a/round0": 0.158868, "cos_b2c/round0": 0.145415, "rms_a/round0": 7.4113, "rms_b/round0": 12.653352, "rms_c/round0": 4.729112, "cos_a2b/round1": 0.418624, "cos_c2b/round1": 0.307832, "cos_b2a/round1": 0.254355, "cos_b2c/round1": 0.114382, "rms_a/round1": 14.427146, "rms_b/round1": 29.070707, "rms_c/round1": 10.148336, "cos_a2b/round2": 0.540005, "cos_c2b/round2": 0.524885, "cos_b2a/round2": 0.101483, "cos_b2c/round2": 0.185587, "rms_a/round2": 40.40691, "rms_b/round2": 79.803513, "rms_c/round2": 19.628433}, "alpha_mean": 0.44896858565332565, "alpha_hist": [0.00421, 0.04938, 0.16685, 0.19707, 0.18372, 0.19211, 0.13541, 0.0632, 0.00803, 3e-05], "alpha_by_type": {"special": {"share": 0.00055, "mean": 0.56526, "std": 0.14026}, "whitespace": {"share": 0.06583, "mean": 0.55005, "std": 0.10727}, "digit": {"share": 0.03412, "mean": 0.42489, "std": 0.1174}, "word_start": {"share": 0.52037, "mean": 0.46088, "std": 0.16203}, "word_cont": {"share": 0.25879, "mean": 0.34257, "std": 0.12158}, "punct": {"share": 0.12035, "mean": 0.57728, "std": 0.15099}}, "alpha_p05": 0.2015, "alpha_p50": 0.4506, "alpha_p95": 0.718}
{"step": 57910, "total_steps": 72479, "tokens": 30361518080, "loss": 4.743265151977539, "l_mix": 2.942904531955719, "l_a": 3.0207596123218536, "l_c": 2.9804424941539764, "z_loss": 40.23457908630371, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15687187016010284, "tokens_per_second": 1316438.7491006826, "eta_seconds": 5802.284289503097, "data_wait_seconds": 1.7464160919189453e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 57920, "total_steps": 72479, "tokens": 30366760960, "loss": 4.573987066745758, "l_mix": 2.837975800037384, "l_a": 2.913350820541382, "l_c": 2.87335342168808, "z_loss": 40.7143177986145, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12566672265529633, "tokens_per_second": 1472004.9938877965, "eta_seconds": 5185.5184076786045, "data_wait_seconds": 1.6182661056518555e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 57930, "total_steps": 72479, "tokens": 30372003840, "loss": 4.739156723022461, "l_mix": 2.940850853919983, "l_a": 3.0140834748744965, "l_c": 2.9802692234516144, "z_loss": 39.04045820236206, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19232328236103058, "tokens_per_second": 1368303.491826293, "eta_seconds": 5574.68877158165, "data_wait_seconds": 3.3229589462280273e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 57940, "total_steps": 72479, "tokens": 30377246720, "loss": 4.680831491947174, "l_mix": 2.904110163450241, "l_a": 2.9834415316581726, "l_c": 2.938962548971176, "z_loss": 39.71123027801514, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1873844563961029, "tokens_per_second": 1392536.9897366867, "eta_seconds": 5473.910774493217, "data_wait_seconds": 2.3573637008666992e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 57950, "total_steps": 72479, "tokens": 30382489600, "loss": 4.80224871635437, "l_mix": 2.980560749769211, "l_a": 3.0539090633392334, "l_c": 3.018384039402008, "z_loss": 39.639437675476074, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15248514711856842, "tokens_per_second": 1446166.7683940076, "eta_seconds": 5267.290411090851, "data_wait_seconds": 2.0712614059448242e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 57960, "total_steps": 72479, "tokens": 30387732480, "loss": 4.278627395629883, "l_mix": 2.651455909013748, "l_a": 2.736840546131134, "l_c": 2.6870641708374023, "z_loss": 41.08680057525635, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18627656996250153, "tokens_per_second": 1417534.5921208214, "eta_seconds": 5369.983571696282, "data_wait_seconds": 1.749396324157715e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 57970, "total_steps": 72479, "tokens": 30392975360, "loss": 4.191631495952606, "l_mix": 2.597447454929352, "l_a": 2.683129757642746, "l_c": 2.6308169066905975, "z_loss": 42.44171953201294, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.205642968416214, "tokens_per_second": 1398516.300906902, "eta_seconds": 5439.260584282874, "data_wait_seconds": 1.7136335372924805e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 57980, "total_steps": 72479, "tokens": 30398218240, "loss": 4.238213419914246, "l_mix": 2.625712901353836, "l_a": 2.7122633159160614, "l_c": 2.6627382338047028, "z_loss": 43.21691131591797, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20405741035938263, "tokens_per_second": 1401243.6690148348, "eta_seconds": 5424.93206577301, "data_wait_seconds": 1.65402889251709e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 57990, "total_steps": 72479, "tokens": 30403461120, "loss": 4.401484429836273, "l_mix": 2.728982985019684, "l_a": 2.812845081090927, "l_c": 2.7621596455574036, "z_loss": 40.312578201293945, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16197796165943146, "tokens_per_second": 1422679.2704443287, "eta_seconds": 5339.5090445280075, "data_wait_seconds": 1.5527009963989258e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 58000, "total_steps": 72479, "tokens": 30408704000, "loss": 4.221995711326599, "l_mix": 2.615706831216812, "l_a": 2.699659675359726, "l_c": 2.6546362936496735, "z_loss": 41.68753957748413, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.3242030441761017, "tokens_per_second": 1368222.2728252402, "eta_seconds": 5548.196446418762, "data_wait_seconds": 1.862645149230957e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16484752, "gate_a2b/L1_absmax": 0.69509441, "gate_a2b/L1_rms": 0.17546006, "gate_a2b/L3_absmean": 0.15743439, "gate_a2b/L3_absmax": 0.50980991, "gate_a2b/L3_rms": 0.163196, "gate_a2b/L5_absmean": 0.2330687, "gate_a2b/L5_absmax": 0.73299342, "gate_a2b/L5_rms": 0.23913012, "gate_c2b/L1_absmean": 0.03184788, "gate_c2b/L1_absmax": 0.63548118, "gate_c2b/L1_rms": 0.06883242, "gate_c2b/L3_absmean": 0.17296463, "gate_c2b/L3_absmax": 0.39228511, "gate_c2b/L3_rms": 0.1766541, "gate_c2b/L5_absmean": 0.49045047, "gate_c2b/L5_absmax": 0.69002318, "gate_c2b/L5_rms": 0.50025034, "gate_b2a/L7_absmean": 0.0503752, "gate_b2a/L7_absmax": 0.33614707, "gate_b2a/L7_rms": 0.05625397, "gate_b2a/L11_absmean": 0.08218011, "gate_b2a/L11_absmax": 0.611031, "gate_b2a/L11_rms": 0.09138234, "gate_b2a/L14_absmean": 0.2133342, "gate_b2a/L14_absmax": 0.82187027, "gate_b2a/L14_rms": 0.22265676, "gate_b2c/L3_absmean": 0.03491688, "gate_b2c/L3_absmax": 0.55360669, "gate_b2c/L3_rms": 0.04560233, "gate_b2c/L5_absmean": 0.0367138, "gate_b2c/L5_absmax": 0.52408338, "gate_b2c/L5_rms": 0.04581632, "gate_b2c/L6_absmean": 0.05318609, "gate_b2c/L6_absmax": 0.34324935, "gate_b2c/L6_rms": 0.05927619}, "streams": {"cos_a2b/round0": 0.359843, "cos_c2b/round0": 0.264073, "cos_b2a/round0": 0.156148, "cos_b2c/round0": 0.144401, "rms_a/round0": 7.39173, "rms_b/round0": 12.677999, "rms_c/round0": 4.804246, "cos_a2b/round1": 0.408542, "cos_c2b/round1": 0.299417, "cos_b2a/round1": 0.24461, "cos_b2c/round1": 0.108577, "rms_a/round1": 14.478196, "rms_b/round1": 29.697021, "rms_c/round1": 10.214348, "cos_a2b/round2": 0.528124, "cos_c2b/round2": 0.524915, "cos_b2a/round2": 0.094627, "cos_b2c/round2": 0.177139, "rms_a/round2": 40.757652, "rms_b/round2": 80.743927, "rms_c/round2": 19.60582}, "alpha_mean": 0.4316688036113874, "alpha_hist": [0.01053, 0.07581, 0.18668, 0.19751, 0.17114, 0.15253, 0.12529, 0.0696, 0.01083, 9e-05], "alpha_by_type": {"special": {"share": 0.00085, "mean": 0.55427, "std": 0.11631}, "whitespace": {"share": 0.02475, "mean": 0.5285, "std": 0.13056}, "digit": {"share": 0.04216, "mean": 0.49515, "std": 0.15855}, "word_start": {"share": 0.57294, "mean": 0.43695, "std": 0.17046}, "word_cont": {"share": 0.24638, "mean": 0.32521, "std": 0.12968}, "punct": {"share": 0.11292, "mean": 0.5913, "std": 0.15958}}, "alpha_p05": 0.18213, "alpha_p50": 0.42656, "alpha_p95": 0.73791}
{"step": 58010, "total_steps": 72479, "tokens": 30413946880, "loss": 3.1456253230571747, "l_mix": 1.9480922669172287, "l_a": 2.0186890959739685, "l_c": 1.9730875194072723, "z_loss": 51.736610412597656, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2984027564525604, "tokens_per_second": 1409398.2833800178, "eta_seconds": 5382.38421421051, "data_wait_seconds": 2.3245811462402344e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 58020, "total_steps": 72479, "tokens": 30419189760, "loss": 4.494583964347839, "l_mix": 2.787308633327484, "l_a": 2.8655819296836853, "l_c": 2.825335383415222, "z_loss": 41.11982250213623, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1531941294670105, "tokens_per_second": 1426545.3708651138, "eta_seconds": 5314.01268184185, "data_wait_seconds": 1.570582389831543e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 58030, "total_steps": 72479, "tokens": 30424432640, "loss": 4.587808847427368, "l_mix": 2.8460997939109802, "l_a": 2.923765778541565, "l_c": 2.8819305896759033, "z_loss": 39.88281726837158, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16376936435699463, "tokens_per_second": 1316342.9252865065, "eta_seconds": 5754.911707639694, "data_wait_seconds": 1.958012580871582e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 58040, "total_steps": 72479, "tokens": 30429675520, "loss": 4.6342432498931885, "l_mix": 2.874955326318741, "l_a": 2.953038811683655, "l_c": 2.9112540781497955, "z_loss": 40.43184566497803, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14219319820404053, "tokens_per_second": 1414066.0176984663, "eta_seconds": 5353.49434697628, "data_wait_seconds": 2.580881118774414e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 58050, "total_steps": 72479, "tokens": 30434918400, "loss": 4.630752116441727, "l_mix": 2.873747229576111, "l_a": 2.9474416971206665, "l_c": 2.9092412292957306, "z_loss": 39.28684329986572, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16315150260925293, "tokens_per_second": 1222763.7023025025, "eta_seconds": 6186.764898037911, "data_wait_seconds": 3.0159950256347656e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 58060, "total_steps": 72479, "tokens": 30440161280, "loss": 4.646895110607147, "l_mix": 2.8833028376102448, "l_a": 2.959459364414215, "l_c": 2.9191811978816986, "z_loss": 40.749170780181885, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15554356575012207, "tokens_per_second": 1394484.58045609, "eta_seconds": 5421.148987913131, "data_wait_seconds": 1.7434358596801758e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 58070, "total_steps": 72479, "tokens": 30445404160, "loss": 4.230923175811768, "l_mix": 2.621040999889374, "l_a": 2.708197683095932, "l_c": 2.6580757796764374, "z_loss": 42.46435260772705, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15539565682411194, "tokens_per_second": 1449831.5177459083, "eta_seconds": 5210.58185005188, "data_wait_seconds": 2.0325183868408203e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 58080, "total_steps": 72479, "tokens": 30450647040, "loss": 4.278794169425964, "l_mix": 2.651845932006836, "l_a": 2.735853433609009, "l_c": 2.6873069405555725, "z_loss": 41.48980140686035, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11996211111545563, "tokens_per_second": 1337574.5507764753, "eta_seconds": 5643.964224362374, "data_wait_seconds": 1.800060272216797e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 58090, "total_steps": 72479, "tokens": 30455889920, "loss": 4.308753460645676, "l_mix": 2.670903831720352, "l_a": 2.755583703517914, "l_c": 2.7039149403572083, "z_loss": 41.25764608383179, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17226482927799225, "tokens_per_second": 1450038.591819706, "eta_seconds": 5202.60638203621, "data_wait_seconds": 3.007054328918457e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 58100, "total_steps": 72479, "tokens": 30461132800, "loss": 4.294171988964081, "l_mix": 2.660791903734207, "l_a": 2.746840327978134, "l_c": 2.697759985923767, "z_loss": 40.83619403839111, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2428189516067505, "tokens_per_second": 1328703.1562632043, "eta_seconds": 5673.755734276771, "data_wait_seconds": 2.345442771911621e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16464911, "gate_a2b/L1_absmax": 0.69547319, "gate_a2b/L1_rms": 0.17526481, "gate_a2b/L3_absmean": 0.15750001, "gate_a2b/L3_absmax": 0.5054993, "gate_a2b/L3_rms": 0.16331388, "gate_a2b/L5_absmean": 0.23245077, "gate_a2b/L5_absmax": 0.73529208, "gate_a2b/L5_rms": 0.23854469, "gate_c2b/L1_absmean": 0.03279028, "gate_c2b/L1_absmax": 0.6338827, "gate_c2b/L1_rms": 0.06917546, "gate_c2b/L3_absmean": 0.17311032, "gate_c2b/L3_absmax": 0.39717197, "gate_c2b/L3_rms": 0.17685927, "gate_c2b/L5_absmean": 0.48965812, "gate_c2b/L5_absmax": 0.69001281, "gate_c2b/L5_rms": 0.49947923, "gate_b2a/L7_absmean": 0.05107881, "gate_b2a/L7_absmax": 0.33630788, "gate_b2a/L7_rms": 0.05694484, "gate_b2a/L11_absmean": 0.0829592, "gate_b2a/L11_absmax": 0.61266983, "gate_b2a/L11_rms": 0.0920654, "gate_b2a/L14_absmean": 0.21250531, "gate_b2a/L14_absmax": 0.81966072, "gate_b2a/L14_rms": 0.22204466, "gate_b2c/L3_absmean": 0.03442253, "gate_b2c/L3_absmax": 0.55770236, "gate_b2c/L3_rms": 0.04543836, "gate_b2c/L5_absmean": 0.03680588, "gate_b2c/L5_absmax": 0.52901489, "gate_b2c/L5_rms": 0.04606264, "gate_b2c/L6_absmean": 0.05296244, "gate_b2c/L6_absmax": 0.34349439, "gate_b2c/L6_rms": 0.05922066}, "streams": {"cos_a2b/round0": 0.358434, "cos_c2b/round0": 0.262085, "cos_b2a/round0": 0.159223, "cos_b2c/round0": 0.14513, "rms_a/round0": 7.42456, "rms_b/round0": 12.727426, "rms_c/round0": 4.806219, "cos_a2b/round1": 0.410818, "cos_c2b/round1": 0.303083, "cos_b2a/round1": 0.253566, "cos_b2c/round1": 0.117649, "rms_a/round1": 14.352905, "rms_b/round1": 29.704708, "rms_c/round1": 10.150329, "cos_a2b/round2": 0.531943, "cos_c2b/round2": 0.524361, "cos_b2a/round2": 0.097142, "cos_b2c/round2": 0.182342, "rms_a/round2": 40.694489, "rms_b/round2": 80.335243, "rms_c/round2": 19.572727}, "alpha_mean": 0.4336700506097628, "alpha_hist": [0.01353, 0.08125, 0.17274, 0.18889, 0.17534, 0.15909, 0.12918, 0.06914, 0.01071, 0.00012], "alpha_by_type": {"special": {"share": 0.0011, "mean": 0.60483, "std": 0.1085}, "whitespace": {"share": 0.028, "mean": 0.54615, "std": 0.12646}, "digit": {"share": 0.03285, "mean": 0.5138, "std": 0.11709}, "word_start": {"share": 0.58452, "mean": 0.43745, "std": 0.17565}, "word_cont": {"share": 0.24402, "mean": 0.33804, "std": 0.1428}, "punct": {"share": 0.10951, "mean": 0.57207, "std": 0.16788}}, "alpha_p05": 0.16804, "alpha_p50": 0.413, "alpha_p95": 0.71003}
{"step": 58110, "total_steps": 72479, "tokens": 30466375680, "loss": 4.341245651245117, "l_mix": 2.6905358135700226, "l_a": 2.778410851955414, "l_c": 2.7239551544189453, "z_loss": 41.76906394958496, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19214190542697906, "tokens_per_second": 1463348.2101404367, "eta_seconds": 5148.121424412728, "data_wait_seconds": 1.8805265426635742e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 58120, "total_steps": 72479, "tokens": 30471618560, "loss": 3.187901109457016, "l_mix": 1.9713762998580933, "l_a": 2.030196875333786, "l_c": 2.0248858630657196, "z_loss": 49.21254825592041, "lr": 0.0023, "wd": 0.01, "grad_norm": 1.6387033462524414, "tokens_per_second": 1396394.1759688247, "eta_seconds": 5391.207956576348, "data_wait_seconds": 2.4646520614624023e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 58130, "total_steps": 72479, "tokens": 30476861440, "loss": 3.1908476054668427, "l_mix": 1.9658776819705963, "l_a": 2.0699516236782074, "l_c": 2.013281524181366, "z_loss": 50.279695987701416, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.7318555116653442, "tokens_per_second": 1446987.7117575686, "eta_seconds": 5199.082515263558, "data_wait_seconds": 2.181529998779297e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 58140, "total_steps": 72479, "tokens": 30482104320, "loss": 4.426171183586121, "l_mix": 2.743246406316757, "l_a": 2.8281810581684113, "l_c": 2.781568229198456, "z_loss": 42.16770267486572, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14436857402324677, "tokens_per_second": 1379807.2616950918, "eta_seconds": 5448.417210650445, "data_wait_seconds": 1.77919864654541e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 58150, "total_steps": 72479, "tokens": 30487347200, "loss": 4.268846869468689, "l_mix": 2.646413654088974, "l_a": 2.7272569239139557, "l_c": 2.6808537542819977, "z_loss": 41.9866828918457, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20721711218357086, "tokens_per_second": 1456483.4024258172, "eta_seconds": 5157.987203621865, "data_wait_seconds": 1.5288591384887695e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 58160, "total_steps": 72479, "tokens": 30492590080, "loss": 4.282435655593872, "l_mix": 2.6537881791591644, "l_a": 2.737312912940979, "l_c": 2.6915116608142853, "z_loss": 40.59519052505493, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2420378029346466, "tokens_per_second": 1362256.4275658315, "eta_seconds": 5510.915360784531, "data_wait_seconds": 1.7315149307250977e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 58170, "total_steps": 72479, "tokens": 30497832960, "loss": 4.349708199501038, "l_mix": 2.696084976196289, "l_a": 2.7833986580371857, "l_c": 2.728678524494171, "z_loss": 40.150968074798584, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1781037598848343, "tokens_per_second": 1422950.8450109728, "eta_seconds": 5272.168759942055, "data_wait_seconds": 2.4974346160888672e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 58180, "total_steps": 72479, "tokens": 30503075840, "loss": 4.262637495994568, "l_mix": 2.6398282051086426, "l_a": 2.7331250607967377, "l_c": 2.676239103078842, "z_loss": 42.630735874176025, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14903494715690613, "tokens_per_second": 1449469.1370826047, "eta_seconds": 5172.096404266357, "data_wait_seconds": 1.6450881958007812e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 58190, "total_steps": 72479, "tokens": 30508318720, "loss": 4.301876902580261, "l_mix": 2.6654771864414215, "l_a": 2.7531598806381226, "l_c": 2.7015055418014526, "z_loss": 42.2736759185791, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1061495915055275, "tokens_per_second": 1469497.428875807, "eta_seconds": 5098.036297845841, "data_wait_seconds": 1.9609928131103516e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 58200, "total_steps": 72479, "tokens": 30513561600, "loss": 4.240493297576904, "l_mix": 2.6271989345550537, "l_a": 2.714667499065399, "l_c": 2.662980020046234, "z_loss": 43.11675691604614, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11945751309394836, "tokens_per_second": 1424726.3637882057, "eta_seconds": 5254.558729505538, "data_wait_seconds": 1.6301870346069336e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16484207, "gate_a2b/L1_absmax": 0.71118098, "gate_a2b/L1_rms": 0.17578432, "gate_a2b/L3_absmean": 0.15620737, "gate_a2b/L3_absmax": 0.50894523, "gate_a2b/L3_rms": 0.16207245, "gate_a2b/L5_absmean": 0.23200618, "gate_a2b/L5_absmax": 0.7414183, "gate_a2b/L5_rms": 0.23815215, "gate_c2b/L1_absmean": 0.03279774, "gate_c2b/L1_absmax": 0.64476216, "gate_c2b/L1_rms": 0.0698979, "gate_c2b/L3_absmean": 0.17349385, "gate_c2b/L3_absmax": 0.39505619, "gate_c2b/L3_rms": 0.17712572, "gate_c2b/L5_absmean": 0.48894292, "gate_c2b/L5_absmax": 0.69250011, "gate_c2b/L5_rms": 0.49869561, "gate_b2a/L7_absmean": 0.05086586, "gate_b2a/L7_absmax": 0.33713758, "gate_b2a/L7_rms": 0.0568324, "gate_b2a/L11_absmean": 0.08262596, "gate_b2a/L11_absmax": 0.60858357, "gate_b2a/L11_rms": 0.09180226, "gate_b2a/L14_absmean": 0.21304742, "gate_b2a/L14_absmax": 0.8249284, "gate_b2a/L14_rms": 0.22272147, "gate_b2c/L3_absmean": 0.03470423, "gate_b2c/L3_absmax": 0.5583778, "gate_b2c/L3_rms": 0.04556955, "gate_b2c/L5_absmean": 0.03664396, "gate_b2c/L5_absmax": 0.52355385, "gate_b2c/L5_rms": 0.04580861, "gate_b2c/L6_absmean": 0.05330155, "gate_b2c/L6_absmax": 0.34527695, "gate_b2c/L6_rms": 0.05957248}, "streams": {"cos_a2b/round0": 0.360298, "cos_c2b/round0": 0.260499, "cos_b2a/round0": 0.153305, "cos_b2c/round0": 0.146645, "rms_a/round0": 7.523733, "rms_b/round0": 12.75312, "rms_c/round0": 4.850531, "cos_a2b/round1": 0.411011, "cos_c2b/round1": 0.303051, "cos_b2a/round1": 0.248997, "cos_b2c/round1": 0.113622, "rms_a/round1": 14.619183, "rms_b/round1": 29.741114, "rms_c/round1": 10.146057, "cos_a2b/round2": 0.53823, "cos_c2b/round2": 0.521466, "cos_b2a/round2": 0.099468, "cos_b2c/round2": 0.185003, "rms_a/round2": 41.571865, "rms_b/round2": 81.387123, "rms_c/round2": 19.292109}, "alpha_mean": 0.4307192217707012, "alpha_hist": [0.01129, 0.07315, 0.17482, 0.20386, 0.18788, 0.15822, 0.11734, 0.06326, 0.01013, 5e-05], "alpha_by_type": {"special": {"share": 0.00092, "mean": 0.54322, "std": 0.08667}, "whitespace": {"share": 0.02711, "mean": 0.52644, "std": 0.12024}, "digit": {"share": 0.06194, "mean": 0.44374, "std": 0.10944}, "word_start": {"share": 0.54079, "mean": 0.44498, "std": 0.16989}, "word_cont": {"share": 0.25731, "mean": 0.31699, "std": 0.12544}, "punct": {"share": 0.11194, "mean": 0.59193, "std": 0.14415}}, "alpha_p05": 0.17526, "alpha_p50": 0.44818, "alpha_p95": 0.69966}
{"step": 58210, "total_steps": 72479, "tokens": 30518804480, "loss": 4.306841850280762, "l_mix": 2.668868124485016, "l_a": 2.7572284638881683, "l_c": 2.7026838064193726, "z_loss": 40.972477436065674, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14626625180244446, "tokens_per_second": 1347886.0512934176, "eta_seconds": 5550.221003341675, "data_wait_seconds": 1.9103288650512695e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 58220, "total_steps": 72479, "tokens": 30524047360, "loss": 4.331435799598694, "l_mix": 2.684652179479599, "l_a": 2.7683537900447845, "l_c": 2.720924824476242, "z_loss": 40.83877992630005, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17862723767757416, "tokens_per_second": 1455165.9801752772, "eta_seconds": 5137.436343240738, "data_wait_seconds": 2.2411346435546875e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 58230, "total_steps": 72479, "tokens": 30529290240, "loss": 4.221932768821716, "l_mix": 2.6156162917613983, "l_a": 2.704152911901474, "l_c": 2.6502350568771362, "z_loss": 42.219926834106445, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13000212609767914, "tokens_per_second": 1432561.7487205125, "eta_seconds": 5214.839582777023, "data_wait_seconds": 3.0934810638427734e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 58240, "total_steps": 72479, "tokens": 30534533120, "loss": 2.7619932889938354, "l_mix": 1.7101668119430542, "l_a": 1.7696960866451263, "l_c": 1.7363921850919724, "z_loss": 50.1633677482605, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16231217980384827, "tokens_per_second": 1419717.75066921, "eta_seconds": 5258.324641275406, "data_wait_seconds": 2.5480985641479492e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 58250, "total_steps": 72479, "tokens": 30539776000, "loss": 4.2703863978385925, "l_mix": 2.6458373069763184, "l_a": 2.7340683043003082, "l_c": 2.681094855070114, "z_loss": 41.60109853744507, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14070697128772736, "tokens_per_second": 1329102.5259401926, "eta_seconds": 5612.880726957321, "data_wait_seconds": 1.5944242477416992e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 58260, "total_steps": 72479, "tokens": 30545018880, "loss": 4.325750172138214, "l_mix": 2.6799393594264984, "l_a": 2.768705815076828, "l_c": 2.717329829931259, "z_loss": 42.45510196685791, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1548466682434082, "tokens_per_second": 1430141.4540957413, "eta_seconds": 5212.666936302186, "data_wait_seconds": 1.5437602996826172e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 58270, "total_steps": 72479, "tokens": 30550261760, "loss": 4.358995199203491, "l_mix": 2.7020501792430878, "l_a": 2.7857232987880707, "l_c": 2.737426370382309, "z_loss": 42.122878074645996, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.24093101918697357, "tokens_per_second": 1391101.7089886488, "eta_seconds": 5355.185852956772, "data_wait_seconds": 2.5898218154907227e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 58280, "total_steps": 72479, "tokens": 30555504640, "loss": 4.33333295583725, "l_mix": 2.68498158454895, "l_a": 2.7712225914001465, "l_c": 2.7232819497585297, "z_loss": 41.641671657562256, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16762606799602509, "tokens_per_second": 1438601.0426979994, "eta_seconds": 5174.725369334221, "data_wait_seconds": 1.9788742065429688e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 58290, "total_steps": 72479, "tokens": 30560747520, "loss": 4.291340947151184, "l_mix": 2.6601644456386566, "l_a": 2.7442691326141357, "l_c": 2.6929852962493896, "z_loss": 41.301820278167725, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16406100988388062, "tokens_per_second": 1464464.245541506, "eta_seconds": 5079.756951832772, "data_wait_seconds": 2.0056962966918945e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 58300, "total_steps": 72479, "tokens": 30565990400, "loss": 4.560929238796234, "l_mix": 2.8291202783584595, "l_a": 2.9075736105442047, "l_c": 2.865122675895691, "z_loss": 40.91610813140869, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1083335354924202, "tokens_per_second": 1421096.2614525803, "eta_seconds": 5231.087966132163, "data_wait_seconds": 1.9311904907226562e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.16481991, "gate_a2b/L1_absmax": 0.70486057, "gate_a2b/L1_rms": 0.17560066, "gate_a2b/L3_absmean": 0.15672968, "gate_a2b/L3_absmax": 0.50604743, "gate_a2b/L3_rms": 0.16248125, "gate_a2b/L5_absmean": 0.23133229, "gate_a2b/L5_absmax": 0.7368688, "gate_a2b/L5_rms": 0.23737024, "gate_c2b/L1_absmean": 0.03243954, "gate_c2b/L1_absmax": 0.64038283, "gate_c2b/L1_rms": 0.06968018, "gate_c2b/L3_absmean": 0.1731544, "gate_c2b/L3_absmax": 0.39219597, "gate_c2b/L3_rms": 0.176809, "gate_c2b/L5_absmean": 0.48964825, "gate_c2b/L5_absmax": 0.68852019, "gate_c2b/L5_rms": 0.49942139, "gate_b2a/L7_absmean": 0.05122148, "gate_b2a/L7_absmax": 0.33582819, "gate_b2a/L7_rms": 0.05708665, "gate_b2a/L11_absmean": 0.08242846, "gate_b2a/L11_absmax": 0.60668457, "gate_b2a/L11_rms": 0.09150273, "gate_b2a/L14_absmean": 0.21384856, "gate_b2a/L14_absmax": 0.82219893, "gate_b2a/L14_rms": 0.2233175, "gate_b2c/L3_absmean": 0.03468633, "gate_b2c/L3_absmax": 0.55515844, "gate_b2c/L3_rms": 0.0454568, "gate_b2c/L5_absmean": 0.03680435, "gate_b2c/L5_absmax": 0.52047837, "gate_b2c/L5_rms": 0.0457851, "gate_b2c/L6_absmean": 0.05311453, "gate_b2c/L6_absmax": 0.34456128, "gate_b2c/L6_rms": 0.05929342}, "streams": {"cos_a2b/round0": 0.355292, "cos_c2b/round0": 0.260228, "cos_b2a/round0": 0.162293, "cos_b2c/round0": 0.145566, "rms_a/round0": 7.360276, "rms_b/round0": 12.788126, "rms_c/round0": 4.726923, "cos_a2b/round1": 0.414799, "cos_c2b/round1": 0.312302, "cos_b2a/round1": 0.257231, "cos_b2c/round1": 0.118634, "rms_a/round1": 14.366145, "rms_b/round1": 29.199587, "rms_c/round1": 10.157145, "cos_a2b/round2": 0.547038, "cos_c2b/round2": 0.520518, "cos_b2a/round2": 0.109363, "cos_b2c/round2": 0.188741, "rms_a/round2": 40.829842, "rms_b/round2": 79.279205, "rms_c/round2": 19.426119}, "alpha_mean": 0.45293293521287126, "alpha_hist": [0.00603, 0.05196, 0.15671, 0.18803, 0.18945, 0.19431, 0.13477, 0.07043, 0.00829, 3e-05], "alpha_by_type": {"special": {"share": 0.00053, "mean": 0.50689, "std": 0.12427}, "whitespace": {"share": 0.06323, "mean": 0.53576, "std": 0.119}, "digit": {"share": 0.02924, "mean": 0.43438, "std": 0.10017}, "word_start": {"share": 0.56224, "mean": 0.47193, "std": 0.1572}, "word_cont": {"share": 0.24629, "mean": 0.33337, "std": 0.1306}, "punct": {"share": 0.09846, "mean": 0.59555, "std": 0.15945}}, "alpha_p05": 0.18654, "alpha_p50": 0.44589, "alpha_p95": 0.73087}
{"step": 58310, "total_steps": 72479, "tokens": 30571233280, "loss": 4.698920369148254, "l_mix": 2.9158456325531006, "l_a": 2.992631196975708, "l_c": 2.9509511590003967, "z_loss": 39.51423406600952, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.129469633102417, "tokens_per_second": 1439192.0344554244, "eta_seconds": 5161.671614456177, "data_wait_seconds": 2.8789043426513672e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 58320, "total_steps": 72479, "tokens": 30576476160, "loss": 4.617957770824432, "l_mix": 2.864903539419174, "l_a": 2.9420009553432465, "l_c": 2.9015128016471863, "z_loss": 41.07583808898926, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14261731505393982, "tokens_per_second": 1462584.378255571, "eta_seconds": 5075.531984591484, "data_wait_seconds": 1.633167266845703e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 58330, "total_steps": 72479, "tokens": 30581719040, "loss": 4.705507338047028, "l_mix": 2.9190106987953186, "l_a": 2.997882217168808, "l_c": 2.957106351852417, "z_loss": 40.33142042160034, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14061900973320007, "tokens_per_second": 1446105.7130580451, "eta_seconds": 5129.7431750774385, "data_wait_seconds": 2.5540590286254883e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 58340, "total_steps": 72479, "tokens": 30586961920, "loss": 4.630167841911316, "l_mix": 2.872854769229889, "l_a": 2.950783908367157, "l_c": 2.9069261252880096, "z_loss": 41.03641748428345, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15941517055034637, "tokens_per_second": 1371335.1428147196, "eta_seconds": 5405.6136976003645, "data_wait_seconds": 2.3186206817626953e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 58350, "total_steps": 72479, "tokens": 30592204800, "loss": 2.8788512647151947, "l_mix": 1.7817268073558807, "l_a": 1.8488274663686752, "l_c": 1.8082539588212967, "z_loss": 50.33702993392944, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15383942425251007, "tokens_per_second": 1455985.7999200043, "eta_seconds": 5087.731729531289, "data_wait_seconds": 2.1129846572875977e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 58360, "total_steps": 72479, "tokens": 30597447680, "loss": 4.402216583490372, "l_mix": 2.728839486837387, "l_a": 2.8145030736923218, "l_c": 2.763420343399048, "z_loss": 40.253650188446045, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1329365372657776, "tokens_per_second": 1184028.0853186862, "eta_seconds": 6251.897538399697, "data_wait_seconds": 1.84476375579834e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 58370, "total_steps": 72479, "tokens": 30602690560, "loss": 4.311175584793091, "l_mix": 2.6713352501392365, "l_a": 2.7574371695518494, "l_c": 2.708696871995926, "z_loss": 41.97084188461304, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13329152762889862, "tokens_per_second": 1475144.9792921753, "eta_seconds": 5014.5439911603935, "data_wait_seconds": 2.0176172256469727e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 58380, "total_steps": 72479, "tokens": 30607933440, "loss": 4.308774292469025, "l_mix": 2.6706089079380035, "l_a": 2.752455711364746, "l_c": 2.7080950140953064, "z_loss": 41.8564887046814, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12905749678611755, "tokens_per_second": 1350685.593479384, "eta_seconds": 5472.729218173027, "data_wait_seconds": 1.9937753677368164e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 58390, "total_steps": 72479, "tokens": 30613176320, "loss": 4.3591039180755615, "l_mix": 2.701733022928238, "l_a": 2.787428468465805, "l_c": 2.7371409833431244, "z_loss": 42.498342514038086, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2515685558319092, "tokens_per_second": 1342285.1773131422, "eta_seconds": 5503.0732342481615, "data_wait_seconds": 2.333521842956543e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 58400, "total_steps": 72479, "tokens": 30618419200, "loss": 4.344589471817017, "l_mix": 2.692995637655258, "l_a": 2.7748221457004547, "l_c": 2.730490356683731, "z_loss": 41.522921085357666, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1514361947774887, "tokens_per_second": 1417823.1284365244, "eta_seconds": 5206.185880279541, "data_wait_seconds": 3.832578659057617e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.16461109, "gate_a2b/L1_absmax": 0.70369816, "gate_a2b/L1_rms": 0.1753509, "gate_a2b/L3_absmean": 0.15680814, "gate_a2b/L3_absmax": 0.50452715, "gate_a2b/L3_rms": 0.16256291, "gate_a2b/L5_absmean": 0.2312337, "gate_a2b/L5_absmax": 0.73659432, "gate_a2b/L5_rms": 0.23730223, "gate_c2b/L1_absmean": 0.0322575, "gate_c2b/L1_absmax": 0.63909775, "gate_c2b/L1_rms": 0.06953158, "gate_c2b/L3_absmean": 0.17293303, "gate_c2b/L3_absmax": 0.39256293, "gate_c2b/L3_rms": 0.17665796, "gate_c2b/L5_absmean": 0.48954344, "gate_c2b/L5_absmax": 0.6887747, "gate_c2b/L5_rms": 0.49935782, "gate_b2a/L7_absmean": 0.05082645, "gate_b2a/L7_absmax": 0.33581969, "gate_b2a/L7_rms": 0.05674354, "gate_b2a/L11_absmean": 0.0822653, "gate_b2a/L11_absmax": 0.60691077, "gate_b2a/L11_rms": 0.09134495, "gate_b2a/L14_absmean": 0.21369252, "gate_b2a/L14_absmax": 0.82266498, "gate_b2a/L14_rms": 0.22315994, "gate_b2c/L3_absmean": 0.0347325, "gate_b2c/L3_absmax": 0.5552994, "gate_b2c/L3_rms": 0.04549261, "gate_b2c/L5_absmean": 0.03681457, "gate_b2c/L5_absmax": 0.52119678, "gate_b2c/L5_rms": 0.04581649, "gate_b2c/L6_absmean": 0.05302685, "gate_b2c/L6_absmax": 0.34355876, "gate_b2c/L6_rms": 0.05918496}, "streams": {"cos_a2b/round0": 0.360295, "cos_c2b/round0": 0.263708, "cos_b2a/round0": 0.148819, "cos_b2c/round0": 0.145813, "rms_a/round0": 7.527689, "rms_b/round0": 12.872085, "rms_c/round0": 4.81184, "cos_a2b/round1": 0.409539, "cos_c2b/round1": 0.293891, "cos_b2a/round1": 0.246035, "cos_b2c/round1": 0.109553, "rms_a/round1": 14.73837, "rms_b/round1": 30.185909, "rms_c/round1": 10.238235, "cos_a2b/round2": 0.532081, "cos_c2b/round2": 0.521522, "cos_b2a/round2": 0.095378, "cos_b2c/round2": 0.176772, "rms_a/round2": 41.779125, "rms_b/round2": 82.17907, "rms_c/round2": 19.85743}, "alpha_mean": 0.4259720957088149, "alpha_hist": [0.00793, 0.07301, 0.18762, 0.21165, 0.18318, 0.15334, 0.11526, 0.05807, 0.00964, 0.00027], "alpha_by_type": {"special": {"share": 0.00096, "mean": 0.5463, "std": 0.12271}, "whitespace": {"share": 0.02893, "mean": 0.52575, "std": 0.13342}, "digit": {"share": 0.05896, "mean": 0.46847, "std": 0.11976}, "word_start": {"share": 0.53423, "mean": 0.43404, "std": 0.16842}, "word_cont": {"share": 0.26093, "mean": 0.32266, "std": 0.12078}, "punct": {"share": 0.116, "mean": 0.57371, "std": 0.1579}}, "alpha_p05": 0.18243, "alpha_p50": 0.40899, "alpha_p95": 0.70579}
{"step": 58410, "total_steps": 72479, "tokens": 30623662080, "loss": 2.863697588443756, "l_mix": 1.7734459340572357, "l_a": 1.833955556154251, "l_c": 1.8002163618803024, "z_loss": 51.54527568817139, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14979992806911469, "tokens_per_second": 1344545.4986278934, "eta_seconds": 5486.023254346847, "data_wait_seconds": 2.8640031814575195e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 58420, "total_steps": 72479, "tokens": 30628904960, "loss": 4.663980543613434, "l_mix": 2.8929868936538696, "l_a": 2.9747928082942963, "l_c": 2.928519070148468, "z_loss": 40.50240421295166, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19732224941253662, "tokens_per_second": 1429461.2259343257, "eta_seconds": 5156.463748908043, "data_wait_seconds": 2.321600914001465e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 58430, "total_steps": 72479, "tokens": 30634147840, "loss": 4.727299153804779, "l_mix": 2.9330569207668304, "l_a": 3.0096654295921326, "l_c": 2.97114160656929, "z_loss": 40.83169364929199, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1584063023328781, "tokens_per_second": 1470944.4424018965, "eta_seconds": 5007.47812063694, "data_wait_seconds": 2.4318695068359375e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 58440, "total_steps": 72479, "tokens": 30639390720, "loss": 4.444726914167404, "l_mix": 2.7584019601345062, "l_a": 2.8298285752534866, "l_c": 2.7912544310092926, "z_loss": 40.83516073226929, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17175167798995972, "tokens_per_second": 1470045.6829426433, "eta_seconds": 5006.973128390312, "data_wait_seconds": 2.6881694793701172e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 58450, "total_steps": 72479, "tokens": 30644633600, "loss": 4.672925412654877, "l_mix": 2.8987125158309937, "l_a": 2.975768893957138, "l_c": 2.9382736682891846, "z_loss": 40.54841470718384, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18702884018421173, "tokens_per_second": 1418113.0625610824, "eta_seconds": 5186.636063218117, "data_wait_seconds": 2.136826515197754e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 58460, "total_steps": 72479, "tokens": 30649876480, "loss": 2.8979812562465668, "l_mix": 1.7940358221530914, "l_a": 1.859377607703209, "l_c": 1.8204402327537537, "z_loss": 51.69189453125, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1505153626203537, "tokens_per_second": 1445673.527868925, "eta_seconds": 5084.130912208557, "data_wait_seconds": 2.142786979675293e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 58470, "total_steps": 72479, "tokens": 30655119360, "loss": 4.45137894153595, "l_mix": 2.7596258521080017, "l_a": 2.842848151922226, "l_c": 2.7963282763957977, "z_loss": 41.529181480407715, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12393894791603088, "tokens_per_second": 1413938.7264668334, "eta_seconds": 5194.532446503639, "data_wait_seconds": 1.5795230865478516e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 58480, "total_steps": 72479, "tokens": 30660362240, "loss": 4.519026815891266, "l_mix": 2.802047699689865, "l_a": 2.883566439151764, "l_c": 2.839696705341339, "z_loss": 40.89220094680786, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1326327621936798, "tokens_per_second": 1425370.678699373, "eta_seconds": 5149.192291998863, "data_wait_seconds": 1.683831214904785e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 58490, "total_steps": 72479, "tokens": 30665605120, "loss": 4.490845322608948, "l_mix": 2.7849509716033936, "l_a": 2.8677627444267273, "l_c": 2.8185514509677887, "z_loss": 41.301655292510986, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1655345857143402, "tokens_per_second": 1444656.9252349662, "eta_seconds": 5076.821149635315, "data_wait_seconds": 1.5914440155029297e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 58500, "total_steps": 72479, "tokens": 30670848000, "loss": 4.376392364501953, "l_mix": 2.712521404027939, "l_a": 2.7961899638175964, "l_c": 2.750046491622925, "z_loss": 42.47651720046997, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1256168782711029, "tokens_per_second": 1375790.433328737, "eta_seconds": 5327.1354229927065, "data_wait_seconds": 1.6421079635620117e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16424863, "gate_a2b/L1_absmax": 0.7021814, "gate_a2b/L1_rms": 0.17488505, "gate_a2b/L3_absmean": 0.15682332, "gate_a2b/L3_absmax": 0.50381148, "gate_a2b/L3_rms": 0.16257843, "gate_a2b/L5_absmean": 0.23157935, "gate_a2b/L5_absmax": 0.73497736, "gate_a2b/L5_rms": 0.23759587, "gate_c2b/L1_absmean": 0.03266545, "gate_c2b/L1_absmax": 0.63789397, "gate_c2b/L1_rms": 0.06936099, "gate_c2b/L3_absmean": 0.17288971, "gate_c2b/L3_absmax": 0.38938633, "gate_c2b/L3_rms": 0.17656949, "gate_c2b/L5_absmean": 0.48905355, "gate_c2b/L5_absmax": 0.69110972, "gate_c2b/L5_rms": 0.49881366, "gate_b2a/L7_absmean": 0.05043679, "gate_b2a/L7_absmax": 0.33519718, "gate_b2a/L7_rms": 0.05633961, "gate_b2a/L11_absmean": 0.08211266, "gate_b2a/L11_absmax": 0.60541791, "gate_b2a/L11_rms": 0.09121, "gate_b2a/L14_absmean": 0.21378344, "gate_b2a/L14_absmax": 0.82127458, "gate_b2a/L14_rms": 0.22324575, "gate_b2c/L3_absmean": 0.03479482, "gate_b2c/L3_absmax": 0.55395365, "gate_b2c/L3_rms": 0.04544658, "gate_b2c/L5_absmean": 0.03675617, "gate_b2c/L5_absmax": 0.5229162, "gate_b2c/L5_rms": 0.04584228, "gate_b2c/L6_absmean": 0.05322241, "gate_b2c/L6_absmax": 0.34384859, "gate_b2c/L6_rms": 0.05935663}, "streams": {"cos_a2b/round0": 0.355842, "cos_c2b/round0": 0.260704, "cos_b2a/round0": 0.157402, "cos_b2c/round0": 0.146006, "rms_a/round0": 7.500351, "rms_b/round0": 12.877024, "rms_c/round0": 4.816991, "cos_a2b/round1": 0.413651, "cos_c2b/round1": 0.306785, "cos_b2a/round1": 0.24611, "cos_b2c/round1": 0.114423, "rms_a/round1": 14.751727, "rms_b/round1": 29.805677, "rms_c/round1": 10.329547, "cos_a2b/round2": 0.537688, "cos_c2b/round2": 0.524968, "cos_b2a/round2": 0.099702, "cos_b2c/round2": 0.184012, "rms_a/round2": 41.457909, "rms_b/round2": 80.794693, "rms_c/round2": 19.829464}, "alpha_mean": 0.43587535120944665, "alpha_hist": [0.01102, 0.07251, 0.16718, 0.19203, 0.18298, 0.17789, 0.12915, 0.05952, 0.00764, 8e-05], "alpha_by_type": {"special": {"share": 0.00133, "mean": 0.52252, "std": 0.11442}, "whitespace": {"share": 0.03201, "mean": 0.50981, "std": 0.11398}, "digit": {"share": 0.02745, "mean": 0.44874, "std": 0.11624}, "word_start": {"share": 0.55919, "mean": 0.45416, "std": 0.16384}, "word_cont": {"share": 0.26404, "mean": 0.32274, "std": 0.13192}, "punct": {"share": 0.11598, "mean": 0.5808, "std": 0.15206}}, "alpha_p05": 0.16106, "alpha_p50": 0.41965, "alpha_p95": 0.72971}
{"step": 58510, "total_steps": 72479, "tokens": 30676090880, "loss": 4.349063694477081, "l_mix": 2.696134567260742, "l_a": 2.77688205242157, "l_c": 2.7328815162181854, "z_loss": 42.30082368850708, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16734999418258667, "tokens_per_second": 1281334.9093481163, "eta_seconds": 5715.741465067864, "data_wait_seconds": 1.7970800399780273e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 58520, "total_steps": 72479, "tokens": 30681333760, "loss": 2.9390311539173126, "l_mix": 1.8189698457717896, "l_a": 1.8883041739463806, "l_c": 1.8452332466840744, "z_loss": 51.821120738983154, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17225703597068787, "tokens_per_second": 1292286.7607191526, "eta_seconds": 5663.244733643532, "data_wait_seconds": 1.704692840576172e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 58530, "total_steps": 72479, "tokens": 30686576640, "loss": 2.913156747817993, "l_mix": 1.8033425509929657, "l_a": 1.8681606650352478, "l_c": 1.8312198221683502, "z_loss": 50.28214931488037, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15164248645305634, "tokens_per_second": 1440607.6978159794, "eta_seconds": 5076.533551144599, "data_wait_seconds": 3.409385681152344e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 58540, "total_steps": 72479, "tokens": 30691819520, "loss": 4.663418412208557, "l_mix": 2.8937049210071564, "l_a": 2.9694004952907562, "l_c": 2.9296445548534393, "z_loss": 39.938621044158936, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12246581166982651, "tokens_per_second": 1468618.5811825267, "eta_seconds": 4976.139159369469, "data_wait_seconds": 1.6957521438598633e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 58550, "total_steps": 72479, "tokens": 30697062400, "loss": 4.557370126247406, "l_mix": 2.8263524174690247, "l_a": 2.90761399269104, "l_c": 2.862444907426834, "z_loss": 40.76152992248535, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18984416127204895, "tokens_per_second": 1468522.565425027, "eta_seconds": 4972.894338798523, "data_wait_seconds": 2.1219253540039062e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 58560, "total_steps": 72479, "tokens": 30702305280, "loss": 4.704323887825012, "l_mix": 2.9196533858776093, "l_a": 2.9942753314971924, "l_c": 2.9546258747577667, "z_loss": 40.45806121826172, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18186432123184204, "tokens_per_second": 1390749.4419365402, "eta_seconds": 5247.217400884629, "data_wait_seconds": 2.3990869522094727e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 58570, "total_steps": 72479, "tokens": 30707548160, "loss": 4.613979458808899, "l_mix": 2.8619083762168884, "l_a": 2.938500612974167, "l_c": 2.9017362892627716, "z_loss": 40.089120388031006, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2572644054889679, "tokens_per_second": 1412721.7982222892, "eta_seconds": 5161.895145368577, "data_wait_seconds": 2.4527311325073242e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 58580, "total_steps": 72479, "tokens": 30712791040, "loss": 4.423467934131622, "l_mix": 2.7428936660289764, "l_a": 2.8245333433151245, "l_c": 2.777380734682083, "z_loss": 41.52665376663208, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15495868027210236, "tokens_per_second": 1402296.014193968, "eta_seconds": 5196.5339972734455, "data_wait_seconds": 1.621246337890625e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 58590, "total_steps": 72479, "tokens": 30718033920, "loss": 4.498388171195984, "l_mix": 2.7885380685329437, "l_a": 2.8733301758766174, "l_c": 2.8261699974536896, "z_loss": 43.044968128204346, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19054192304611206, "tokens_per_second": 1364810.442031298, "eta_seconds": 5335.419343042374, "data_wait_seconds": 2.7805566787719727e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 58600, "total_steps": 72479, "tokens": 30723276800, "loss": 4.538014233112335, "l_mix": 2.8131677210330963, "l_a": 2.8991958796977997, "l_c": 2.850292682647705, "z_loss": 41.93077230453491, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14028505980968475, "tokens_per_second": 1363142.924025329, "eta_seconds": 5338.099933433533, "data_wait_seconds": 1.683831214904785e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16454627, "gate_a2b/L1_absmax": 0.69874036, "gate_a2b/L1_rms": 0.17511009, "gate_a2b/L3_absmean": 0.15689354, "gate_a2b/L3_absmax": 0.50238824, "gate_a2b/L3_rms": 0.16261628, "gate_a2b/L5_absmean": 0.23133884, "gate_a2b/L5_absmax": 0.73462844, "gate_a2b/L5_rms": 0.23740108, "gate_c2b/L1_absmean": 0.03309378, "gate_c2b/L1_absmax": 0.6359939, "gate_c2b/L1_rms": 0.06925427, "gate_c2b/L3_absmean": 0.1723775, "gate_c2b/L3_absmax": 0.39196196, "gate_c2b/L3_rms": 0.17602295, "gate_c2b/L5_absmean": 0.48850414, "gate_c2b/L5_absmax": 0.69041789, "gate_c2b/L5_rms": 0.49825776, "gate_b2a/L7_absmean": 0.05074276, "gate_b2a/L7_absmax": 0.3346, "gate_b2a/L7_rms": 0.05654496, "gate_b2a/L11_absmean": 0.08188915, "gate_b2a/L11_absmax": 0.60743642, "gate_b2a/L11_rms": 0.09103727, "gate_b2a/L14_absmean": 0.21394692, "gate_b2a/L14_absmax": 0.82290244, "gate_b2a/L14_rms": 0.22338311, "gate_b2c/L3_absmean": 0.03506052, "gate_b2c/L3_absmax": 0.55390978, "gate_b2c/L3_rms": 0.04566056, "gate_b2c/L5_absmean": 0.03668206, "gate_b2c/L5_absmax": 0.52652621, "gate_b2c/L5_rms": 0.04582569, "gate_b2c/L6_absmean": 0.05327722, "gate_b2c/L6_absmax": 0.34566095, "gate_b2c/L6_rms": 0.05939791}, "streams": {"cos_a2b/round0": 0.356301, "cos_c2b/round0": 0.25966, "cos_b2a/round0": 0.159972, "cos_b2c/round0": 0.144965, "rms_a/round0": 7.419702, "rms_b/round0": 12.856535, "rms_c/round0": 4.824025, "cos_a2b/round1": 0.418605, "cos_c2b/round1": 0.305626, "cos_b2a/round1": 0.255717, "cos_b2c/round1": 0.11414, "rms_a/round1": 14.491654, "rms_b/round1": 29.539665, "rms_c/round1": 10.165867, "cos_a2b/round2": 0.540423, "cos_c2b/round2": 0.523221, "cos_b2a/round2": 0.098026, "cos_b2c/round2": 0.187026, "rms_a/round2": 40.81707, "rms_b/round2": 80.050453, "rms_c/round2": 19.588142}, "alpha_mean": 0.43247760464967655, "alpha_hist": [0.00723, 0.06357, 0.1851, 0.2132, 0.18385, 0.15376, 0.11629, 0.06676, 0.01013, 0.00011], "alpha_by_type": {"special": {"share": 0.00096, "mean": 0.55473, "std": 0.12868}, "whitespace": {"share": 0.0235, "mean": 0.50236, "std": 0.12017}, "digit": {"share": 0.02577, "mean": 0.41773, "std": 0.11785}, "word_start": {"share": 0.59377, "mean": 0.45042, "std": 0.1643}, "word_cont": {"share": 0.25343, "mean": 0.32026, "std": 0.12068}, "punct": {"share": 0.10257, "mean": 0.59241, "std": 0.1567}}, "alpha_p05": 0.18099, "alpha_p50": 0.40993, "alpha_p95": 0.71383}
{"step": 58610, "total_steps": 72479, "tokens": 30728519680, "loss": 4.5067548751831055, "l_mix": 2.7939091622829437, "l_a": 2.875345081090927, "l_c": 2.834140181541443, "z_loss": 41.6111626625061, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12192647904157639, "tokens_per_second": 1313491.6297314018, "eta_seconds": 5535.893878126145, "data_wait_seconds": 1.6182661056518555e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 58620, "total_steps": 72479, "tokens": 30733762560, "loss": 4.350800514221191, "l_mix": 2.69631764292717, "l_a": 2.7825603783130646, "l_c": 2.7323824167251587, "z_loss": 41.97012233734131, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1414472460746765, "tokens_per_second": 1384309.3222848277, "eta_seconds": 5248.904471731186, "data_wait_seconds": 1.9818544387817383e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 58630, "total_steps": 72479, "tokens": 30739005440, "loss": 3.070810616016388, "l_mix": 1.901643767952919, "l_a": 1.9685162156820297, "l_c": 1.928706556558609, "z_loss": 49.82394504547119, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14767716825008392, "tokens_per_second": 1475572.689640623, "eta_seconds": 4920.7094729900355, "data_wait_seconds": 2.2977590560913086e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 58640, "total_steps": 72479, "tokens": 30744248320, "loss": 2.8004259765148163, "l_mix": 1.7336098402738571, "l_a": 1.7973778396844864, "l_c": 1.7586759328842163, "z_loss": 51.66066837310791, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1534661054611206, "tokens_per_second": 1474269.3480734574, "eta_seconds": 4921.503415560723, "data_wait_seconds": 2.1219253540039062e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 58650, "total_steps": 72479, "tokens": 30749491200, "loss": 4.50268417596817, "l_mix": 2.791009336709976, "l_a": 2.877388745546341, "l_c": 2.828193575143814, "z_loss": 41.22238254547119, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14714927971363068, "tokens_per_second": 1447187.9740754054, "eta_seconds": 5009.977198457718, "data_wait_seconds": 2.5212764739990234e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 58660, "total_steps": 72479, "tokens": 30754734080, "loss": 4.43156111240387, "l_mix": 2.747685581445694, "l_a": 2.8293034434318542, "l_c": 2.7836145162582397, "z_loss": 40.161160469055176, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15555010735988617, "tokens_per_second": 1470717.092101363, "eta_seconds": 4926.260741043091, "data_wait_seconds": 1.5795230865478516e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 58670, "total_steps": 72479, "tokens": 30759976960, "loss": 4.5611624121665955, "l_mix": 2.8279394805431366, "l_a": 2.9121066331863403, "l_c": 2.865302950143814, "z_loss": 41.12912368774414, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13804839551448822, "tokens_per_second": 1455963.7242774186, "eta_seconds": 4972.577868032456, "data_wait_seconds": 2.3126602172851562e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 58680, "total_steps": 72479, "tokens": 30765219840, "loss": 4.380367875099182, "l_mix": 2.7151304483413696, "l_a": 2.8005243241786957, "l_c": 2.750266879796982, "z_loss": 42.270034313201904, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15010134875774384, "tokens_per_second": 1352324.4178760024, "eta_seconds": 5349.788864541054, "data_wait_seconds": 1.9788742065429688e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 58690, "total_steps": 72479, "tokens": 30770462720, "loss": 4.500395834445953, "l_mix": 2.7907105088233948, "l_a": 2.8723024725914, "l_c": 2.826648622751236, "z_loss": 40.62804365158081, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1303281933069229, "tokens_per_second": 1460906.7129262304, "eta_seconds": 4948.5755442380905, "data_wait_seconds": 1.8596649169921875e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 58700, "total_steps": 72479, "tokens": 30775705600, "loss": 4.491835176944733, "l_mix": 2.784535735845566, "l_a": 2.870810091495514, "l_c": 2.8201880753040314, "z_loss": 40.85717725753784, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1365261673927307, "tokens_per_second": 1409582.5830715473, "eta_seconds": 5125.038035202027, "data_wait_seconds": 1.6391277313232422e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.16471112, "gate_a2b/L1_absmax": 0.6992783, "gate_a2b/L1_rms": 0.17525853, "gate_a2b/L3_absmean": 0.15654527, "gate_a2b/L3_absmax": 0.50311393, "gate_a2b/L3_rms": 0.16229162, "gate_a2b/L5_absmean": 0.23123346, "gate_a2b/L5_absmax": 0.73390752, "gate_a2b/L5_rms": 0.23728059, "gate_c2b/L1_absmean": 0.03241884, "gate_c2b/L1_absmax": 0.63704133, "gate_c2b/L1_rms": 0.06895858, "gate_c2b/L3_absmean": 0.17257598, "gate_c2b/L3_absmax": 0.39197594, "gate_c2b/L3_rms": 0.17618601, "gate_c2b/L5_absmean": 0.48772264, "gate_c2b/L5_absmax": 0.69095689, "gate_c2b/L5_rms": 0.49744642, "gate_b2a/L7_absmean": 0.05031346, "gate_b2a/L7_absmax": 0.33494577, "gate_b2a/L7_rms": 0.05620053, "gate_b2a/L11_absmean": 0.08215734, "gate_b2a/L11_absmax": 0.60681391, "gate_b2a/L11_rms": 0.09129064, "gate_b2a/L14_absmean": 0.21369807, "gate_b2a/L14_absmax": 0.82110828, "gate_b2a/L14_rms": 0.22310334, "gate_b2c/L3_absmean": 0.03485764, "gate_b2c/L3_absmax": 0.55335647, "gate_b2c/L3_rms": 0.04551865, "gate_b2c/L5_absmean": 0.0368617, "gate_b2c/L5_absmax": 0.52741921, "gate_b2c/L5_rms": 0.04599389, "gate_b2c/L6_absmean": 0.053262, "gate_b2c/L6_absmax": 0.34342447, "gate_b2c/L6_rms": 0.05934799}, "streams": {"cos_a2b/round0": 0.357646, "cos_c2b/round0": 0.25724, "cos_b2a/round0": 0.159943, "cos_b2c/round0": 0.144653, "rms_a/round0": 7.481294, "rms_b/round0": 12.934837, "rms_c/round0": 4.836915, "cos_a2b/round1": 0.412553, "cos_c2b/round1": 0.304063, "cos_b2a/round1": 0.252594, "cos_b2c/round1": 0.11514, "rms_a/round1": 14.729259, "rms_b/round1": 29.886642, "rms_c/round1": 10.424092, "cos_a2b/round2": 0.543149, "cos_c2b/round2": 0.517175, "cos_b2a/round2": 0.098398, "cos_b2c/round2": 0.182723, "rms_a/round2": 41.739471, "rms_b/round2": 80.452492, "rms_c/round2": 20.134222}, "alpha_mean": 0.439675066061767, "alpha_hist": [0.00789, 0.06119, 0.17152, 0.20345, 0.18481, 0.16777, 0.12769, 0.06673, 0.00883, 0.00012], "alpha_by_type": {"special": {"share": 0.0016, "mean": 0.54577, "std": 0.13965}, "whitespace": {"share": 0.02834, "mean": 0.52301, "std": 0.12028}, "digit": {"share": 0.03078, "mean": 0.48126, "std": 0.11595}, "word_start": {"share": 0.57156, "mean": 0.45382, "std": 0.16546}, "word_cont": {"share": 0.25871, "mean": 0.33539, "std": 0.13041}, "punct": {"share": 0.10901, "mean": 0.57803, "std": 0.15975}}, "alpha_p05": 0.18097, "alpha_p50": 0.45484, "alpha_p95": 0.72642}
{"step": 58710, "total_steps": 72479, "tokens": 30780948480, "loss": 4.5234140157699585, "l_mix": 2.804388701915741, "l_a": 2.888363540172577, "l_c": 2.8417205214500427, "z_loss": 41.314706325531006, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11551264673471451, "tokens_per_second": 1324181.5365353418, "eta_seconds": 5451.61012506485, "data_wait_seconds": 2.1755695343017578e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 58720, "total_steps": 72479, "tokens": 30786191360, "loss": 4.295583128929138, "l_mix": 2.6620624661445618, "l_a": 2.7457346320152283, "l_c": 2.6993337869644165, "z_loss": 42.40318727493286, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12033551186323166, "tokens_per_second": 1432920.6723252642, "eta_seconds": 5034.248393034935, "data_wait_seconds": 2.6702880859375e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 58730, "total_steps": 72479, "tokens": 30791434240, "loss": 4.454321503639221, "l_mix": 2.7610999047756195, "l_a": 2.846524864435196, "l_c": 2.7975468933582306, "z_loss": 41.82419443130493, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17643822729587555, "tokens_per_second": 1394028.6981462785, "eta_seconds": 5170.9378161191935, "data_wait_seconds": 1.8656253814697266e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 58740, "total_steps": 72479, "tokens": 30796677120, "loss": 2.780199706554413, "l_mix": 1.7208403795957565, "l_a": 1.7831306457519531, "l_c": 1.7480670809745789, "z_loss": 51.49786949157715, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18697324395179749, "tokens_per_second": 1460573.9892469947, "eta_seconds": 4931.754834079742, "data_wait_seconds": 1.761317253112793e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 58750, "total_steps": 72479, "tokens": 30801920000, "loss": 3.0364295840263367, "l_mix": 1.8801973909139633, "l_a": 1.9460439682006836, "l_c": 1.9080629646778107, "z_loss": 49.35540246963501, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19974203407764435, "tokens_per_second": 1469082.9488986337, "eta_seconds": 4899.621193885803, "data_wait_seconds": 1.77919864654541e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 58760, "total_steps": 72479, "tokens": 30807162880, "loss": 4.516285598278046, "l_mix": 2.800084710121155, "l_a": 2.8852995336055756, "l_c": 2.835369825363159, "z_loss": 40.59976387023926, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14192213118076324, "tokens_per_second": 1438690.17349805, "eta_seconds": 4999.483005094528, "data_wait_seconds": 2.720952033996582e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 58770, "total_steps": 72479, "tokens": 30812405760, "loss": 4.597089469432831, "l_mix": 2.8513890206813812, "l_a": 2.927679568529129, "l_c": 2.8913219571113586, "z_loss": 41.37672185897827, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16489730775356293, "tokens_per_second": 1443668.8572158979, "eta_seconds": 4978.609988069535, "data_wait_seconds": 2.6911497116088867e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 58780, "total_steps": 72479, "tokens": 30817648640, "loss": 4.424062788486481, "l_mix": 2.7422936260700226, "l_a": 2.8274868428707123, "l_c": 2.7784099876880646, "z_loss": 41.25031089782715, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14010339975357056, "tokens_per_second": 1468722.0647222034, "eta_seconds": 4890.116029787064, "data_wait_seconds": 1.895427703857422e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 58790, "total_steps": 72479, "tokens": 30822891520, "loss": 4.491802394390106, "l_mix": 2.784253031015396, "l_a": 2.867558002471924, "l_c": 2.8242730498313904, "z_loss": 41.91970682144165, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18041838705539703, "tokens_per_second": 1332616.7471441913, "eta_seconds": 5385.628274130821, "data_wait_seconds": 2.3543834686279297e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 58800, "total_steps": 72479, "tokens": 30828134400, "loss": 4.595522403717041, "l_mix": 2.8505064845085144, "l_a": 2.930501103401184, "l_c": 2.8862182796001434, "z_loss": 41.067466259002686, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18274237215518951, "tokens_per_second": 1397472.9048197563, "eta_seconds": 5131.931737112998, "data_wait_seconds": 1.4513731002807617e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.16464745, "gate_a2b/L1_absmax": 0.69685876, "gate_a2b/L1_rms": 0.17515916, "gate_a2b/L3_absmean": 0.15647903, "gate_a2b/L3_absmax": 0.50294179, "gate_a2b/L3_rms": 0.16222617, "gate_a2b/L5_absmean": 0.23047581, "gate_a2b/L5_absmax": 0.7346257, "gate_a2b/L5_rms": 0.23660412, "gate_c2b/L1_absmean": 0.03202457, "gate_c2b/L1_absmax": 0.63470572, "gate_c2b/L1_rms": 0.06888414, "gate_c2b/L3_absmean": 0.17206369, "gate_c2b/L3_absmax": 0.38907394, "gate_c2b/L3_rms": 0.17570381, "gate_c2b/L5_absmean": 0.48742524, "gate_c2b/L5_absmax": 0.68183064, "gate_c2b/L5_rms": 0.49718049, "gate_b2a/L7_absmean": 0.05039206, "gate_b2a/L7_absmax": 0.3347131, "gate_b2a/L7_rms": 0.05632033, "gate_b2a/L11_absmean": 0.08206078, "gate_b2a/L11_absmax": 0.60744095, "gate_b2a/L11_rms": 0.09120486, "gate_b2a/L14_absmean": 0.21359351, "gate_b2a/L14_absmax": 0.82004935, "gate_b2a/L14_rms": 0.22301072, "gate_b2c/L3_absmean": 0.03493293, "gate_b2c/L3_absmax": 0.55255991, "gate_b2c/L3_rms": 0.04552005, "gate_b2c/L5_absmean": 0.03689727, "gate_b2c/L5_absmax": 0.528678, "gate_b2c/L5_rms": 0.04603392, "gate_b2c/L6_absmean": 0.05331231, "gate_b2c/L6_absmax": 0.34517169, "gate_b2c/L6_rms": 0.05943045}, "streams": {"cos_a2b/round0": 0.355942, "cos_c2b/round0": 0.258351, "cos_b2a/round0": 0.163105, "cos_b2c/round0": 0.146318, "rms_a/round0": 7.414066, "rms_b/round0": 12.894611, "rms_c/round0": 4.852118, "cos_a2b/round1": 0.415355, "cos_c2b/round1": 0.310378, "cos_b2a/round1": 0.25611, "cos_b2c/round1": 0.112755, "rms_a/round1": 14.561108, "rms_b/round1": 29.750734, "rms_c/round1": 10.308496, "cos_a2b/round2": 0.539889, "cos_c2b/round2": 0.522483, "cos_b2a/round2": 0.099372, "cos_b2c/round2": 0.187397, "rms_a/round2": 41.411278, "rms_b/round2": 80.820457, "rms_c/round2": 19.962406}, "alpha_mean": 0.43848200759024536, "alpha_hist": [0.00597, 0.05664, 0.18172, 0.20927, 0.18616, 0.16064, 0.12085, 0.06767, 0.01099, 9e-05], "alpha_by_type": {"special": {"share": 0.00104, "mean": 0.57333, "std": 0.1185}, "whitespace": {"share": 0.02345, "mean": 0.52088, "std": 0.12131}, "digit": {"share": 0.02217, "mean": 0.40712, "std": 0.11594}, "word_start": {"share": 0.59407, "mean": 0.4586, "std": 0.16133}, "word_cont": {"share": 0.2532, "mean": 0.32957, "std": 0.12689}, "punct": {"share": 0.10606, "mean": 0.57284, "std": 0.17107}}, "alpha_p05": 0.20307, "alpha_p50": 0.42084, "alpha_p95": 0.74129}
{"step": 58810, "total_steps": 72479, "tokens": 30833377280, "loss": 4.385731518268585, "l_mix": 2.7183942198753357, "l_a": 2.803406238555908, "l_c": 2.754384756088257, "z_loss": 41.68902254104614, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11272773146629333, "tokens_per_second": 1469150.9656923139, "eta_seconds": 4877.982480597496, "data_wait_seconds": 1.8775463104248047e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 58820, "total_steps": 72479, "tokens": 30838620160, "loss": 4.720766603946686, "l_mix": 2.9296599328517914, "l_a": 3.004070997238159, "l_c": 2.966284394264221, "z_loss": 40.05892562866211, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11717060953378677, "tokens_per_second": 1441273.9229455525, "eta_seconds": 4968.69448478222, "data_wait_seconds": 2.8967857360839844e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 58830, "total_steps": 72479, "tokens": 30843863040, "loss": 4.677346229553223, "l_mix": 2.9018633663654327, "l_a": 2.979575514793396, "l_c": 2.938700348138809, "z_loss": 40.74271297454834, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12551386654376984, "tokens_per_second": 1427864.4876468626, "eta_seconds": 5011.684914016723, "data_wait_seconds": 2.142786979675293e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 58840, "total_steps": 72479, "tokens": 30849105920, "loss": 4.7356473207473755, "l_mix": 2.9383569061756134, "l_a": 3.0167134404182434, "l_c": 2.9742545783519745, "z_loss": 40.10323143005371, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14723852276802063, "tokens_per_second": 1362884.151890004, "eta_seconds": 5246.787866806984, "data_wait_seconds": 2.3752450942993164e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 58850, "total_steps": 72479, "tokens": 30854348800, "loss": 4.7002235651016235, "l_mix": 2.915855258703232, "l_a": 2.9955357015132904, "l_c": 2.9523586332798004, "z_loss": 41.07507801055908, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14991509914398193, "tokens_per_second": 1297649.3119179597, "eta_seconds": 5506.511725759506, "data_wait_seconds": 4.673004150390625e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 58860, "total_steps": 72479, "tokens": 30859591680, "loss": 2.865123450756073, "l_mix": 1.7735849469900131, "l_a": 1.8375974148511887, "l_c": 1.8008640110492706, "z_loss": 51.05464553833008, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13624326884746552, "tokens_per_second": 1436372.1343826556, "eta_seconds": 4971.050399184227, "data_wait_seconds": 3.0040740966796875e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 58870, "total_steps": 72479, "tokens": 30864834560, "loss": 4.453146457672119, "l_mix": 2.7604868710041046, "l_a": 2.845041334629059, "l_c": 2.797156900167465, "z_loss": 42.31214189529419, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12750039994716644, "tokens_per_second": 1407631.6355636069, "eta_seconds": 5068.822845220566, "data_wait_seconds": 3.790855407714844e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 58880, "total_steps": 72479, "tokens": 30870077440, "loss": 4.430484712123871, "l_mix": 2.7468964755535126, "l_a": 2.8283787071704865, "l_c": 2.7835819125175476, "z_loss": 41.611257553100586, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14459170401096344, "tokens_per_second": 1464492.1389783681, "eta_seconds": 4868.440275120734, "data_wait_seconds": 1.8537044525146484e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 58890, "total_steps": 72479, "tokens": 30875320320, "loss": 4.515836298465729, "l_mix": 2.7999062836170197, "l_a": 2.8815901577472687, "l_c": 2.8381766378879547, "z_loss": 41.4479193687439, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1358327865600586, "tokens_per_second": 1447640.4109464684, "eta_seconds": 4921.491261315346, "data_wait_seconds": 1.8090009689331055e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 58900, "total_steps": 72479, "tokens": 30880563200, "loss": 4.474977791309357, "l_mix": 2.7741735875606537, "l_a": 2.8586972057819366, "l_c": 2.810650199651718, "z_loss": 41.80636692047119, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11609447747468948, "tokens_per_second": 1444412.6751435301, "eta_seconds": 4928.859234285354, "data_wait_seconds": 2.0235776901245117e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16488509, "gate_a2b/L1_absmax": 0.6969462, "gate_a2b/L1_rms": 0.17534049, "gate_a2b/L3_absmean": 0.15644929, "gate_a2b/L3_absmax": 0.50379217, "gate_a2b/L3_rms": 0.16215184, "gate_a2b/L5_absmean": 0.23054543, "gate_a2b/L5_absmax": 0.73422515, "gate_a2b/L5_rms": 0.23661542, "gate_c2b/L1_absmean": 0.03303814, "gate_c2b/L1_absmax": 0.63460261, "gate_c2b/L1_rms": 0.06906682, "gate_c2b/L3_absmean": 0.17202656, "gate_c2b/L3_absmax": 0.38761818, "gate_c2b/L3_rms": 0.17564276, "gate_c2b/L5_absmean": 0.48733902, "gate_c2b/L5_absmax": 0.68528563, "gate_c2b/L5_rms": 0.49710178, "gate_b2a/L7_absmean": 0.05080979, "gate_b2a/L7_absmax": 0.33407941, "gate_b2a/L7_rms": 0.05659638, "gate_b2a/L11_absmean": 0.08194321, "gate_b2a/L11_absmax": 0.607629, "gate_b2a/L11_rms": 0.09110483, "gate_b2a/L14_absmean": 0.21387273, "gate_b2a/L14_absmax": 0.81996983, "gate_b2a/L14_rms": 0.22322355, "gate_b2c/L3_absmean": 0.03496905, "gate_b2c/L3_absmax": 0.5522964, "gate_b2c/L3_rms": 0.04556122, "gate_b2c/L5_absmean": 0.03690297, "gate_b2c/L5_absmax": 0.52826673, "gate_b2c/L5_rms": 0.04606292, "gate_b2c/L6_absmean": 0.0533792, "gate_b2c/L6_absmax": 0.34418547, "gate_b2c/L6_rms": 0.05946912}, "streams": {"cos_a2b/round0": 0.352617, "cos_c2b/round0": 0.250329, "cos_b2a/round0": 0.156322, "cos_b2c/round0": 0.139201, "rms_a/round0": 7.451318, "rms_b/round0": 13.022108, "rms_c/round0": 4.921947, "cos_a2b/round1": 0.405989, "cos_c2b/round1": 0.301309, "cos_b2a/round1": 0.249594, "cos_b2c/round1": 0.10902, "rms_a/round1": 14.664566, "rms_b/round1": 30.080023, "rms_c/round1": 10.42328, "cos_a2b/round2": 0.525777, "cos_c2b/round2": 0.533426, "cos_b2a/round2": 0.090125, "cos_b2c/round2": 0.182197, "rms_a/round2": 41.285503, "rms_b/round2": 81.078064, "rms_c/round2": 20.204618}, "alpha_mean": 0.4128236250495334, "alpha_hist": [0.01239, 0.07921, 0.19351, 0.22139, 0.18428, 0.15512, 0.0995, 0.04889, 0.00555, 0.00015], "alpha_by_type": {"special": {"share": 0.00127, "mean": 0.5535, "std": 0.12377}, "whitespace": {"share": 0.02505, "mean": 0.50675, "std": 0.12635}, "digit": {"share": 0.03552, "mean": 0.44065, "std": 0.10686}, "word_start": {"share": 0.53439, "mean": 0.42774, "std": 0.16395}, "word_cont": {"share": 0.25082, "mean": 0.30856, "std": 0.12806}, "punct": {"share": 0.15294, "mean": 0.50869, "std": 0.15728}}, "alpha_p05": 0.16886, "alpha_p50": 0.40333, "alpha_p95": 0.72875}
{"step": 58910, "total_steps": 72479, "tokens": 30885806080, "loss": 4.5078330636024475, "l_mix": 2.7947119772434235, "l_a": 2.878639817237854, "l_c": 2.8317635655403137, "z_loss": 41.708024978637695, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11753938347101212, "tokens_per_second": 1445312.2725839322, "eta_seconds": 4922.1638859272, "data_wait_seconds": 2.3156404495239258e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 58920, "total_steps": 72479, "tokens": 30891048960, "loss": 4.316714733839035, "l_mix": 2.6758487820625305, "l_a": 2.758458584547043, "l_c": 2.7110943496227264, "z_loss": 42.38538360595703, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17136329412460327, "tokens_per_second": 1306350.7075260808, "eta_seconds": 5441.740071058273, "data_wait_seconds": 2.327561378479004e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 58930, "total_steps": 72479, "tokens": 30896291840, "loss": 4.64090222120285, "l_mix": 2.8795685470104218, "l_a": 2.955322951078415, "l_c": 2.9157893359661102, "z_loss": 39.839455127716064, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13702216744422913, "tokens_per_second": 1357687.6012710638, "eta_seconds": 5232.115330028533, "data_wait_seconds": 2.5957822799682617e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 58940, "total_steps": 72479, "tokens": 30901534720, "loss": 4.631763279438019, "l_mix": 2.8738445341587067, "l_a": 2.950771063566208, "l_c": 2.9089581072330475, "z_loss": 40.183143615722656, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13598306477069855, "tokens_per_second": 1423721.9450283397, "eta_seconds": 4985.759513497353, "data_wait_seconds": 2.804398536682129e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 58950, "total_steps": 72479, "tokens": 30906777600, "loss": 4.595057189464569, "l_mix": 2.8517402708530426, "l_a": 2.9253602027893066, "l_c": 2.8856959640979767, "z_loss": 40.95199012756348, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13769884407520294, "tokens_per_second": 1412134.1144311002, "eta_seconds": 5022.959419727325, "data_wait_seconds": 2.3692846298217773e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 58960, "total_steps": 72479, "tokens": 30912020480, "loss": 4.6261818408966064, "l_mix": 2.869035989046097, "l_a": 2.953313708305359, "l_c": 2.9038389325141907, "z_loss": 40.464874267578125, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15676216781139374, "tokens_per_second": 1365703.659445924, "eta_seconds": 5189.888320922852, "data_wait_seconds": 2.4646520614624023e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 58970, "total_steps": 72479, "tokens": 30917263360, "loss": 4.724173605442047, "l_mix": 2.931216061115265, "l_a": 3.0081840455532074, "l_c": 2.968340903520584, "z_loss": 39.336628913879395, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14553247392177582, "tokens_per_second": 1441611.2359473072, "eta_seconds": 4912.979599070549, "data_wait_seconds": 2.282857894897461e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 58980, "total_steps": 72479, "tokens": 30922506240, "loss": 4.435564458370209, "l_mix": 2.750112384557724, "l_a": 2.8314146995544434, "l_c": 2.7867587506771088, "z_loss": 42.01691389083862, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1461121290922165, "tokens_per_second": 1471521.8403263786, "eta_seconds": 4809.553971982003, "data_wait_seconds": 2.1159648895263672e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 58990, "total_steps": 72479, "tokens": 30927749120, "loss": 4.494769990444183, "l_mix": 2.7866742610931396, "l_a": 2.8700181543827057, "l_c": 2.823634058237076, "z_loss": 42.017465591430664, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1480371505022049, "tokens_per_second": 1449599.561783897, "eta_seconds": 4878.672026705742, "data_wait_seconds": 2.41696834564209e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 59000, "total_steps": 72479, "tokens": 30932992000, "loss": 4.509079575538635, "l_mix": 2.794560968875885, "l_a": 2.8811775147914886, "l_c": 2.8338841795921326, "z_loss": 42.727942943573, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1656101495027542, "tokens_per_second": 1423374.1556383306, "eta_seconds": 4964.877241873741, "data_wait_seconds": 1.58846378326416e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16455436, "gate_a2b/L1_absmax": 0.69539094, "gate_a2b/L1_rms": 0.17505863, "gate_a2b/L3_absmean": 0.15650912, "gate_a2b/L3_absmax": 0.50379735, "gate_a2b/L3_rms": 0.16223036, "gate_a2b/L5_absmean": 0.23011275, "gate_a2b/L5_absmax": 0.73453629, "gate_a2b/L5_rms": 0.23617829, "gate_c2b/L1_absmean": 0.03281816, "gate_c2b/L1_absmax": 0.63347203, "gate_c2b/L1_rms": 0.06904672, "gate_c2b/L3_absmean": 0.17186676, "gate_c2b/L3_absmax": 0.38935223, "gate_c2b/L3_rms": 0.17550452, "gate_c2b/L5_absmean": 0.48715922, "gate_c2b/L5_absmax": 0.68807757, "gate_c2b/L5_rms": 0.49696928, "gate_b2a/L7_absmean": 0.05106179, "gate_b2a/L7_absmax": 0.33343932, "gate_b2a/L7_rms": 0.05684367, "gate_b2a/L11_absmean": 0.08212788, "gate_b2a/L11_absmax": 0.60691059, "gate_b2a/L11_rms": 0.09129114, "gate_b2a/L14_absmean": 0.21377547, "gate_b2a/L14_absmax": 0.82015818, "gate_b2a/L14_rms": 0.22312394, "gate_b2c/L3_absmean": 0.03500172, "gate_b2c/L3_absmax": 0.55141598, "gate_b2c/L3_rms": 0.04561004, "gate_b2c/L5_absmean": 0.03689787, "gate_b2c/L5_absmax": 0.52862835, "gate_b2c/L5_rms": 0.04608291, "gate_b2c/L6_absmean": 0.05303095, "gate_b2c/L6_absmax": 0.34412807, "gate_b2c/L6_rms": 0.05911689}, "streams": {"cos_a2b/round0": 0.352785, "cos_c2b/round0": 0.252576, "cos_b2a/round0": 0.157226, "cos_b2c/round0": 0.142409, "rms_a/round0": 7.444568, "rms_b/round0": 12.982818, "rms_c/round0": 4.861686, "cos_a2b/round1": 0.409747, "cos_c2b/round1": 0.300288, "cos_b2a/round1": 0.249015, "cos_b2c/round1": 0.113153, "rms_a/round1": 14.780239, "rms_b/round1": 30.161966, "rms_c/round1": 10.42347, "cos_a2b/round2": 0.53237, "cos_c2b/round2": 0.524404, "cos_b2a/round2": 0.097422, "cos_b2c/round2": 0.182329, "rms_a/round2": 41.677826, "rms_b/round2": 80.856056, "rms_c/round2": 20.246397}, "alpha_mean": 0.42425761722043376, "alpha_hist": [0.01141, 0.08076, 0.1938, 0.20224, 0.16911, 0.15276, 0.1152, 0.06427, 0.01038, 6e-05], "alpha_by_type": {"special": {"share": 0.00142, "mean": 0.52853, "std": 0.13189}, "whitespace": {"share": 0.02742, "mean": 0.51647, "std": 0.12786}, "digit": {"share": 0.03397, "mean": 0.45558, "std": 0.1137}, "word_start": {"share": 0.57159, "mean": 0.44622, "std": 0.1701}, "word_cont": {"share": 0.26163, "mean": 0.30634, "std": 0.12779}, "punct": {"share": 0.10397, "mean": 0.56425, "std": 0.16767}}, "alpha_p05": 0.16265, "alpha_p50": 0.40169, "alpha_p95": 0.75128}
{"step": 59010, "total_steps": 72479, "tokens": 30938234880, "loss": 4.470926940441132, "l_mix": 2.7717093527317047, "l_a": 2.853524476289749, "l_c": 2.810533791780472, "z_loss": 41.435462474823, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.127653568983078, "tokens_per_second": 1476142.332147308, "eta_seconds": 4783.844293475151, "data_wait_seconds": 1.8388032913208008e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 59020, "total_steps": 72479, "tokens": 30943477760, "loss": 4.511841177940369, "l_mix": 2.7970724999904633, "l_a": 2.8829360008239746, "l_c": 2.8329595029354095, "z_loss": 41.85205316543579, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14338092505931854, "tokens_per_second": 1464804.6961395044, "eta_seconds": 4817.292169117927, "data_wait_seconds": 1.8984079360961914e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 59030, "total_steps": 72479, "tokens": 30948720640, "loss": 4.473600387573242, "l_mix": 2.7731844186782837, "l_a": 2.859197974205017, "l_c": 2.808855175971985, "z_loss": 42.57178211212158, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1875225156545639, "tokens_per_second": 1446748.0031450482, "eta_seconds": 4873.79232366085, "data_wait_seconds": 1.850724220275879e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 59040, "total_steps": 72479, "tokens": 30953963520, "loss": 3.068185716867447, "l_mix": 1.8876755982637405, "l_a": 2.0000558495521545, "l_c": 1.934977874159813, "z_loss": 45.48949432373047, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12968388199806213, "tokens_per_second": 1429057.411466963, "eta_seconds": 4930.45721988678, "data_wait_seconds": 2.5004148483276367e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 59050, "total_steps": 72479, "tokens": 30959206400, "loss": 3.076848179101944, "l_mix": 1.8930665850639343, "l_a": 2.0029653310775757, "l_c": 1.9429732412099838, "z_loss": 45.230464935302734, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16266579926013947, "tokens_per_second": 1431484.0990761395, "eta_seconds": 4918.4364370822905, "data_wait_seconds": 3.293156623840332e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 59060, "total_steps": 72479, "tokens": 30964449280, "loss": 4.62639707326889, "l_mix": 2.869957536458969, "l_a": 2.951627105474472, "l_c": 2.903171420097351, "z_loss": 40.91730737686157, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1908036470413208, "tokens_per_second": 1388480.7047118563, "eta_seconds": 5066.992035341263, "data_wait_seconds": 2.60770320892334e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 59070, "total_steps": 72479, "tokens": 30969692160, "loss": 4.642461061477661, "l_mix": 2.8800733983516693, "l_a": 2.9583857357501984, "l_c": 2.9162396788597107, "z_loss": 40.37348747253418, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12620733678340912, "tokens_per_second": 1389943.5239821821, "eta_seconds": 5057.887367868423, "data_wait_seconds": 1.773238182067871e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 59080, "total_steps": 72479, "tokens": 30974935040, "loss": 4.590959787368774, "l_mix": 2.8479798436164856, "l_a": 2.9254495203495026, "l_c": 2.884483575820923, "z_loss": 40.136786460876465, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14007166028022766, "tokens_per_second": 1434410.0941951948, "eta_seconds": 4897.438285207749, "data_wait_seconds": 3.2067298889160156e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 59090, "total_steps": 72479, "tokens": 30980177920, "loss": 4.387323021888733, "l_mix": 2.7198800444602966, "l_a": 2.8033580482006073, "l_c": 2.7547852396965027, "z_loss": 41.13375663757324, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1210208460688591, "tokens_per_second": 1452288.5689666981, "eta_seconds": 4833.538032317161, "data_wait_seconds": 1.5288591384887695e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 59100, "total_steps": 72479, "tokens": 30985420800, "loss": 4.453239858150482, "l_mix": 2.7606802582740784, "l_a": 2.8441825211048126, "l_c": 2.7976823449134827, "z_loss": 42.081414222717285, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11778517812490463, "tokens_per_second": 1457212.9659119148, "eta_seconds": 4813.606052160263, "data_wait_seconds": 2.1785497665405273e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16402958, "gate_a2b/L1_absmax": 0.69416279, "gate_a2b/L1_rms": 0.17452094, "gate_a2b/L3_absmean": 0.15625039, "gate_a2b/L3_absmax": 0.50281388, "gate_a2b/L3_rms": 0.16193494, "gate_a2b/L5_absmean": 0.2297997, "gate_a2b/L5_absmax": 0.73378175, "gate_a2b/L5_rms": 0.23586552, "gate_c2b/L1_absmean": 0.03305718, "gate_c2b/L1_absmax": 0.63332033, "gate_c2b/L1_rms": 0.06906345, "gate_c2b/L3_absmean": 0.1722375, "gate_c2b/L3_absmax": 0.39022204, "gate_c2b/L3_rms": 0.175813, "gate_c2b/L5_absmean": 0.48633358, "gate_c2b/L5_absmax": 0.69167542, "gate_c2b/L5_rms": 0.49615657, "gate_b2a/L7_absmean": 0.0503143, "gate_b2a/L7_absmax": 0.33286643, "gate_b2a/L7_rms": 0.05623351, "gate_b2a/L11_absmean": 0.08241239, "gate_b2a/L11_absmax": 0.60847384, "gate_b2a/L11_rms": 0.09154765, "gate_b2a/L14_absmean": 0.21362838, "gate_b2a/L14_absmax": 0.82136905, "gate_b2a/L14_rms": 0.22303338, "gate_b2c/L3_absmean": 0.03493723, "gate_b2c/L3_absmax": 0.55003709, "gate_b2c/L3_rms": 0.04549562, "gate_b2c/L5_absmean": 0.03674794, "gate_b2c/L5_absmax": 0.5272755, "gate_b2c/L5_rms": 0.04592331, "gate_b2c/L6_absmean": 0.05321762, "gate_b2c/L6_absmax": 0.34378472, "gate_b2c/L6_rms": 0.05934305}, "streams": {"cos_a2b/round0": 0.358014, "cos_c2b/round0": 0.255479, "cos_b2a/round0": 0.152196, "cos_b2c/round0": 0.142854, "rms_a/round0": 7.483186, "rms_b/round0": 12.826849, "rms_c/round0": 4.854444, "cos_a2b/round1": 0.405768, "cos_c2b/round1": 0.298862, "cos_b2a/round1": 0.245787, "cos_b2c/round1": 0.110099, "rms_a/round1": 14.643119, "rms_b/round1": 29.972385, "rms_c/round1": 10.335932, "cos_a2b/round2": 0.529771, "cos_c2b/round2": 0.522616, "cos_b2a/round2": 0.094168, "cos_b2c/round2": 0.179976, "rms_a/round2": 41.24802, "rms_b/round2": 80.918854, "rms_c/round2": 19.950346}, "alpha_mean": 0.43121748760394496, "alpha_hist": [0.00896, 0.06976, 0.18504, 0.20921, 0.17862, 0.15135, 0.11795, 0.06796, 0.01083, 0.00031], "alpha_by_type": {"special": {"share": 0.00121, "mean": 0.5436, "std": 0.11}, "whitespace": {"share": 0.02893, "mean": 0.5377, "std": 0.13657}, "digit": {"share": 0.05891, "mean": 0.47258, "std": 0.11569}, "word_start": {"share": 0.54034, "mean": 0.44188, "std": 0.17066}, "word_cont": {"share": 0.26306, "mean": 0.32327, "std": 0.12502}, "punct": {"share": 0.10754, "mean": 0.58913, "std": 0.15882}}, "alpha_p05": 0.1747, "alpha_p50": 0.40568, "alpha_p95": 0.74167}
{"step": 59110, "total_steps": 72479, "tokens": 30990663680, "loss": 4.4179967045784, "l_mix": 2.7384733259677887, "l_a": 2.8218429386615753, "l_c": 2.7765683829784393, "z_loss": 42.603744983673096, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13971808552742004, "tokens_per_second": 1424208.988219345, "eta_seconds": 4921.47313349247, "data_wait_seconds": 1.7136335372924805e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 59120, "total_steps": 72479, "tokens": 30995906560, "loss": 4.499979496002197, "l_mix": 2.790064752101898, "l_a": 2.8724818229675293, "l_c": 2.8272337913513184, "z_loss": 41.137980937957764, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1330486387014389, "tokens_per_second": 1358773.2388096878, "eta_seconds": 5154.6227081537245, "data_wait_seconds": 2.0503997802734375e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 59130, "total_steps": 72479, "tokens": 31001149440, "loss": 4.448160231113434, "l_mix": 2.7574778497219086, "l_a": 2.8418441712856293, "l_c": 2.793763607740402, "z_loss": 41.917105197906494, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.10338834673166275, "tokens_per_second": 1381168.3087794897, "eta_seconds": 5067.2466690063475, "data_wait_seconds": 2.0325183868408203e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 59140, "total_steps": 72479, "tokens": 31006392320, "loss": 4.425136685371399, "l_mix": 2.742820769548416, "l_a": 2.8272306621074677, "l_c": 2.7804890871047974, "z_loss": 42.16305494308472, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14611981809139252, "tokens_per_second": 1322625.9239349575, "eta_seconds": 5287.570359420777, "data_wait_seconds": 2.0265579223632812e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 59150, "total_steps": 72479, "tokens": 31011635200, "loss": 3.0649257600307465, "l_mix": 1.8867107778787613, "l_a": 1.9930868297815323, "l_c": 1.9342962801456451, "z_loss": 44.80237817764282, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1934247761964798, "tokens_per_second": 1440992.7604817518, "eta_seconds": 4849.597405099868, "data_wait_seconds": 2.8789043426513672e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 59160, "total_steps": 72479, "tokens": 31016878080, "loss": 3.1084790527820587, "l_mix": 1.913555085659027, "l_a": 2.023409053683281, "l_c": 1.959670826792717, "z_loss": 44.976526737213135, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17607013881206512, "tokens_per_second": 1442483.0279149984, "eta_seconds": 4840.952535915375, "data_wait_seconds": 1.7255544662475586e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 59170, "total_steps": 72479, "tokens": 31022120960, "loss": 3.0767235159873962, "l_mix": 1.8931409865617752, "l_a": 2.007128670811653, "l_c": 1.938146248459816, "z_loss": 47.18105173110962, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2597689926624298, "tokens_per_second": 1434094.6599105587, "eta_seconds": 4865.612561750412, "data_wait_seconds": 2.4974346160888672e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 59180, "total_steps": 72479, "tokens": 31027363840, "loss": 3.133448898792267, "l_mix": 1.9285805225372314, "l_a": 2.0404027700424194, "l_c": 1.9758248925209045, "z_loss": 45.55995988845825, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19207385182380676, "tokens_per_second": 1367732.1837986216, "eta_seconds": 5097.859211468697, "data_wait_seconds": 2.1576881408691406e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 59190, "total_steps": 72479, "tokens": 31032606720, "loss": 3.068963199853897, "l_mix": 1.887977346777916, "l_a": 2.0021830052137375, "l_c": 1.9344363361597061, "z_loss": 45.20976161956787, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.25532618165016174, "tokens_per_second": 1404991.9273250534, "eta_seconds": 4958.9347073793415, "data_wait_seconds": 2.1219253540039062e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 59200, "total_steps": 72479, "tokens": 31037849600, "loss": 4.371528327465057, "l_mix": 2.7092197835445404, "l_a": 2.7944490015506744, "l_c": 2.74657940864563, "z_loss": 42.632405281066895, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1391357183456421, "tokens_per_second": 1395335.9694595342, "eta_seconds": 4989.493931484222, "data_wait_seconds": 1.576542854309082e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.16437058, "gate_a2b/L1_absmax": 0.69507921, "gate_a2b/L1_rms": 0.17478229, "gate_a2b/L3_absmean": 0.15596424, "gate_a2b/L3_absmax": 0.50087565, "gate_a2b/L3_rms": 0.16170569, "gate_a2b/L5_absmean": 0.23009591, "gate_a2b/L5_absmax": 0.73290622, "gate_a2b/L5_rms": 0.23620549, "gate_c2b/L1_absmean": 0.03306771, "gate_c2b/L1_absmax": 0.63494748, "gate_c2b/L1_rms": 0.06909836, "gate_c2b/L3_absmean": 0.17152113, "gate_c2b/L3_absmax": 0.39189211, "gate_c2b/L3_rms": 0.17517924, "gate_c2b/L5_absmean": 0.48587275, "gate_c2b/L5_absmax": 0.68304509, "gate_c2b/L5_rms": 0.49556595, "gate_b2a/L7_absmean": 0.05058859, "gate_b2a/L7_absmax": 0.33219343, "gate_b2a/L7_rms": 0.05637816, "gate_b2a/L11_absmean": 0.08198544, "gate_b2a/L11_absmax": 0.6087541, "gate_b2a/L11_rms": 0.0911786, "gate_b2a/L14_absmean": 0.21400782, "gate_b2a/L14_absmax": 0.81976646, "gate_b2a/L14_rms": 0.22335592, "gate_b2c/L3_absmean": 0.03490913, "gate_b2c/L3_absmax": 0.54971021, "gate_b2c/L3_rms": 0.04541622, "gate_b2c/L5_absmean": 0.03697133, "gate_b2c/L5_absmax": 0.52832645, "gate_b2c/L5_rms": 0.04610987, "gate_b2c/L6_absmean": 0.05330314, "gate_b2c/L6_absmax": 0.34373826, "gate_b2c/L6_rms": 0.05940866}, "streams": {"cos_a2b/round0": 0.359416, "cos_c2b/round0": 0.264738, "cos_b2a/round0": 0.162489, "cos_b2c/round0": 0.148895, "rms_a/round0": 7.500375, "rms_b/round0": 12.786707, "rms_c/round0": 4.849392, "cos_a2b/round1": 0.418177, "cos_c2b/round1": 0.306748, "cos_b2a/round1": 0.255814, "cos_b2c/round1": 0.111489, "rms_a/round1": 14.666683, "rms_b/round1": 29.678574, "rms_c/round1": 10.300925, "cos_a2b/round2": 0.539466, "cos_c2b/round2": 0.528638, "cos_b2a/round2": 0.09437, "cos_b2c/round2": 0.18008, "rms_a/round2": 41.534035, "rms_b/round2": 80.945023, "rms_c/round2": 20.01856}, "alpha_mean": 0.42807946671659636, "alpha_hist": [0.00668, 0.07091, 0.18892, 0.21483, 0.17717, 0.15205, 0.11528, 0.06531, 0.00879, 6e-05], "alpha_by_type": {"special": {"share": 0.00101, "mean": 0.552, "std": 0.12321}, "whitespace": {"share": 0.02486, "mean": 0.5144, "std": 0.12689}, "digit": {"share": 0.02946, "mean": 0.43817, "std": 0.12665}, "word_start": {"share": 0.6073, "mean": 0.4407, "std": 0.16806}, "word_cont": {"share": 0.24141, "mean": 0.32582, "std": 0.12268}, "punct": {"share": 0.09596, "mean": 0.57873, "std": 0.16186}}, "alpha_p05": 0.16559, "alpha_p50": 0.40663, "alpha_p95": 0.71243}
{"step": 59210, "total_steps": 72479, "tokens": 31043092480, "loss": 4.483115494251251, "l_mix": 2.7796285450458527, "l_a": 2.8609246909618378, "l_c": 2.817364811897278, "z_loss": 41.5909743309021, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12072999030351639, "tokens_per_second": 1411930.018291973, "eta_seconds": 4927.1404261350635, "data_wait_seconds": 1.862645149230957e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 59220, "total_steps": 72479, "tokens": 31048335360, "loss": 4.657746732234955, "l_mix": 2.8896475732326508, "l_a": 2.9705308377742767, "l_c": 2.9231328070163727, "z_loss": 41.806899070739746, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16118642687797546, "tokens_per_second": 1395240.3554012044, "eta_seconds": 4982.32047624588, "data_wait_seconds": 1.6123056411743164e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 59230, "total_steps": 72479, "tokens": 31053578240, "loss": 4.4193655252456665, "l_mix": 2.738661289215088, "l_a": 2.826024830341339, "l_c": 2.7763227820396423, "z_loss": 41.92614555358887, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11024585366249084, "tokens_per_second": 1463522.2472010017, "eta_seconds": 4746.283649110794, "data_wait_seconds": 1.6868114471435547e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 59240, "total_steps": 72479, "tokens": 31058821120, "loss": 4.593069791793823, "l_mix": 2.8479589223861694, "l_a": 2.932024359703064, "l_c": 2.885011315345764, "z_loss": 42.27008390426636, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13365495204925537, "tokens_per_second": 1460733.6854592043, "eta_seconds": 4751.755163240433, "data_wait_seconds": 1.4901161193847656e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 59250, "total_steps": 72479, "tokens": 31064064000, "loss": 4.453126430511475, "l_mix": 2.7609563171863556, "l_a": 2.8435201048851013, "l_c": 2.797046661376953, "z_loss": 41.30246877670288, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15392425656318665, "tokens_per_second": 1467910.1843065573, "eta_seconds": 4724.9525387525555, "data_wait_seconds": 3.167986869812012e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 59260, "total_steps": 72479, "tokens": 31069306880, "loss": 3.12625190615654, "l_mix": 1.9228324443101883, "l_a": 2.0351971685886383, "l_c": 1.9762010425329208, "z_loss": 44.981773853302, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2461138516664505, "tokens_per_second": 1272881.381416522, "eta_seconds": 5444.78312997818, "data_wait_seconds": 2.5004148483276367e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 59270, "total_steps": 72479, "tokens": 31074549760, "loss": 3.1256677210330963, "l_mix": 1.9234982877969742, "l_a": 2.0377286821603775, "l_c": 1.9695024937391281, "z_loss": 45.403221130371094, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16958561539649963, "tokens_per_second": 1409511.0257029533, "eta_seconds": 4913.2784814834595, "data_wait_seconds": 1.6361474990844727e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 59280, "total_steps": 72479, "tokens": 31079792640, "loss": 3.096856415271759, "l_mix": 1.9054198414087296, "l_a": 2.0191565454006195, "l_c": 1.9522983580827713, "z_loss": 46.84049081802368, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19298811256885529, "tokens_per_second": 1413883.0892125077, "eta_seconds": 4894.377310824394, "data_wait_seconds": 2.294778823852539e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 59290, "total_steps": 72479, "tokens": 31085035520, "loss": 3.0760926008224487, "l_mix": 1.8929139375686646, "l_a": 2.002693235874176, "l_c": 1.941235512495041, "z_loss": 44.59431838989258, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17704804241657257, "tokens_per_second": 1321339.8854947016, "eta_seconds": 5233.198897504807, "data_wait_seconds": 1.7762184143066406e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 59300, "total_steps": 72479, "tokens": 31090278400, "loss": 4.399045169353485, "l_mix": 2.7277073562145233, "l_a": 2.807621717453003, "l_c": 2.7635043561458588, "z_loss": 41.64463233947754, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13770662248134613, "tokens_per_second": 1430674.412103781, "eta_seconds": 4829.604481315612, "data_wait_seconds": 1.677870750427246e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16422608, "gate_a2b/L1_absmax": 0.69465065, "gate_a2b/L1_rms": 0.17463641, "gate_a2b/L3_absmean": 0.15565403, "gate_a2b/L3_absmax": 0.50082737, "gate_a2b/L3_rms": 0.16138475, "gate_a2b/L5_absmean": 0.22998495, "gate_a2b/L5_absmax": 0.73216885, "gate_a2b/L5_rms": 0.23601101, "gate_c2b/L1_absmean": 0.03286479, "gate_c2b/L1_absmax": 0.63435447, "gate_c2b/L1_rms": 0.06877004, "gate_c2b/L3_absmean": 0.17168504, "gate_c2b/L3_absmax": 0.39362076, "gate_c2b/L3_rms": 0.17530414, "gate_c2b/L5_absmean": 0.48596975, "gate_c2b/L5_absmax": 0.68583435, "gate_c2b/L5_rms": 0.49569589, "gate_b2a/L7_absmean": 0.05080832, "gate_b2a/L7_absmax": 0.33253324, "gate_b2a/L7_rms": 0.05661425, "gate_b2a/L11_absmean": 0.08197117, "gate_b2a/L11_absmax": 0.61008906, "gate_b2a/L11_rms": 0.09118009, "gate_b2a/L14_absmean": 0.21360157, "gate_b2a/L14_absmax": 0.82030505, "gate_b2a/L14_rms": 0.22302307, "gate_b2c/L3_absmean": 0.03488691, "gate_b2c/L3_absmax": 0.54856855, "gate_b2c/L3_rms": 0.04540596, "gate_b2c/L5_absmean": 0.0367559, "gate_b2c/L5_absmax": 0.52788699, "gate_b2c/L5_rms": 0.04597849, "gate_b2c/L6_absmean": 0.05338046, "gate_b2c/L6_absmax": 0.34318054, "gate_b2c/L6_rms": 0.05946128}, "streams": {"cos_a2b/round0": 0.351953, "cos_c2b/round0": 0.251774, "cos_b2a/round0": 0.158215, "cos_b2c/round0": 0.143987, "rms_a/round0": 7.439742, "rms_b/round0": 12.97353, "rms_c/round0": 4.85007, "cos_a2b/round1": 0.41057, "cos_c2b/round1": 0.310959, "cos_b2a/round1": 0.255556, "cos_b2c/round1": 0.10666, "rms_a/round1": 14.596608, "rms_b/round1": 29.760487, "rms_c/round1": 10.364073, "cos_a2b/round2": 0.536419, "cos_c2b/round2": 0.527891, "cos_b2a/round2": 0.100824, "cos_b2c/round2": 0.186246, "rms_a/round2": 41.562454, "rms_b/round2": 80.25634, "rms_c/round2": 19.92911}, "alpha_mean": 0.44350885203323287, "alpha_hist": [0.0083, 0.05283, 0.17288, 0.19531, 0.19479, 0.17107, 0.12804, 0.06784, 0.00883, 0.00011], "alpha_by_type": {"special": {"share": 0.00105, "mean": 0.54751, "std": 0.11114}, "whitespace": {"share": 0.022, "mean": 0.5261, "std": 0.11151}, "digit": {"share": 0.02238, "mean": 0.44074, "std": 0.12031}, "word_start": {"share": 0.57678, "mean": 0.46529, "std": 0.16113}, "word_cont": {"share": 0.23438, "mean": 0.32854, "std": 0.12838}, "punct": {"share": 0.1434, "mean": 0.53081, "std": 0.17467}}, "alpha_p05": 0.19406, "alpha_p50": 0.44903, "alpha_p95": 0.71859}
{"step": 59310, "total_steps": 72479, "tokens": 31095521280, "loss": 4.484727680683136, "l_mix": 2.7809953689575195, "l_a": 2.8602063953876495, "l_c": 2.818901002407074, "z_loss": 40.84695529937744, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1459866464138031, "tokens_per_second": 1286381.277244349, "eta_seconds": 5367.264584875106, "data_wait_seconds": 1.8268823623657227e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 59320, "total_steps": 72479, "tokens": 31100764160, "loss": 4.3617053627967834, "l_mix": 2.7035220861434937, "l_a": 2.786933958530426, "l_c": 2.740343362092972, "z_loss": 41.85472774505615, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13452254235744476, "tokens_per_second": 1266515.136971962, "eta_seconds": 5447.314122509956, "data_wait_seconds": 1.9222497940063477e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 59330, "total_steps": 72479, "tokens": 31106007040, "loss": 4.445726931095123, "l_mix": 2.755816698074341, "l_a": 2.8406839072704315, "l_c": 2.7923495769500732, "z_loss": 41.57896423339844, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12099147588014603, "tokens_per_second": 1397971.3009838525, "eta_seconds": 4931.333645510674, "data_wait_seconds": 1.5616416931152344e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 59340, "total_steps": 72479, "tokens": 31111249920, "loss": 4.643695414066315, "l_mix": 2.8805659115314484, "l_a": 2.960896849632263, "l_c": 2.916200965642929, "z_loss": 40.538371086120605, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12397022545337677, "tokens_per_second": 1455609.3524407346, "eta_seconds": 4732.464806199074, "data_wait_seconds": 2.8759241104125977e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 59350, "total_steps": 72479, "tokens": 31116492800, "loss": 4.552126109600067, "l_mix": 2.823619246482849, "l_a": 2.9015088081359863, "l_c": 2.8601805567741394, "z_loss": 40.6500678062439, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1292039006948471, "tokens_per_second": 1468954.3917878368, "eta_seconds": 4685.902564764023, "data_wait_seconds": 1.704692840576172e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 59360, "total_steps": 72479, "tokens": 31121735680, "loss": 4.71179860830307, "l_mix": 2.92338103055954, "l_a": 3.002418965101242, "l_c": 2.9589724838733673, "z_loss": 40.33455038070679, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1589018553495407, "tokens_per_second": 1416792.1764725817, "eta_seconds": 4854.723498773575, "data_wait_seconds": 2.1159648895263672e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 59370, "total_steps": 72479, "tokens": 31126978560, "loss": 4.662178993225098, "l_mix": 2.892097234725952, "l_a": 2.9696205258369446, "l_c": 2.930651992559433, "z_loss": 40.53643798828125, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1959504336118698, "tokens_per_second": 946273.7586009308, "eta_seconds": 7263.111049556732, "data_wait_seconds": 1.8328428268432617e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 59380, "total_steps": 72479, "tokens": 31132221440, "loss": 3.062257409095764, "l_mix": 1.885065644979477, "l_a": 1.9933554828166962, "l_c": 1.930617019534111, "z_loss": 45.156776428222656, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.192634716629982, "tokens_per_second": 1420638.1430989339, "eta_seconds": 4834.199718880654, "data_wait_seconds": 1.665949821472168e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 59390, "total_steps": 72479, "tokens": 31137464320, "loss": 3.098631590604782, "l_mix": 1.907059133052826, "l_a": 2.0168933272361755, "l_c": 1.9550144970417023, "z_loss": 45.357744216918945, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13491635024547577, "tokens_per_second": 1433379.179831584, "eta_seconds": 4787.57172460556, "data_wait_seconds": 2.3066997528076172e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 59400, "total_steps": 72479, "tokens": 31142707200, "loss": 3.0968192517757416, "l_mix": 1.9052916914224625, "l_a": 2.019176334142685, "l_c": 1.952582135796547, "z_loss": 47.686853885650635, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.22802869975566864, "tokens_per_second": 1405355.488901324, "eta_seconds": 4879.308335971832, "data_wait_seconds": 2.1636486053466797e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16440943, "gate_a2b/L1_absmax": 0.69272912, "gate_a2b/L1_rms": 0.17480154, "gate_a2b/L3_absmean": 0.15581833, "gate_a2b/L3_absmax": 0.50057006, "gate_a2b/L3_rms": 0.1615254, "gate_a2b/L5_absmean": 0.22929187, "gate_a2b/L5_absmax": 0.73218787, "gate_a2b/L5_rms": 0.23533306, "gate_c2b/L1_absmean": 0.0330577, "gate_c2b/L1_absmax": 0.63295108, "gate_c2b/L1_rms": 0.06893118, "gate_c2b/L3_absmean": 0.17132586, "gate_c2b/L3_absmax": 0.39291596, "gate_c2b/L3_rms": 0.17497003, "gate_c2b/L5_absmean": 0.48519108, "gate_c2b/L5_absmax": 0.68447047, "gate_c2b/L5_rms": 0.49484292, "gate_b2a/L7_absmean": 0.05063261, "gate_b2a/L7_absmax": 0.33281586, "gate_b2a/L7_rms": 0.05644831, "gate_b2a/L11_absmean": 0.08203316, "gate_b2a/L11_absmax": 0.60882461, "gate_b2a/L11_rms": 0.09118688, "gate_b2a/L14_absmean": 0.2133178, "gate_b2a/L14_absmax": 0.82113588, "gate_b2a/L14_rms": 0.22270782, "gate_b2c/L3_absmean": 0.0348379, "gate_b2c/L3_absmax": 0.54996181, "gate_b2c/L3_rms": 0.04544088, "gate_b2c/L5_absmean": 0.03697793, "gate_b2c/L5_absmax": 0.53013408, "gate_b2c/L5_rms": 0.04616164, "gate_b2c/L6_absmean": 0.05336864, "gate_b2c/L6_absmax": 0.34297639, "gate_b2c/L6_rms": 0.0594762}, "streams": {"cos_a2b/round0": 0.354179, "cos_c2b/round0": 0.224734, "cos_b2a/round0": 0.147654, "cos_b2c/round0": 0.125203, "rms_a/round0": 7.800465, "rms_b/round0": 13.103884, "rms_c/round0": 4.912298, "cos_a2b/round1": 0.409787, "cos_c2b/round1": 0.306948, "cos_b2a/round1": 0.235983, "cos_b2c/round1": 0.121461, "rms_a/round1": 15.654344, "rms_b/round1": 31.345192, "rms_c/round1": 11.072945, "cos_a2b/round2": 0.523679, "cos_c2b/round2": 0.509334, "cos_b2a/round2": 0.093576, "cos_b2c/round2": 0.173208, "rms_a/round2": 43.010441, "rms_b/round2": 82.496513, "rms_c/round2": 21.701906}, "alpha_mean": 0.4364160672628259, "alpha_hist": [0.03139, 0.13741, 0.17107, 0.14621, 0.12212, 0.12122, 0.11229, 0.09595, 0.05696, 0.0054], "alpha_by_type": {"special": {"share": 0.00116, "mean": 0.7101, "std": 0.12016}, "whitespace": {"share": 0.04503, "mean": 0.63504, "std": 0.15848}, "digit": {"share": 0.00703, "mean": 0.36691, "std": 0.19491}, "word_start": {"share": 0.57613, "mean": 0.43695, "std": 0.21102}, "word_cont": {"share": 0.24358, "mean": 0.28977, "std": 0.15115}, "punct": {"share": 0.12708, "mean": 0.64607, "std": 0.18015}}, "alpha_p05": 0.10053, "alpha_p50": 0.38376, "alpha_p95": 0.80716}
{"step": 59410, "total_steps": 72479, "tokens": 31147950080, "loss": 4.4359883069992065, "l_mix": 2.7495559751987457, "l_a": 2.8323369920253754, "l_c": 2.7891036570072174, "z_loss": 41.64199686050415, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17415273189544678, "tokens_per_second": 1422445.98323485, "eta_seconds": 4816.998292207718, "data_wait_seconds": 1.7076730728149414e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 59420, "total_steps": 72479, "tokens": 31153192960, "loss": 4.504398167133331, "l_mix": 2.792297124862671, "l_a": 2.876759171485901, "l_c": 2.830243766307831, "z_loss": 41.52881622314453, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.171145498752594, "tokens_per_second": 1478363.3860469519, "eta_seconds": 4631.254437589645, "data_wait_seconds": 1.633167266845703e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 59430, "total_steps": 72479, "tokens": 31158435840, "loss": 4.454277276992798, "l_mix": 2.7603655755519867, "l_a": 2.849267899990082, "l_c": 2.7971041798591614, "z_loss": 41.861756801605225, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.160696342587471, "tokens_per_second": 1329604.2262658936, "eta_seconds": 5145.466580843925, "data_wait_seconds": 1.7404556274414062e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 59440, "total_steps": 72479, "tokens": 31163678720, "loss": 4.382610619068146, "l_mix": 2.7159940004348755, "l_a": 2.8028408885002136, "l_c": 2.7525478303432465, "z_loss": 41.23534679412842, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1206403598189354, "tokens_per_second": 1471233.5768695357, "eta_seconds": 4646.570972466468, "data_wait_seconds": 1.7404556274414062e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 59450, "total_steps": 72479, "tokens": 31168921600, "loss": 3.0768794417381287, "l_mix": 1.893129825592041, "l_a": 2.005289778113365, "l_c": 1.94054214656353, "z_loss": 45.112850189208984, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18216203153133392, "tokens_per_second": 1357195.730952956, "eta_seconds": 5033.134275484085, "data_wait_seconds": 3.629922866821289e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 59460, "total_steps": 72479, "tokens": 31174164480, "loss": 3.078711360692978, "l_mix": 1.8937412947416306, "l_a": 2.0103919506073, "l_c": 1.939508005976677, "z_loss": 47.1626992225647, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2470606416463852, "tokens_per_second": 1440278.7781404017, "eta_seconds": 4739.155763173103, "data_wait_seconds": 1.6242265701293945e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 59470, "total_steps": 72479, "tokens": 31179407360, "loss": 3.1496223509311676, "l_mix": 1.9390400797128677, "l_a": 2.050331711769104, "l_c": 1.9849424362182617, "z_loss": 46.14664936065674, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2355446219444275, "tokens_per_second": 1469696.7020207748, "eta_seconds": 4640.727969670295, "data_wait_seconds": 1.570582389831543e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 59480, "total_steps": 72479, "tokens": 31184650240, "loss": 4.66954505443573, "l_mix": 2.896883189678192, "l_a": 2.9754767417907715, "l_c": 2.933395951986313, "z_loss": 39.90316867828369, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1919950395822525, "tokens_per_second": 1368843.835027864, "eta_seconds": 4978.814629983902, "data_wait_seconds": 1.850724220275879e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 59490, "total_steps": 72479, "tokens": 31189893120, "loss": 3.0476673543453217, "l_mix": 1.8756885826587677, "l_a": 1.9858208000659943, "l_c": 1.920774906873703, "z_loss": 46.021831035614014, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14363054931163788, "tokens_per_second": 1460582.429181265, "eta_seconds": 4662.507706475258, "data_wait_seconds": 1.6421079635620117e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 59500, "total_steps": 72479, "tokens": 31195136000, "loss": 3.0952320992946625, "l_mix": 1.9036148637533188, "l_a": 2.0171990543603897, "l_c": 1.9548583924770355, "z_loss": 47.364251136779785, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.21056360006332397, "tokens_per_second": 1446769.0387278672, "eta_seconds": 4703.400314664841, "data_wait_seconds": 1.800060272216797e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16318806, "gate_a2b/L1_absmax": 0.69616187, "gate_a2b/L1_rms": 0.17388004, "gate_a2b/L3_absmean": 0.15575261, "gate_a2b/L3_absmax": 0.50182432, "gate_a2b/L3_rms": 0.16149494, "gate_a2b/L5_absmean": 0.22899167, "gate_a2b/L5_absmax": 0.72986203, "gate_a2b/L5_rms": 0.23511523, "gate_c2b/L1_absmean": 0.03380898, "gate_c2b/L1_absmax": 0.63027442, "gate_c2b/L1_rms": 0.06940286, "gate_c2b/L3_absmean": 0.17157838, "gate_c2b/L3_absmax": 0.38973847, "gate_c2b/L3_rms": 0.17526682, "gate_c2b/L5_absmean": 0.48356801, "gate_c2b/L5_absmax": 0.68159509, "gate_c2b/L5_rms": 0.4930588, "gate_b2a/L7_absmean": 0.05067454, "gate_b2a/L7_absmax": 0.33347556, "gate_b2a/L7_rms": 0.05647023, "gate_b2a/L11_absmean": 0.0826312, "gate_b2a/L11_absmax": 0.61670107, "gate_b2a/L11_rms": 0.09198213, "gate_b2a/L14_absmean": 0.21269605, "gate_b2a/L14_absmax": 0.82123226, "gate_b2a/L14_rms": 0.22220746, "gate_b2c/L3_absmean": 0.03490697, "gate_b2c/L3_absmax": 0.551916, "gate_b2c/L3_rms": 0.04554808, "gate_b2c/L5_absmean": 0.03686427, "gate_b2c/L5_absmax": 0.52789718, "gate_b2c/L5_rms": 0.0460789, "gate_b2c/L6_absmean": 0.05356241, "gate_b2c/L6_absmax": 0.34474075, "gate_b2c/L6_rms": 0.05979544}, "streams": {"cos_a2b/round0": 0.353052, "cos_c2b/round0": 0.214724, "cos_b2a/round0": 0.145227, "cos_b2c/round0": 0.121024, "rms_a/round0": 7.806254, "rms_b/round0": 13.009576, "rms_c/round0": 5.0074, "cos_a2b/round1": 0.406217, "cos_c2b/round1": 0.307894, "cos_b2a/round1": 0.23935, "cos_b2c/round1": 0.121208, "rms_a/round1": 15.439739, "rms_b/round1": 31.520121, "rms_c/round1": 11.045239, "cos_a2b/round2": 0.51734, "cos_c2b/round2": 0.507309, "cos_b2a/round2": 0.092052, "cos_b2c/round2": 0.172911, "rms_a/round2": 42.430241, "rms_b/round2": 82.67054, "rms_c/round2": 21.083782}, "alpha_mean": 0.41104812713631134, "alpha_hist": [0.04312, 0.17256, 0.18236, 0.13538, 0.11092, 0.10904, 0.10533, 0.08859, 0.04939, 0.00331], "alpha_by_type": {"special": {"share": 0.00113, "mean": 0.65559, "std": 0.10041}, "whitespace": {"share": 0.03307, "mean": 0.63955, "std": 0.15517}, "digit": {"share": 0.0087, "mean": 0.5248, "std": 0.22441}, "word_start": {"share": 0.57979, "mean": 0.4126, "std": 0.21378}, "word_cont": {"share": 0.25697, "mean": 0.26769, "std": 0.14742}, "punct": {"share": 0.12035, "mean": 0.63639, "std": 0.18341}}, "alpha_p05": 0.10448, "alpha_p50": 0.35892, "alpha_p95": 0.78465}
{"step": 59510, "total_steps": 72479, "tokens": 31200378880, "loss": 3.1241810619831085, "l_mix": 1.9218988418579102, "l_a": 2.0368980914354324, "l_c": 1.9707091450691223, "z_loss": 46.42919158935547, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17986199259757996, "tokens_per_second": 1434285.0077136846, "eta_seconds": 4740.683361697197, "data_wait_seconds": 1.868605613708496e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 59520, "total_steps": 72479, "tokens": 31205621760, "loss": 4.399948418140411, "l_mix": 2.7266264855861664, "l_a": 2.8123599588871, "l_c": 2.7653796672821045, "z_loss": 42.827232360839844, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12651701271533966, "tokens_per_second": 1337668.852629261, "eta_seconds": 5079.170512676239, "data_wait_seconds": 2.2232532501220703e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 59530, "total_steps": 72479, "tokens": 31210864640, "loss": 4.490785360336304, "l_mix": 2.7835032045841217, "l_a": 2.869529575109482, "l_c": 2.8214105665683746, "z_loss": 41.90614891052246, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11888818442821503, "tokens_per_second": 1398219.6544570974, "eta_seconds": 4855.464082741738, "data_wait_seconds": 2.5331974029541016e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 59540, "total_steps": 72479, "tokens": 31216107520, "loss": 4.499897241592407, "l_mix": 2.7896401584148407, "l_a": 2.876692920923233, "l_c": 2.8241637349128723, "z_loss": 41.78314781188965, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14390183985233307, "tokens_per_second": 1388854.2784229906, "eta_seconds": 4884.430668783188, "data_wait_seconds": 1.4692544937133789e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 59550, "total_steps": 72479, "tokens": 31221350400, "loss": 4.403327405452728, "l_mix": 2.7295934557914734, "l_a": 2.8135726153850555, "l_c": 2.7655406296253204, "z_loss": 42.1940655708313, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13314901292324066, "tokens_per_second": 1258967.042808148, "eta_seconds": 5384.191421627998, "data_wait_seconds": 2.518296241760254e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 59560, "total_steps": 72479, "tokens": 31226593280, "loss": 3.0620067417621613, "l_mix": 1.8835377395153046, "l_a": 1.9964735805988312, "l_c": 1.931756466627121, "z_loss": 46.412126541137695, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13747148215770721, "tokens_per_second": 1431509.7252461289, "eta_seconds": 4731.561757874489, "data_wait_seconds": 3.331899642944336e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 59570, "total_steps": 72479, "tokens": 31231836160, "loss": 3.0879831314086914, "l_mix": 1.9004372656345367, "l_a": 2.011587619781494, "l_c": 1.9468985944986343, "z_loss": 45.75965642929077, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14763104915618896, "tokens_per_second": 1306274.7366417, "eta_seconds": 5181.171772027016, "data_wait_seconds": 1.7583370208740234e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 59580, "total_steps": 72479, "tokens": 31237079040, "loss": 3.0814981758594513, "l_mix": 1.8969981968402863, "l_a": 2.0077628940343857, "l_c": 1.9405702203512192, "z_loss": 46.65966749191284, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2598866820335388, "tokens_per_second": 1410811.7564298718, "eta_seconds": 4793.545900917054, "data_wait_seconds": 1.728534698486328e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 59590, "total_steps": 72479, "tokens": 31242321920, "loss": 3.1514331996440887, "l_mix": 1.9398119896650314, "l_a": 2.0505908131599426, "l_c": 1.9881462901830673, "z_loss": 45.84229516983032, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16873997449874878, "tokens_per_second": 1432294.9829184834, "eta_seconds": 4717.98624765873, "data_wait_seconds": 2.2023916244506836e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 59600, "total_steps": 72479, "tokens": 31247564800, "loss": 4.5923861265182495, "l_mix": 2.849624752998352, "l_a": 2.928670048713684, "l_c": 2.8805344104766846, "z_loss": 41.03576326370239, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.21056215465068817, "tokens_per_second": 1322503.9043534878, "eta_seconds": 5105.697707033157, "data_wait_seconds": 1.558661460876465e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.16332409, "gate_a2b/L1_absmax": 0.70981264, "gate_a2b/L1_rms": 0.17393756, "gate_a2b/L3_absmean": 0.15532291, "gate_a2b/L3_absmax": 0.51184589, "gate_a2b/L3_rms": 0.16118968, "gate_a2b/L5_absmean": 0.22895661, "gate_a2b/L5_absmax": 0.74207056, "gate_a2b/L5_rms": 0.23509069, "gate_c2b/L1_absmean": 0.03353391, "gate_c2b/L1_absmax": 0.64319074, "gate_c2b/L1_rms": 0.0702121, "gate_c2b/L3_absmean": 0.1716844, "gate_c2b/L3_absmax": 0.3969323, "gate_c2b/L3_rms": 0.17542367, "gate_c2b/L5_absmean": 0.483634, "gate_c2b/L5_absmax": 0.67880166, "gate_c2b/L5_rms": 0.49313119, "gate_b2a/L7_absmean": 0.05083948, "gate_b2a/L7_absmax": 0.3338986, "gate_b2a/L7_rms": 0.05667767, "gate_b2a/L11_absmean": 0.08254582, "gate_b2a/L11_absmax": 0.61862874, "gate_b2a/L11_rms": 0.09197672, "gate_b2a/L14_absmean": 0.21370013, "gate_b2a/L14_absmax": 0.82258022, "gate_b2a/L14_rms": 0.2231781, "gate_b2c/L3_absmean": 0.03475135, "gate_b2c/L3_absmax": 0.55762124, "gate_b2c/L3_rms": 0.04565199, "gate_b2c/L5_absmean": 0.03668766, "gate_b2c/L5_absmax": 0.53953689, "gate_b2c/L5_rms": 0.04625883, "gate_b2c/L6_absmean": 0.05344154, "gate_b2c/L6_absmax": 0.34326321, "gate_b2c/L6_rms": 0.05967033}, "streams": {"cos_a2b/round0": 0.354023, "cos_c2b/round0": 0.25863, "cos_b2a/round0": 0.16779, "cos_b2c/round0": 0.148793, "rms_a/round0": 7.496055, "rms_b/round0": 12.843051, "rms_c/round0": 4.825971, "cos_a2b/round1": 0.415349, "cos_c2b/round1": 0.312746, "cos_b2a/round1": 0.256435, "cos_b2c/round1": 0.116583, "rms_a/round1": 14.611442, "rms_b/round1": 29.648315, "rms_c/round1": 10.339369, "cos_a2b/round2": 0.551619, "cos_c2b/round2": 0.516218, "cos_b2a/round2": 0.102366, "cos_b2c/round2": 0.190062, "rms_a/round2": 41.697529, "rms_b/round2": 80.578949, "rms_c/round2": 20.02869}, "alpha_mean": 0.45888479791337033, "alpha_hist": [0.00447, 0.04353, 0.14764, 0.19397, 0.1972, 0.19292, 0.13628, 0.0746, 0.00925, 0.00014], "alpha_by_type": {"special": {"share": 0.00095, "mean": 0.55061, "std": 0.14072}, "whitespace": {"share": 0.05161, "mean": 0.55211, "std": 0.11816}, "digit": {"share": 0.02815, "mean": 0.47475, "std": 0.1029}, "word_start": {"share": 0.5843, "mean": 0.47263, "std": 0.15718}, "word_cont": {"share": 0.23222, "mean": 0.3417, "std": 0.12263}, "punct": {"share": 0.10277, "mean": 0.59351, "std": 0.15534}}, "alpha_p05": 0.19962, "alpha_p50": 0.45556, "alpha_p95": 0.73867}
{"step": 59610, "total_steps": 72479, "tokens": 31252807680, "loss": 4.6091936230659485, "l_mix": 2.8598862290382385, "l_a": 2.934556692838669, "l_c": 2.896467685699463, "z_loss": 40.86478090286255, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.217013418674469, "tokens_per_second": 1334716.8287997649, "eta_seconds": 5055.051473402977, "data_wait_seconds": 2.1636486053466797e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 59620, "total_steps": 72479, "tokens": 31258050560, "loss": 3.079545706510544, "l_mix": 1.8951665759086609, "l_a": 2.00696037709713, "l_c": 1.940969780087471, "z_loss": 47.40009355545044, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2234691083431244, "tokens_per_second": 1417380.638380457, "eta_seconds": 4756.534137296677, "data_wait_seconds": 3.165006637573242e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 59630, "total_steps": 72479, "tokens": 31263293440, "loss": 4.482165515422821, "l_mix": 2.7791598439216614, "l_a": 2.8605731427669525, "l_c": 2.8161120414733887, "z_loss": 41.22060203552246, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14231227338314056, "tokens_per_second": 1454924.9021571698, "eta_seconds": 4630.188473653793, "data_wait_seconds": 2.1696090698242188e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 59640, "total_steps": 72479, "tokens": 31268536320, "loss": 4.430677056312561, "l_mix": 2.746722459793091, "l_a": 2.8309595584869385, "l_c": 2.782222092151642, "z_loss": 41.65502738952637, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14705362915992737, "tokens_per_second": 1338619.0403864824, "eta_seconds": 5028.565580582619, "data_wait_seconds": 2.378225326538086e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 59650, "total_steps": 72479, "tokens": 31273779200, "loss": 4.4088698625564575, "l_mix": 2.733279228210449, "l_a": 2.8173999190330505, "l_c": 2.7679018676280975, "z_loss": 42.69466495513916, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17721210420131683, "tokens_per_second": 1425699.571226199, "eta_seconds": 4717.747615098953, "data_wait_seconds": 2.5004148483276367e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 59660, "total_steps": 72479, "tokens": 31279022080, "loss": 4.38692045211792, "l_mix": 2.7187134325504303, "l_a": 2.80510476231575, "l_c": 2.755585163831711, "z_loss": 42.761672496795654, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15039144456386566, "tokens_per_second": 1398228.1892869384, "eta_seconds": 4806.688867735863, "data_wait_seconds": 1.9371509552001953e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 59670, "total_steps": 72479, "tokens": 31284264960, "loss": 3.116452604532242, "l_mix": 1.917755275964737, "l_a": 2.031397670507431, "l_c": 1.9642598330974579, "z_loss": 45.93356800079346, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14996974170207977, "tokens_per_second": 1447860.491427229, "eta_seconds": 4638.295631217957, "data_wait_seconds": 1.5616416931152344e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 59680, "total_steps": 72479, "tokens": 31289507840, "loss": 3.119342416524887, "l_mix": 1.9193768054246902, "l_a": 2.0327694565057755, "l_c": 1.9671157598495483, "z_loss": 46.71932411193848, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18266916275024414, "tokens_per_second": 1408614.374051125, "eta_seconds": 4763.803518986702, "data_wait_seconds": 2.8818845748901367e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 59690, "total_steps": 72479, "tokens": 31294750720, "loss": 3.073041260242462, "l_mix": 1.8914409279823303, "l_a": 1.999662309885025, "l_c": 1.9390052258968353, "z_loss": 45.7875599861145, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15498529374599457, "tokens_per_second": 1413536.2733111298, "eta_seconds": 4743.507017540931, "data_wait_seconds": 2.008676528930664e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 59700, "total_steps": 72479, "tokens": 31299993600, "loss": 3.1389365792274475, "l_mix": 1.9324941635131836, "l_a": 2.043348342180252, "l_c": 1.9781262874603271, "z_loss": 45.91297721862793, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.23951610922813416, "tokens_per_second": 1418162.9055019496, "eta_seconds": 4724.33478975296, "data_wait_seconds": 3.263354301452637e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.16336262, "gate_a2b/L1_absmax": 0.70613784, "gate_a2b/L1_rms": 0.17383042, "gate_a2b/L3_absmean": 0.15539798, "gate_a2b/L3_absmax": 0.51000315, "gate_a2b/L3_rms": 0.1612217, "gate_a2b/L5_absmean": 0.22852068, "gate_a2b/L5_absmax": 0.73855168, "gate_a2b/L5_rms": 0.23459402, "gate_c2b/L1_absmean": 0.03343922, "gate_c2b/L1_absmax": 0.64012456, "gate_c2b/L1_rms": 0.06958275, "gate_c2b/L3_absmean": 0.17119871, "gate_c2b/L3_absmax": 0.39519173, "gate_c2b/L3_rms": 0.17494725, "gate_c2b/L5_absmean": 0.48365971, "gate_c2b/L5_absmax": 0.68460757, "gate_c2b/L5_rms": 0.49333116, "gate_b2a/L7_absmean": 0.05101462, "gate_b2a/L7_absmax": 0.33309823, "gate_b2a/L7_rms": 0.05677084, "gate_b2a/L11_absmean": 0.08216576, "gate_b2a/L11_absmax": 0.61652106, "gate_b2a/L11_rms": 0.09155653, "gate_b2a/L14_absmean": 0.21435703, "gate_b2a/L14_absmax": 0.82167071, "gate_b2a/L14_rms": 0.22371563, "gate_b2c/L3_absmean": 0.03479891, "gate_b2c/L3_absmax": 0.55511981, "gate_b2c/L3_rms": 0.04557411, "gate_b2c/L5_absmean": 0.03681556, "gate_b2c/L5_absmax": 0.53270012, "gate_b2c/L5_rms": 0.04613553, "gate_b2c/L6_absmean": 0.05328254, "gate_b2c/L6_absmax": 0.34167928, "gate_b2c/L6_rms": 0.0594037}, "streams": {"cos_a2b/round0": 0.352374, "cos_c2b/round0": 0.220509, "cos_b2a/round0": 0.14714, "cos_b2c/round0": 0.123892, "rms_a/round0": 7.822875, "rms_b/round0": 13.167712, "rms_c/round0": 5.023526, "cos_a2b/round1": 0.407673, "cos_c2b/round1": 0.30395, "cos_b2a/round1": 0.242826, "cos_b2c/round1": 0.119114, "rms_a/round1": 15.548518, "rms_b/round1": 31.612162, "rms_c/round1": 11.158672, "cos_a2b/round2": 0.515691, "cos_c2b/round2": 0.512338, "cos_b2a/round2": 0.087837, "cos_b2c/round2": 0.172531, "rms_a/round2": 42.711849, "rms_b/round2": 82.933548, "rms_c/round2": 21.732239}, "alpha_mean": 0.4146020820074341, "alpha_hist": [0.02866, 0.15186, 0.19475, 0.15596, 0.12199, 0.11287, 0.10211, 0.08595, 0.04396, 0.00189], "alpha_by_type": {"special": {"share": 0.00116, "mean": 0.59042, "std": 0.08972}, "whitespace": {"share": 0.03876, "mean": 0.60534, "std": 0.16809}, "digit": {"share": 0.00916, "mean": 0.3608, "std": 0.16819}, "word_start": {"share": 0.5629, "mean": 0.42234, "std": 0.20501}, "word_cont": {"share": 0.26851, "mean": 0.2784, "std": 0.13794}, "punct": {"share": 0.11952, "mean": 0.6247, "std": 0.18238}}, "alpha_p05": 0.12765, "alpha_p50": 0.36116, "alpha_p95": 0.79881}
{"step": 59710, "total_steps": 72479, "tokens": 31305236480, "loss": 2.8353647887706757, "l_mix": 1.7545087784528732, "l_a": 1.8227310478687286, "l_c": 1.78012216091156, "z_loss": 52.91181755065918, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18423698842525482, "tokens_per_second": 1319251.5798467526, "eta_seconds": 5074.569228696822, "data_wait_seconds": 2.485513687133789e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 59720, "total_steps": 72479, "tokens": 31310479360, "loss": 4.368073999881744, "l_mix": 2.707012861967087, "l_a": 2.7957732379436493, "l_c": 2.7410965859889984, "z_loss": 43.02640771865845, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2590259313583374, "tokens_per_second": 1468754.1426757665, "eta_seconds": 4554.4658548593525, "data_wait_seconds": 1.895427703857422e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 59730, "total_steps": 72479, "tokens": 31315722240, "loss": 4.3420597314834595, "l_mix": 2.691311299800873, "l_a": 2.7745141088962555, "l_c": 2.7279804944992065, "z_loss": 41.98450469970703, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13621202111244202, "tokens_per_second": 1438983.9032814114, "eta_seconds": 4645.0468950748445, "data_wait_seconds": 2.4139881134033203e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 59740, "total_steps": 72479, "tokens": 31320965120, "loss": 4.464172542095184, "l_mix": 2.76755490899086, "l_a": 2.852195769548416, "l_c": 2.803196460008621, "z_loss": 41.3740873336792, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12983113527297974, "tokens_per_second": 1438679.63158153, "eta_seconds": 4642.385062932968, "data_wait_seconds": 2.0444393157958984e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 59750, "total_steps": 72479, "tokens": 31326208000, "loss": 4.312161594629288, "l_mix": 2.673422694206238, "l_a": 2.7553586661815643, "l_c": 2.7071041464805603, "z_loss": 41.24848699569702, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13078825175762177, "tokens_per_second": 1411413.1058185997, "eta_seconds": 4728.354812979698, "data_wait_seconds": 1.8894672393798828e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 59760, "total_steps": 72479, "tokens": 31331450880, "loss": 4.3971720933914185, "l_mix": 2.7264353036880493, "l_a": 2.807775616645813, "l_c": 2.7613469660282135, "z_loss": 41.34469938278198, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1406233310699463, "tokens_per_second": 1441653.482059176, "eta_seconds": 4625.535300254822, "data_wait_seconds": 2.0325183868408203e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 59770, "total_steps": 72479, "tokens": 31336693760, "loss": 4.448688209056854, "l_mix": 2.7580886483192444, "l_a": 2.8411888480186462, "l_c": 2.794142872095108, "z_loss": 41.037954330444336, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14495323598384857, "tokens_per_second": 1281420.178216332, "eta_seconds": 5199.837106728553, "data_wait_seconds": 3.11434268951416e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 59780, "total_steps": 72479, "tokens": 31341936640, "loss": 3.088228553533554, "l_mix": 1.8998915702104568, "l_a": 2.0171165764331818, "l_c": 1.9440065026283264, "z_loss": 46.372100830078125, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.46925991773605347, "tokens_per_second": 1388935.4213454316, "eta_seconds": 4793.5513845205305, "data_wait_seconds": 1.683831214904785e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 59790, "total_steps": 72479, "tokens": 31347179520, "loss": 3.121843934059143, "l_mix": 1.9212452620267868, "l_a": 2.0310314148664474, "l_c": 1.9709641486406326, "z_loss": 45.89957666397095, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2213694006204605, "tokens_per_second": 1391423.6914061166, "eta_seconds": 4781.211124324798, "data_wait_seconds": 2.3126602172851562e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 59800, "total_steps": 72479, "tokens": 31352422400, "loss": 3.092396378517151, "l_mix": 1.9030133485794067, "l_a": 2.0130417943000793, "l_c": 1.9515682458877563, "z_loss": 44.73466110229492, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17913098633289337, "tokens_per_second": 1432198.1544390868, "eta_seconds": 4641.430050301552, "data_wait_seconds": 2.0384788513183594e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16308981, "gate_a2b/L1_absmax": 0.70472986, "gate_a2b/L1_rms": 0.17363159, "gate_a2b/L3_absmean": 0.15512921, "gate_a2b/L3_absmax": 0.49779615, "gate_a2b/L3_rms": 0.16094379, "gate_a2b/L5_absmean": 0.2280774, "gate_a2b/L5_absmax": 0.73596931, "gate_a2b/L5_rms": 0.23406835, "gate_c2b/L1_absmean": 0.03401947, "gate_c2b/L1_absmax": 0.63792944, "gate_c2b/L1_rms": 0.06998171, "gate_c2b/L3_absmean": 0.17162836, "gate_c2b/L3_absmax": 0.39015985, "gate_c2b/L3_rms": 0.17533554, "gate_c2b/L5_absmean": 0.48358583, "gate_c2b/L5_absmax": 0.67797792, "gate_c2b/L5_rms": 0.49329951, "gate_b2a/L7_absmean": 0.05063801, "gate_b2a/L7_absmax": 0.33660591, "gate_b2a/L7_rms": 0.05637302, "gate_b2a/L11_absmean": 0.08214706, "gate_b2a/L11_absmax": 0.62131137, "gate_b2a/L11_rms": 0.09159055, "gate_b2a/L14_absmean": 0.21351664, "gate_b2a/L14_absmax": 0.8212195, "gate_b2a/L14_rms": 0.22299322, "gate_b2c/L3_absmean": 0.03480892, "gate_b2c/L3_absmax": 0.56146812, "gate_b2c/L3_rms": 0.04578363, "gate_b2c/L5_absmean": 0.03643719, "gate_b2c/L5_absmax": 0.54093885, "gate_b2c/L5_rms": 0.04608136, "gate_b2c/L6_absmean": 0.05293923, "gate_b2c/L6_absmax": 0.34134048, "gate_b2c/L6_rms": 0.05889891}, "streams": {"cos_a2b/round0": 0.354556, "cos_c2b/round0": 0.220156, "cos_b2a/round0": 0.14917, "cos_b2c/round0": 0.123295, "rms_a/round0": 7.868226, "rms_b/round0": 13.155243, "rms_c/round0": 4.997014, "cos_a2b/round1": 0.412172, "cos_c2b/round1": 0.307801, "cos_b2a/round1": 0.242087, "cos_b2c/round1": 0.1196, "rms_a/round1": 15.659339, "rms_b/round1": 31.605221, "rms_c/round1": 11.044067, "cos_a2b/round2": 0.521149, "cos_c2b/round2": 0.509968, "cos_b2a/round2": 0.089004, "cos_b2c/round2": 0.173991, "rms_a/round2": 42.782974, "rms_b/round2": 83.027847, "rms_c/round2": 21.494902}, "alpha_mean": 0.42280113100486716, "alpha_hist": [0.0278, 0.14598, 0.18806, 0.15359, 0.12296, 0.11136, 0.10825, 0.09219, 0.04765, 0.00215], "alpha_by_type": {"special": {"share": 0.00116, "mean": 0.60583, "std": 0.14331}, "whitespace": {"share": 0.03523, "mean": 0.63189, "std": 0.16892}, "digit": {"share": 0.01067, "mean": 0.40137, "std": 0.18393}, "word_start": {"share": 0.57956, "mean": 0.42557, "std": 0.20709}, "word_cont": {"share": 0.25177, "mean": 0.28618, "std": 0.14378}, "punct": {"share": 0.12161, "mean": 0.632, "std": 0.18044}}, "alpha_p05": 0.11717, "alpha_p50": 0.39911, "alpha_p95": 0.80441}
{"step": 59810, "total_steps": 72479, "tokens": 31357665280, "loss": 3.16597580909729, "l_mix": 1.949094757437706, "l_a": 2.0615706741809845, "l_c": 1.9946993589401245, "z_loss": 43.830820083618164, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16396993398666382, "tokens_per_second": 1413439.6932039144, "eta_seconds": 4699.319471454621, "data_wait_seconds": 1.6450881958007812e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 59820, "total_steps": 72479, "tokens": 31362908160, "loss": 2.731946498155594, "l_mix": 1.6913737058639526, "l_a": 1.7522579282522202, "l_c": 1.7163177728652954, "z_loss": 51.78422784805298, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18719398975372314, "tokens_per_second": 1466676.1924368197, "eta_seconds": 4525.171831536293, "data_wait_seconds": 2.002716064453125e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 59830, "total_steps": 72479, "tokens": 31368151040, "loss": 4.384250164031982, "l_mix": 2.7175273299217224, "l_a": 2.8008740544319153, "l_c": 2.754868298768997, "z_loss": 42.01347303390503, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13849949836730957, "tokens_per_second": 1432896.676243971, "eta_seconds": 4628.190588998795, "data_wait_seconds": 1.6897916793823242e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 59840, "total_steps": 72479, "tokens": 31373393920, "loss": 4.419644117355347, "l_mix": 2.740135908126831, "l_a": 2.8235411643981934, "l_c": 2.774819165468216, "z_loss": 41.04916334152222, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20253100991249084, "tokens_per_second": 1406977.2380183623, "eta_seconds": 4709.725113487244, "data_wait_seconds": 1.6927719116210938e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 59850, "total_steps": 72479, "tokens": 31378636800, "loss": 4.354291200637817, "l_mix": 2.698678821325302, "l_a": 2.7826727628707886, "l_c": 2.736035019159317, "z_loss": 42.113391399383545, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.144631028175354, "tokens_per_second": 1365748.8684259902, "eta_seconds": 4848.060507369041, "data_wait_seconds": 2.1159648895263672e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 59860, "total_steps": 72479, "tokens": 31383879680, "loss": 4.451936364173889, "l_mix": 2.7592141032218933, "l_a": 2.8454686105251312, "l_c": 2.796938270330429, "z_loss": 42.468772888183594, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16167768836021423, "tokens_per_second": 1435031.2653083066, "eta_seconds": 4610.345733880997, "data_wait_seconds": 2.384185791015625e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 59870, "total_steps": 72479, "tokens": 31389122560, "loss": 4.467795550823212, "l_mix": 2.770836740732193, "l_a": 2.8497320413589478, "l_c": 2.80679714679718, "z_loss": 40.572786808013916, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20014578104019165, "tokens_per_second": 1406847.6195750253, "eta_seconds": 4698.978979682922, "data_wait_seconds": 2.1219253540039062e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 59880, "total_steps": 72479, "tokens": 31394365440, "loss": 4.41891747713089, "l_mix": 2.7386359870433807, "l_a": 2.825673609972, "l_c": 2.7752647697925568, "z_loss": 42.485631465911865, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20806576311588287, "tokens_per_second": 1407762.7503462555, "eta_seconds": 4692.200095772743, "data_wait_seconds": 2.1338462829589844e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 59890, "total_steps": 72479, "tokens": 31399608320, "loss": 4.78367155790329, "l_mix": 2.968919038772583, "l_a": 3.048228621482849, "l_c": 3.0009464025497437, "z_loss": 40.33016490936279, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17072264850139618, "tokens_per_second": 1395513.1556991006, "eta_seconds": 4729.630534148217, "data_wait_seconds": 2.0831823348999023e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 59900, "total_steps": 72479, "tokens": 31404851200, "loss": 3.1272518634796143, "l_mix": 1.9247592240571976, "l_a": 2.035961151123047, "l_c": 1.972347468137741, "z_loss": 43.487096309661865, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.22041982412338257, "tokens_per_second": 1420673.3866059761, "eta_seconds": 4642.17800810337, "data_wait_seconds": 2.059340476989746e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.1628145, "gate_a2b/L1_absmax": 0.70059073, "gate_a2b/L1_rms": 0.17324576, "gate_a2b/L3_absmean": 0.15524556, "gate_a2b/L3_absmax": 0.50430161, "gate_a2b/L3_rms": 0.16101569, "gate_a2b/L5_absmean": 0.22769924, "gate_a2b/L5_absmax": 0.73539186, "gate_a2b/L5_rms": 0.23377237, "gate_c2b/L1_absmean": 0.03391826, "gate_c2b/L1_absmax": 0.63537961, "gate_c2b/L1_rms": 0.06959093, "gate_c2b/L3_absmean": 0.17096014, "gate_c2b/L3_absmax": 0.3918786, "gate_c2b/L3_rms": 0.17460532, "gate_c2b/L5_absmean": 0.48368201, "gate_c2b/L5_absmax": 0.6866017, "gate_c2b/L5_rms": 0.49331978, "gate_b2a/L7_absmean": 0.05070647, "gate_b2a/L7_absmax": 0.33536711, "gate_b2a/L7_rms": 0.05644517, "gate_b2a/L11_absmean": 0.08226629, "gate_b2a/L11_absmax": 0.61817628, "gate_b2a/L11_rms": 0.09166504, "gate_b2a/L14_absmean": 0.21428308, "gate_b2a/L14_absmax": 0.81826925, "gate_b2a/L14_rms": 0.22370039, "gate_b2c/L3_absmean": 0.03482965, "gate_b2c/L3_absmax": 0.55721748, "gate_b2c/L3_rms": 0.0456324, "gate_b2c/L5_absmean": 0.03682442, "gate_b2c/L5_absmax": 0.53452265, "gate_b2c/L5_rms": 0.04623377, "gate_b2c/L6_absmean": 0.05329957, "gate_b2c/L6_absmax": 0.34187087, "gate_b2c/L6_rms": 0.0592574}, "streams": {"cos_a2b/round0": 0.35308, "cos_c2b/round0": 0.221965, "cos_b2a/round0": 0.14831, "cos_b2c/round0": 0.124251, "rms_a/round0": 7.870664, "rms_b/round0": 13.202877, "rms_c/round0": 5.021846, "cos_a2b/round1": 0.41048, "cos_c2b/round1": 0.308442, "cos_b2a/round1": 0.241603, "cos_b2c/round1": 0.118015, "rms_a/round1": 15.632087, "rms_b/round1": 31.69269, "rms_c/round1": 11.222529, "cos_a2b/round2": 0.514318, "cos_c2b/round2": 0.517992, "cos_b2a/round2": 0.089469, "cos_b2c/round2": 0.172268, "rms_a/round2": 42.970364, "rms_b/round2": 83.644051, "rms_c/round2": 21.899122}, "alpha_mean": 0.4089433268188145, "alpha_hist": [0.0316, 0.14732, 0.20399, 0.15877, 0.12477, 0.1098, 0.10039, 0.08344, 0.03781, 0.00211], "alpha_by_type": {"special": {"share": 0.00117, "mean": 0.60931, "std": 0.13249}, "whitespace": {"share": 0.0332, "mean": 0.60372, "std": 0.16972}, "digit": {"share": 0.01138, "mean": 0.38753, "std": 0.16903}, "word_start": {"share": 0.57617, "mean": 0.4143, "std": 0.20219}, "word_cont": {"share": 0.26042, "mean": 0.27752, "std": 0.14073}, "punct": {"share": 0.11765, "mean": 0.61874, "std": 0.17773}}, "alpha_p05": 0.10339, "alpha_p50": 0.36342, "alpha_p95": 0.78662}
{"step": 59910, "total_steps": 72479, "tokens": 31410094080, "loss": 3.106016308069229, "l_mix": 1.9110338538885117, "l_a": 2.024034321308136, "l_c": 1.9592407196760178, "z_loss": 46.24558687210083, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19045467674732208, "tokens_per_second": 1456506.940888897, "eta_seconds": 4524.369700551033, "data_wait_seconds": 2.181529998779297e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 59920, "total_steps": 72479, "tokens": 31415336960, "loss": 3.121940016746521, "l_mix": 1.9217287451028824, "l_a": 2.033220410346985, "l_c": 1.9674833863973618, "z_loss": 46.78382205963135, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2790995240211487, "tokens_per_second": 1315369.7962777521, "eta_seconds": 5005.8417113065725, "data_wait_seconds": 1.9609928131103516e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 59930, "total_steps": 72479, "tokens": 31420579840, "loss": 3.1060054898262024, "l_mix": 1.9237632304430008, "l_a": 1.988975703716278, "l_c": 1.951831877231598, "z_loss": 49.81331539154053, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2216346114873886, "tokens_per_second": 1470100.2261831241, "eta_seconds": 4475.402421426773, "data_wait_seconds": 2.6524066925048828e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 59940, "total_steps": 72479, "tokens": 31425822720, "loss": 4.382017254829407, "l_mix": 2.716367781162262, "l_a": 2.801224857568741, "l_c": 2.750939667224884, "z_loss": 42.29956245422363, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17586690187454224, "tokens_per_second": 1448457.1134445434, "eta_seconds": 4538.6550771713255, "data_wait_seconds": 1.7434358596801758e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 59950, "total_steps": 72479, "tokens": 31431065600, "loss": 4.409129798412323, "l_mix": 2.7331389486789703, "l_a": 2.8166205286979675, "l_c": 2.7700156569480896, "z_loss": 41.95355844497681, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1572548747062683, "tokens_per_second": 1394978.5462003255, "eta_seconds": 4708.892742395401, "data_wait_seconds": 2.345442771911621e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 59960, "total_steps": 72479, "tokens": 31436308480, "loss": 4.420622289180756, "l_mix": 2.740636020898819, "l_a": 2.822313904762268, "l_c": 2.777640253305435, "z_loss": 42.04560041427612, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15958333015441895, "tokens_per_second": 1423032.7980366368, "eta_seconds": 4612.375400662422, "data_wait_seconds": 1.683831214904785e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 59970, "total_steps": 72479, "tokens": 31441551360, "loss": 4.359811186790466, "l_mix": 2.7018215656280518, "l_a": 2.788984090089798, "l_c": 2.7376478910446167, "z_loss": 43.19482135772705, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13813416659832, "tokens_per_second": 1392547.5717475794, "eta_seconds": 4709.583158993721, "data_wait_seconds": 1.7404556274414062e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 59980, "total_steps": 72479, "tokens": 31446794240, "loss": 4.438056111335754, "l_mix": 2.751295119524002, "l_a": 2.834190011024475, "l_c": 2.7883467078208923, "z_loss": 40.7788987159729, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.26312029361724854, "tokens_per_second": 1409268.3088081158, "eta_seconds": 4649.98444302082, "data_wait_seconds": 1.6957521438598633e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 59990, "total_steps": 72479, "tokens": 31452037120, "loss": 4.449345648288727, "l_mix": 2.758678615093231, "l_a": 2.842378705739975, "l_c": 2.793177843093872, "z_loss": 41.87415361404419, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13850736618041992, "tokens_per_second": 1428242.9527620296, "eta_seconds": 4584.537119078636, "data_wait_seconds": 2.7239322662353516e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 60000, "total_steps": 72479, "tokens": 31457280000, "loss": 4.708371639251709, "l_mix": 2.9216947853565216, "l_a": 2.999766170978546, "l_c": 2.955823093652725, "z_loss": 41.29442358016968, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17546147108078003, "tokens_per_second": 1414977.910782436, "eta_seconds": 4623.810663151741, "data_wait_seconds": 1.671910285949707e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.16331856, "gate_a2b/L1_absmax": 0.69905818, "gate_a2b/L1_rms": 0.17359392, "gate_a2b/L3_absmean": 0.1548201, "gate_a2b/L3_absmax": 0.50147092, "gate_a2b/L3_rms": 0.16053276, "gate_a2b/L5_absmean": 0.22767834, "gate_a2b/L5_absmax": 0.73512423, "gate_a2b/L5_rms": 0.23370788, "gate_c2b/L1_absmean": 0.0343008, "gate_c2b/L1_absmax": 0.63396293, "gate_c2b/L1_rms": 0.06962796, "gate_c2b/L3_absmean": 0.17101066, "gate_c2b/L3_absmax": 0.39210632, "gate_c2b/L3_rms": 0.17471157, "gate_c2b/L5_absmean": 0.48310444, "gate_c2b/L5_absmax": 0.6803506, "gate_c2b/L5_rms": 0.49270061, "gate_b2a/L7_absmean": 0.0508209, "gate_b2a/L7_absmax": 0.33403078, "gate_b2a/L7_rms": 0.05658191, "gate_b2a/L11_absmean": 0.08213355, "gate_b2a/L11_absmax": 0.61603832, "gate_b2a/L11_rms": 0.09149872, "gate_b2a/L14_absmean": 0.2143174, "gate_b2a/L14_absmax": 0.8197974, "gate_b2a/L14_rms": 0.22372629, "gate_b2c/L3_absmean": 0.03479838, "gate_b2c/L3_absmax": 0.5566498, "gate_b2c/L3_rms": 0.04560602, "gate_b2c/L5_absmean": 0.03691743, "gate_b2c/L5_absmax": 0.5298081, "gate_b2c/L5_rms": 0.0461206, "gate_b2c/L6_absmean": 0.05336761, "gate_b2c/L6_absmax": 0.34301969, "gate_b2c/L6_rms": 0.05934321}, "streams": {"cos_a2b/round0": 0.354271, "cos_c2b/round0": 0.250519, "cos_b2a/round0": 0.162051, "cos_b2c/round0": 0.143516, "rms_a/round0": 7.51761, "rms_b/round0": 12.91686, "rms_c/round0": 4.867207, "cos_a2b/round1": 0.412642, "cos_c2b/round1": 0.309421, "cos_b2a/round1": 0.252019, "cos_b2c/round1": 0.11407, "rms_a/round1": 14.758286, "rms_b/round1": 29.744745, "rms_c/round1": 10.459099, "cos_a2b/round2": 0.540912, "cos_c2b/round2": 0.524219, "cos_b2a/round2": 0.095335, "cos_b2c/round2": 0.190454, "rms_a/round2": 41.655598, "rms_b/round2": 80.793327, "rms_c/round2": 20.171696}, "alpha_mean": 0.4405879797074732, "alpha_hist": [0.00385, 0.04799, 0.16599, 0.21584, 0.20912, 0.17422, 0.11824, 0.05783, 0.0067, 0.00023], "alpha_by_type": {"special": {"share": 0.00087, "mean": 0.56958, "std": 0.15078}, "whitespace": {"share": 0.05995, "mean": 0.52607, "std": 0.11111}, "digit": {"share": 0.02693, "mean": 0.47053, "std": 0.12251}, "word_start": {"share": 0.55103, "mean": 0.45195, "std": 0.15643}, "word_cont": {"share": 0.25179, "mean": 0.34102, "std": 0.1208}, "punct": {"share": 0.10944, "mean": 0.55725, "std": 0.15862}}, "alpha_p05": 0.20307, "alpha_p50": 0.43446, "alpha_p95": 0.70028}
{"step": 60010, "total_steps": 72479, "tokens": 31462522880, "loss": 4.600642025470734, "l_mix": 2.854453295469284, "l_a": 2.9299530684947968, "l_c": 2.8906761407852173, "z_loss": 41.380530834198, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15552380681037903, "tokens_per_second": 1437641.7273563198, "eta_seconds": 4547.271373391151, "data_wait_seconds": 2.256035804748535e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 60020, "total_steps": 72479, "tokens": 31467765760, "loss": 4.717979907989502, "l_mix": 2.927464097738266, "l_a": 3.0051717162132263, "l_c": 2.9632137417793274, "z_loss": 40.39792585372925, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15702961385250092, "tokens_per_second": 1253218.0724285138, "eta_seconds": 5212.264597606659, "data_wait_seconds": 1.7076730728149414e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 60030, "total_steps": 72479, "tokens": 31473008640, "loss": 3.0850606560707092, "l_mix": 1.8984528630971909, "l_a": 2.009949266910553, "l_c": 1.945409893989563, "z_loss": 45.8061408996582, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.157968670129776, "tokens_per_second": 1364480.5913853229, "eta_seconds": 4783.403555321694, "data_wait_seconds": 2.1666288375854492e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 60040, "total_steps": 72479, "tokens": 31478251520, "loss": 3.1047753989696503, "l_mix": 1.9108210802078247, "l_a": 2.0215402841567993, "l_c": 1.9583073407411575, "z_loss": 45.45413017272949, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1737230271100998, "tokens_per_second": 1419314.932875956, "eta_seconds": 4594.905810499192, "data_wait_seconds": 2.4437904357910156e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 60050, "total_steps": 72479, "tokens": 31483494400, "loss": 4.404621005058289, "l_mix": 2.7302453219890594, "l_a": 2.8147300481796265, "l_c": 2.7665219604969025, "z_loss": 42.71165895462036, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12432987242937088, "tokens_per_second": 1409411.0201906408, "eta_seconds": 4623.474244666099, "data_wait_seconds": 2.849102020263672e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 60060, "total_steps": 72479, "tokens": 31488737280, "loss": 4.432654619216919, "l_mix": 2.748040646314621, "l_a": 2.830920934677124, "l_c": 2.784458726644516, "z_loss": 41.967713356018066, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20208898186683655, "tokens_per_second": 1455706.1924464277, "eta_seconds": 4472.834357500076, "data_wait_seconds": 2.4199485778808594e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 60070, "total_steps": 72479, "tokens": 31493980160, "loss": 4.530791163444519, "l_mix": 2.8090954422950745, "l_a": 2.8932586312294006, "l_c": 2.8457269370555878, "z_loss": 41.627055168151855, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1681012213230133, "tokens_per_second": 1324221.9649258922, "eta_seconds": 4912.990393090248, "data_wait_seconds": 2.130866050720215e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 60080, "total_steps": 72479, "tokens": 31499223040, "loss": 4.333025872707367, "l_mix": 2.686830759048462, "l_a": 2.7665717005729675, "l_c": 2.7207449972629547, "z_loss": 41.79311752319336, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.20635563135147095, "tokens_per_second": 1461588.6440163366, "eta_seconds": 4447.658333015442, "data_wait_seconds": 2.5004148483276367e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 60090, "total_steps": 72479, "tokens": 31504465920, "loss": 4.4266273975372314, "l_mix": 2.74380224943161, "l_a": 2.8255882561206818, "l_c": 2.7838286459445953, "z_loss": 42.48228979110718, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16910424828529358, "tokens_per_second": 1354242.5529006566, "eta_seconds": 4796.337272143363, "data_wait_seconds": 1.7702579498291016e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 60100, "total_steps": 72479, "tokens": 31509708800, "loss": 3.101635903120041, "l_mix": 1.908285140991211, "l_a": 2.0220571011304855, "l_c": 1.95577871799469, "z_loss": 46.251283168792725, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1606265753507614, "tokens_per_second": 1369760.6853141095, "eta_seconds": 4738.1715810537335, "data_wait_seconds": 2.0802021026611328e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.16247356, "gate_a2b/L1_absmax": 0.70273703, "gate_a2b/L1_rms": 0.17297724, "gate_a2b/L3_absmean": 0.1542172, "gate_a2b/L3_absmax": 0.50655687, "gate_a2b/L3_rms": 0.16004695, "gate_a2b/L5_absmean": 0.22700956, "gate_a2b/L5_absmax": 0.73692125, "gate_a2b/L5_rms": 0.23312338, "gate_c2b/L1_absmean": 0.03467681, "gate_c2b/L1_absmax": 0.63660336, "gate_c2b/L1_rms": 0.06981684, "gate_c2b/L3_absmean": 0.17033489, "gate_c2b/L3_absmax": 0.39689285, "gate_c2b/L3_rms": 0.17414129, "gate_c2b/L5_absmean": 0.483217, "gate_c2b/L5_absmax": 0.67945361, "gate_c2b/L5_rms": 0.49283361, "gate_b2a/L7_absmean": 0.05086371, "gate_b2a/L7_absmax": 0.33519143, "gate_b2a/L7_rms": 0.05671091, "gate_b2a/L11_absmean": 0.08308624, "gate_b2a/L11_absmax": 0.61881727, "gate_b2a/L11_rms": 0.09236403, "gate_b2a/L14_absmean": 0.21404627, "gate_b2a/L14_absmax": 0.8289814, "gate_b2a/L14_rms": 0.22364007, "gate_b2c/L3_absmean": 0.03495598, "gate_b2c/L3_absmax": 0.55670965, "gate_b2c/L3_rms": 0.0457737, "gate_b2c/L5_absmean": 0.03624313, "gate_b2c/L5_absmax": 0.52676481, "gate_b2c/L5_rms": 0.04550323, "gate_b2c/L6_absmean": 0.05315644, "gate_b2c/L6_absmax": 0.34509164, "gate_b2c/L6_rms": 0.05925583}, "streams": {"cos_a2b/round0": 0.350421, "cos_c2b/round0": 0.218207, "cos_b2a/round0": 0.146763, "cos_b2c/round0": 0.123916, "rms_a/round0": 7.798056, "rms_b/round0": 13.209941, "rms_c/round0": 5.038094, "cos_a2b/round1": 0.407817, "cos_c2b/round1": 0.30441, "cos_b2a/round1": 0.240848, "cos_b2c/round1": 0.120864, "rms_a/round1": 15.629684, "rms_b/round1": 31.969069, "rms_c/round1": 11.163489, "cos_a2b/round2": 0.510917, "cos_c2b/round2": 0.512106, "cos_b2a/round2": 0.093183, "cos_b2c/round2": 0.172786, "rms_a/round2": 43.011459, "rms_b/round2": 83.830811, "rms_c/round2": 21.556828}, "alpha_mean": 0.3967254159703799, "alpha_hist": [0.04822, 0.18173, 0.18846, 0.14195, 0.11168, 0.10141, 0.10168, 0.0838, 0.03896, 0.00211], "alpha_by_type": {"special": {"share": 0.00102, "mean": 0.62486, "std": 0.1172}, "whitespace": {"share": 0.03285, "mean": 0.62324, "std": 0.16381}, "digit": {"share": 0.01111, "mean": 0.3763, "std": 0.17098}, "word_start": {"share": 0.57063, "mean": 0.40325, "std": 0.21391}, "word_cont": {"share": 0.26936, "mean": 0.26274, "std": 0.14229}, "punct": {"share": 0.11502, "mean": 0.61338, "std": 0.18413}}, "alpha_p05": 0.10339, "alpha_p50": 0.34687, "alpha_p95": 0.78066}
{"step": 60110, "total_steps": 72479, "tokens": 31514951680, "loss": 3.0351809561252594, "l_mix": 1.8791482001543045, "l_a": 1.946191668510437, "l_c": 1.907250553369522, "z_loss": 51.656471729278564, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14418169856071472, "tokens_per_second": 1381349.116908263, "eta_seconds": 4694.626573848725, "data_wait_seconds": 1.8477439880371094e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 60120, "total_steps": 72479, "tokens": 31520194560, "loss": 4.607328504323959, "l_mix": 2.8593346774578094, "l_a": 2.934372127056122, "l_c": 2.892273962497711, "z_loss": 40.82797336578369, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13612423837184906, "tokens_per_second": 1453641.5255789948, "eta_seconds": 4457.546979761123, "data_wait_seconds": 2.7060508728027344e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 60130, "total_steps": 72479, "tokens": 31525437440, "loss": 4.651751697063446, "l_mix": 2.8863007724285126, "l_a": 2.9630463123321533, "l_c": 2.9217897951602936, "z_loss": 40.36629390716553, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14428675174713135, "tokens_per_second": 1442839.8406659747, "eta_seconds": 4487.284263658524, "data_wait_seconds": 1.7762184143066406e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 60140, "total_steps": 72479, "tokens": 31530680320, "loss": 4.665461540222168, "l_mix": 2.8949674665927887, "l_a": 2.970517158508301, "l_c": 2.9311294555664062, "z_loss": 40.869550704956055, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1274411529302597, "tokens_per_second": 1353166.305242784, "eta_seconds": 4780.779425954818, "data_wait_seconds": 2.187490463256836e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 60150, "total_steps": 72479, "tokens": 31535923200, "loss": 4.607536733150482, "l_mix": 2.858587294816971, "l_a": 2.9354330599308014, "l_c": 2.8943982124328613, "z_loss": 40.59310245513916, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12281949073076248, "tokens_per_second": 1442548.319971808, "eta_seconds": 4480.922172594071, "data_wait_seconds": 2.3066997528076172e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 60160, "total_steps": 72479, "tokens": 31541166080, "loss": 4.6256343722343445, "l_mix": 2.8695070445537567, "l_a": 2.9489865601062775, "l_c": 2.9047706723213196, "z_loss": 41.38793992996216, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13100549578666687, "tokens_per_second": 1367652.5636696485, "eta_seconds": 4722.474145531654, "data_wait_seconds": 1.9848346710205078e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 60170, "total_steps": 72479, "tokens": 31546408960, "loss": 4.459430396556854, "l_mix": 2.7649300396442413, "l_a": 2.8496683835983276, "l_c": 2.7986658215522766, "z_loss": 40.851130962371826, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16084569692611694, "tokens_per_second": 1443295.2461560282, "eta_seconds": 4471.338078045845, "data_wait_seconds": 4.082918167114258e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 60180, "total_steps": 72479, "tokens": 31551651840, "loss": 4.440274953842163, "l_mix": 2.753421425819397, "l_a": 2.8326541781425476, "l_c": 2.7901905477046967, "z_loss": 40.47327470779419, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.24243080615997314, "tokens_per_second": 1428674.4305986993, "eta_seconds": 4513.427253890038, "data_wait_seconds": 2.2977590560913086e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 60190, "total_steps": 72479, "tokens": 31556894720, "loss": 4.4671242237091064, "l_mix": 2.7691063582897186, "l_a": 2.8552741706371307, "l_c": 2.8047847747802734, "z_loss": 42.49379777908325, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.21335725486278534, "tokens_per_second": 1370845.4707888013, "eta_seconds": 4700.001108288765, "data_wait_seconds": 2.3066997528076172e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 60200, "total_steps": 72479, "tokens": 31562137600, "loss": 4.4010990858078, "l_mix": 2.72785821557045, "l_a": 2.813011407852173, "l_c": 2.764457792043686, "z_loss": 42.81370496749878, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18551787734031677, "tokens_per_second": 1443113.6745170609, "eta_seconds": 4461.001559114456, "data_wait_seconds": 2.276897430419922e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16199178, "gate_a2b/L1_absmax": 0.6997723, "gate_a2b/L1_rms": 0.17247356, "gate_a2b/L3_absmean": 0.15454805, "gate_a2b/L3_absmax": 0.50293058, "gate_a2b/L3_rms": 0.16026409, "gate_a2b/L5_absmean": 0.22668333, "gate_a2b/L5_absmax": 0.73648745, "gate_a2b/L5_rms": 0.23275302, "gate_c2b/L1_absmean": 0.03433065, "gate_c2b/L1_absmax": 0.63321656, "gate_c2b/L1_rms": 0.06954729, "gate_c2b/L3_absmean": 0.17075473, "gate_c2b/L3_absmax": 0.39351574, "gate_c2b/L3_rms": 0.17441705, "gate_c2b/L5_absmean": 0.4829587, "gate_c2b/L5_absmax": 0.6762625, "gate_c2b/L5_rms": 0.49258035, "gate_b2a/L7_absmean": 0.05071358, "gate_b2a/L7_absmax": 0.33463129, "gate_b2a/L7_rms": 0.05653264, "gate_b2a/L11_absmean": 0.08248825, "gate_b2a/L11_absmax": 0.61832523, "gate_b2a/L11_rms": 0.09181843, "gate_b2a/L14_absmean": 0.21447828, "gate_b2a/L14_absmax": 0.82475281, "gate_b2a/L14_rms": 0.22398306, "gate_b2c/L3_absmean": 0.03472599, "gate_b2c/L3_absmax": 0.55638385, "gate_b2c/L3_rms": 0.04552424, "gate_b2c/L5_absmean": 0.0366764, "gate_b2c/L5_absmax": 0.52910656, "gate_b2c/L5_rms": 0.04591968, "gate_b2c/L6_absmean": 0.05317618, "gate_b2c/L6_absmax": 0.34387401, "gate_b2c/L6_rms": 0.05921925}, "streams": {"cos_a2b/round0": 0.355063, "cos_c2b/round0": 0.252041, "cos_b2a/round0": 0.159562, "cos_b2c/round0": 0.145611, "rms_a/round0": 7.537804, "rms_b/round0": 12.948342, "rms_c/round0": 4.865418, "cos_a2b/round1": 0.414308, "cos_c2b/round1": 0.303352, "cos_b2a/round1": 0.250173, "cos_b2c/round1": 0.108374, "rms_a/round1": 14.876645, "rms_b/round1": 30.149647, "rms_c/round1": 10.363224, "cos_a2b/round2": 0.536976, "cos_c2b/round2": 0.524349, "cos_b2a/round2": 0.095443, "cos_b2c/round2": 0.179236, "rms_a/round2": 42.172283, "rms_b/round2": 81.958969, "rms_c/round2": 20.155466}, "alpha_mean": 0.43034149309046654, "alpha_hist": [0.00752, 0.06735, 0.19102, 0.2108, 0.1777, 0.14833, 0.12106, 0.06863, 0.00754, 3e-05], "alpha_by_type": {"special": {"share": 0.00087, "mean": 0.56166, "std": 0.10917}, "whitespace": {"share": 0.02188, "mean": 0.50974, "std": 0.11867}, "digit": {"share": 0.0332, "mean": 0.42965, "std": 0.1187}, "word_start": {"share": 0.57452, "mean": 0.4469, "std": 0.16893}, "word_cont": {"share": 0.26633, "mean": 0.32719, "std": 0.12498}, "punct": {"share": 0.1032, "mean": 0.58668, "std": 0.15436}}, "alpha_p05": 0.17781, "alpha_p50": 0.42155, "alpha_p95": 0.72545}
{"step": 60210, "total_steps": 72479, "tokens": 31567380480, "loss": 3.1116423308849335, "l_mix": 1.9152284264564514, "l_a": 2.026495024561882, "l_c": 1.9615511149168015, "z_loss": 44.8051061630249, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.21176454424858093, "tokens_per_second": 1465194.4087546673, "eta_seconds": 4390.195207929611, "data_wait_seconds": 1.990795135498047e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 60220, "total_steps": 72479, "tokens": 31572623360, "loss": 3.101782262325287, "l_mix": 1.9216690063476562, "l_a": 1.9856958985328674, "l_c": 1.9480148255825043, "z_loss": 49.891024112701416, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18580371141433716, "tokens_per_second": 1416304.718796606, "eta_seconds": 4538.0393828392025, "data_wait_seconds": 1.8328428268432617e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 60230, "total_steps": 72479, "tokens": 31577866240, "loss": 4.442215859889984, "l_mix": 2.753414660692215, "l_a": 2.8390188217163086, "l_c": 2.7903182208538055, "z_loss": 42.69213104248047, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12369372695684433, "tokens_per_second": 1457839.9373327713, "eta_seconds": 4405.15007686615, "data_wait_seconds": 1.6897916793823242e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 60240, "total_steps": 72479, "tokens": 31583109120, "loss": 4.430929362773895, "l_mix": 2.747405022382736, "l_a": 2.8299335837364197, "l_c": 2.7818142473697662, "z_loss": 41.02879571914673, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11802747845649719, "tokens_per_second": 1441643.369229808, "eta_seconds": 4451.004297566414, "data_wait_seconds": 1.6629695892333984e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 60250, "total_steps": 72479, "tokens": 31588352000, "loss": 4.371018409729004, "l_mix": 2.7093369364738464, "l_a": 2.794331729412079, "l_c": 2.7446062862873077, "z_loss": 42.538140296936035, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15381686389446259, "tokens_per_second": 1417078.858188058, "eta_seconds": 4524.460946512222, "data_wait_seconds": 2.3484230041503906e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 60260, "total_steps": 72479, "tokens": 31593594880, "loss": 4.314013838768005, "l_mix": 2.6745366752147675, "l_a": 2.755690783262253, "l_c": 2.7092327177524567, "z_loss": 42.058326721191406, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16257071495056152, "tokens_per_second": 787066.5240460418, "eta_seconds": 8139.432787799836, "data_wait_seconds": 2.5451183319091797e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 60270, "total_steps": 72479, "tokens": 31598837760, "loss": 4.561056196689606, "l_mix": 2.828176498413086, "l_a": 2.9126815497875214, "l_c": 2.8635834455490112, "z_loss": 42.21385049819946, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.206447571516037, "tokens_per_second": 1481967.3009307317, "eta_seconds": 4319.280316090584, "data_wait_seconds": 1.9162893295288086e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 60280, "total_steps": 72479, "tokens": 31604080640, "loss": 4.461637914180756, "l_mix": 2.7664775252342224, "l_a": 2.8485613465309143, "l_c": 2.801973193883896, "z_loss": 40.95775270462036, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.23228336870670319, "tokens_per_second": 1450959.0946374927, "eta_seconds": 4407.973550486565, "data_wait_seconds": 2.3484230041503906e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 60290, "total_steps": 72479, "tokens": 31609323520, "loss": 4.500483691692352, "l_mix": 2.7910732328891754, "l_a": 2.8702328205108643, "l_c": 2.8278017938137054, "z_loss": 41.75002098083496, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15665648877620697, "tokens_per_second": 1458199.9408715793, "eta_seconds": 4382.4898444175715, "data_wait_seconds": 3.24249267578125e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 60300, "total_steps": 72479, "tokens": 31614566400, "loss": 4.37029230594635, "l_mix": 2.7086827158927917, "l_a": 2.7937395870685577, "l_c": 2.7449586987495422, "z_loss": 41.950467109680176, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1316012442111969, "tokens_per_second": 1434373.9787300047, "eta_seconds": 4451.630918216705, "data_wait_seconds": 1.8805265426635742e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16266952, "gate_a2b/L1_absmax": 0.69697291, "gate_a2b/L1_rms": 0.17306696, "gate_a2b/L3_absmean": 0.15443662, "gate_a2b/L3_absmax": 0.50158656, "gate_a2b/L3_rms": 0.1601523, "gate_a2b/L5_absmean": 0.2265359, "gate_a2b/L5_absmax": 0.73401523, "gate_a2b/L5_rms": 0.23258762, "gate_c2b/L1_absmean": 0.03415064, "gate_c2b/L1_absmax": 0.6316402, "gate_c2b/L1_rms": 0.06959973, "gate_c2b/L3_absmean": 0.17050627, "gate_c2b/L3_absmax": 0.3943598, "gate_c2b/L3_rms": 0.17420501, "gate_c2b/L5_absmean": 0.48272201, "gate_c2b/L5_absmax": 0.67975658, "gate_c2b/L5_rms": 0.4923245, "gate_b2a/L7_absmean": 0.05038061, "gate_b2a/L7_absmax": 0.33296123, "gate_b2a/L7_rms": 0.05630625, "gate_b2a/L11_absmean": 0.08244216, "gate_b2a/L11_absmax": 0.61563158, "gate_b2a/L11_rms": 0.09171405, "gate_b2a/L14_absmean": 0.21438505, "gate_b2a/L14_absmax": 0.82230103, "gate_b2a/L14_rms": 0.22379121, "gate_b2c/L3_absmean": 0.03492285, "gate_b2c/L3_absmax": 0.55517989, "gate_b2c/L3_rms": 0.04561538, "gate_b2c/L5_absmean": 0.03671489, "gate_b2c/L5_absmax": 0.52828908, "gate_b2c/L5_rms": 0.04594367, "gate_b2c/L6_absmean": 0.05321878, "gate_b2c/L6_absmax": 0.34322459, "gate_b2c/L6_rms": 0.05920821}, "streams": {"cos_a2b/round0": 0.355023, "cos_c2b/round0": 0.252333, "cos_b2a/round0": 0.166079, "cos_b2c/round0": 0.149022, "rms_a/round0": 7.552728, "rms_b/round0": 12.99781, "rms_c/round0": 4.884139, "cos_a2b/round1": 0.414631, "cos_c2b/round1": 0.31146, "cos_b2a/round1": 0.255694, "cos_b2c/round1": 0.115974, "rms_a/round1": 14.908834, "rms_b/round1": 30.169336, "rms_c/round1": 10.435566, "cos_a2b/round2": 0.545818, "cos_c2b/round2": 0.516824, "cos_b2a/round2": 0.096806, "cos_b2c/round2": 0.184863, "rms_a/round2": 42.690136, "rms_b/round2": 81.583939, "rms_c/round2": 20.195068}, "alpha_mean": 0.4499487091233334, "alpha_hist": [0.00787, 0.06691, 0.16542, 0.18498, 0.16913, 0.1748, 0.13512, 0.08304, 0.01271, 2e-05], "alpha_by_type": {"special": {"share": 0.00128, "mean": 0.58559, "std": 0.09929}, "whitespace": {"share": 0.02187, "mean": 0.54636, "std": 0.11685}, "digit": {"share": 0.02663, "mean": 0.44185, "std": 0.14376}, "word_start": {"share": 0.61252, "mean": 0.47107, "std": 0.16665}, "word_cont": {"share": 0.23959, "mean": 0.32088, "std": 0.13314}, "punct": {"share": 0.09811, "mean": 0.61223, "std": 0.15468}}, "alpha_p05": 0.18243, "alpha_p50": 0.43614, "alpha_p95": 0.73564}
{"step": 60310, "total_steps": 72479, "tokens": 31619809280, "loss": 4.399910092353821, "l_mix": 2.7266741394996643, "l_a": 2.8131074011325836, "l_c": 2.764345347881317, "z_loss": 42.625028133392334, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14815214276313782, "tokens_per_second": 1429539.7486844994, "eta_seconds": 4463.0173297882075, "data_wait_seconds": 2.1964311599731445e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 60320, "total_steps": 72479, "tokens": 31625052160, "loss": 3.1438948214054108, "l_mix": 1.9341470450162888, "l_a": 2.047254040837288, "l_c": 1.9852383732795715, "z_loss": 45.33415985107422, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2450226992368698, "tokens_per_second": 1435744.552701094, "eta_seconds": 4440.077992987633, "data_wait_seconds": 2.5510787963867188e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 60330, "total_steps": 72479, "tokens": 31630295040, "loss": 2.994195282459259, "l_mix": 1.854032650589943, "l_a": 1.9214020073413849, "l_c": 1.8791399002075195, "z_loss": 50.763115882873535, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1697263866662979, "tokens_per_second": 1437175.2293162346, "eta_seconds": 4432.009947061539, "data_wait_seconds": 2.7060508728027344e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 60340, "total_steps": 72479, "tokens": 31635537920, "loss": 4.470035910606384, "l_mix": 2.771547108888626, "l_a": 2.855637788772583, "l_c": 2.805991441011429, "z_loss": 40.9996919631958, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15718422830104828, "tokens_per_second": 1441628.8145655547, "eta_seconds": 4414.681482291222, "data_wait_seconds": 2.321600914001465e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 60350, "total_steps": 72479, "tokens": 31640780800, "loss": 4.43587201833725, "l_mix": 2.75042787194252, "l_a": 2.8312558829784393, "l_c": 2.786890983581543, "z_loss": 41.12446165084839, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1686146855354309, "tokens_per_second": 1470959.2014495116, "eta_seconds": 4323.090093684196, "data_wait_seconds": 1.5288591384887695e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 60360, "total_steps": 72479, "tokens": 31646023680, "loss": 4.3739606738090515, "l_mix": 2.7101640701293945, "l_a": 2.7983150482177734, "l_c": 2.7476733922958374, "z_loss": 42.49246025085449, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.25294268131256104, "tokens_per_second": 1441956.4601635053, "eta_seconds": 4406.406467556953, "data_wait_seconds": 1.6868114471435547e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 60370, "total_steps": 72479, "tokens": 31651266560, "loss": 4.350132882595062, "l_mix": 2.6969112753868103, "l_a": 2.7801676094532013, "l_c": 2.7305708527565002, "z_loss": 43.058085441589355, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2527430057525635, "tokens_per_second": 1296539.0118224947, "eta_seconds": 4896.577221441269, "data_wait_seconds": 3.3855438232421875e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 60380, "total_steps": 72479, "tokens": 31656509440, "loss": 4.258211731910706, "l_mix": 2.6383571326732635, "l_a": 2.723780632019043, "l_c": 2.6757345497608185, "z_loss": 43.163288593292236, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18698853254318237, "tokens_per_second": 1416316.5773187962, "eta_seconds": 4478.773046636581, "data_wait_seconds": 1.8596649169921875e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 60390, "total_steps": 72479, "tokens": 31661752320, "loss": 4.352139532566071, "l_mix": 2.6977534890174866, "l_a": 2.781296730041504, "l_c": 2.7333232164382935, "z_loss": 41.52658176422119, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.21715182065963745, "tokens_per_second": 1422792.8588536475, "eta_seconds": 4454.701605057716, "data_wait_seconds": 2.9534101486206055e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 60400, "total_steps": 72479, "tokens": 31666995200, "loss": 4.6509053111076355, "l_mix": 2.8870668709278107, "l_a": 2.9591417610645294, "l_c": 2.9203194677829742, "z_loss": 40.63652944564819, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13981841504573822, "tokens_per_second": 1421016.6433292406, "eta_seconds": 4456.580281257629, "data_wait_seconds": 1.7970800399780273e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.16233818, "gate_a2b/L1_absmax": 0.69685626, "gate_a2b/L1_rms": 0.17272383, "gate_a2b/L3_absmean": 0.15403962, "gate_a2b/L3_absmax": 0.50052917, "gate_a2b/L3_rms": 0.1597624, "gate_a2b/L5_absmean": 0.2262928, "gate_a2b/L5_absmax": 0.73247612, "gate_a2b/L5_rms": 0.2323301, "gate_c2b/L1_absmean": 0.03461606, "gate_c2b/L1_absmax": 0.63228279, "gate_c2b/L1_rms": 0.06968338, "gate_c2b/L3_absmean": 0.17028332, "gate_c2b/L3_absmax": 0.39289561, "gate_c2b/L3_rms": 0.17394201, "gate_c2b/L5_absmean": 0.48302022, "gate_c2b/L5_absmax": 0.67279088, "gate_c2b/L5_rms": 0.49255422, "gate_b2a/L7_absmean": 0.05078735, "gate_b2a/L7_absmax": 0.33303788, "gate_b2a/L7_rms": 0.05661756, "gate_b2a/L11_absmean": 0.08280601, "gate_b2a/L11_absmax": 0.61704469, "gate_b2a/L11_rms": 0.09206043, "gate_b2a/L14_absmean": 0.21465009, "gate_b2a/L14_absmax": 0.82256365, "gate_b2a/L14_rms": 0.22406214, "gate_b2c/L3_absmean": 0.0347787, "gate_b2c/L3_absmax": 0.55433893, "gate_b2c/L3_rms": 0.04550725, "gate_b2c/L5_absmean": 0.03683883, "gate_b2c/L5_absmax": 0.52514416, "gate_b2c/L5_rms": 0.04592175, "gate_b2c/L6_absmean": 0.0532647, "gate_b2c/L6_absmax": 0.34243003, "gate_b2c/L6_rms": 0.05927404}, "streams": {"cos_a2b/round0": 0.353346, "cos_c2b/round0": 0.263855, "cos_b2a/round0": 0.169358, "cos_b2c/round0": 0.150349, "rms_a/round0": 7.544755, "rms_b/round0": 12.900973, "rms_c/round0": 4.717422, "cos_a2b/round1": 0.423701, "cos_c2b/round1": 0.305207, "cos_b2a/round1": 0.257306, "cos_b2c/round1": 0.113078, "rms_a/round1": 14.915069, "rms_b/round1": 29.438129, "rms_c/round1": 10.387706, "cos_a2b/round2": 0.553265, "cos_c2b/round2": 0.511824, "cos_b2a/round2": 0.106079, "cos_b2c/round2": 0.186196, "rms_a/round2": 42.838959, "rms_b/round2": 82.84536, "rms_c/round2": 20.252203}, "alpha_mean": 0.4987170347903316, "alpha_hist": [0.00304, 0.03836, 0.1068, 0.15767, 0.16237, 0.20953, 0.20343, 0.10672, 0.01204, 5e-05], "alpha_by_type": {"special": {"share": 0.00047, "mean": 0.59259, "std": 0.12782}, "whitespace": {"share": 0.07826, "mean": 0.60966, "std": 0.1112}, "digit": {"share": 0.02298, "mean": 0.46301, "std": 0.1221}, "word_start": {"share": 0.52295, "mean": 0.51702, "std": 0.14806}, "word_cont": {"share": 0.24336, "mean": 0.35585, "std": 0.13962}, "punct": {"share": 0.13197, "mean": 0.62971, "std": 0.13519}}, "alpha_p05": 0.2124, "alpha_p50": 0.5051, "alpha_p95": 0.74428}
{"step": 60410, "total_steps": 72479, "tokens": 31672238080, "loss": 4.698862850666046, "l_mix": 2.9155878126621246, "l_a": 2.9932411909103394, "l_c": 2.951008766889572, "z_loss": 39.9284405708313, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1635090708732605, "tokens_per_second": 1387751.9379268088, "eta_seconds": 4559.627480292321, "data_wait_seconds": 1.7434358596801758e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 60420, "total_steps": 72479, "tokens": 31677480960, "loss": 4.69003701210022, "l_mix": 2.9101352095603943, "l_a": 2.9856292605400085, "l_c": 2.9473767578601837, "z_loss": 41.51544523239136, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17513948678970337, "tokens_per_second": 1416658.4617854462, "eta_seconds": 4462.888665509224, "data_wait_seconds": 2.1189451217651367e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 60430, "total_steps": 72479, "tokens": 31682723840, "loss": 4.654393315315247, "l_mix": 2.88733971118927, "l_a": 2.9660265147686005, "l_c": 2.924151837825775, "z_loss": 41.16553020477295, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19771650433540344, "tokens_per_second": 1438102.2270938864, "eta_seconds": 4392.696147036552, "data_wait_seconds": 1.8417835235595703e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 60440, "total_steps": 72479, "tokens": 31687966720, "loss": 2.9288837611675262, "l_mix": 1.8139403462409973, "l_a": 1.8788089752197266, "l_c": 1.8376688361167908, "z_loss": 50.275607109069824, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.24202828109264374, "tokens_per_second": 1451744.7559671921, "eta_seconds": 4347.80508491993, "data_wait_seconds": 1.755356788635254e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 60450, "total_steps": 72479, "tokens": 31693209600, "loss": 4.506108522415161, "l_mix": 2.793519139289856, "l_a": 2.8790057003498077, "l_c": 2.8296254575252533, "z_loss": 42.00126791000366, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1256389170885086, "tokens_per_second": 1377317.7121760864, "eta_seconds": 4578.943766021728, "data_wait_seconds": 0.010947704315185547, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 60460, "total_steps": 72479, "tokens": 31698452480, "loss": 4.377097427845001, "l_mix": 2.713603436946869, "l_a": 2.7940803468227386, "l_c": 2.750899374485016, "z_loss": 42.13023567199707, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16812646389007568, "tokens_per_second": 1442681.2880127574, "eta_seconds": 4367.851391959191, "data_wait_seconds": 2.5510787963867188e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 60470, "total_steps": 72479, "tokens": 31703695360, "loss": 4.29565042257309, "l_mix": 2.6621745824813843, "l_a": 2.7500500679016113, "l_c": 2.6948693096637726, "z_loss": 43.1457724571228, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1950182169675827, "tokens_per_second": 1402637.425154872, "eta_seconds": 4488.811206007004, "data_wait_seconds": 1.77919864654541e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 60480, "total_steps": 72479, "tokens": 31708938240, "loss": 4.455060601234436, "l_mix": 2.761600762605667, "l_a": 2.844683438539505, "l_c": 2.800182431936264, "z_loss": 43.093162536621094, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15502217411994934, "tokens_per_second": 1421212.7201730255, "eta_seconds": 4426.453283667564, "data_wait_seconds": 2.1636486053466797e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 60490, "total_steps": 72479, "tokens": 31714181120, "loss": 4.313113033771515, "l_mix": 2.6737333238124847, "l_a": 2.756950259208679, "l_c": 2.7076485455036163, "z_loss": 42.228415966033936, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1508394181728363, "tokens_per_second": 1376424.0615359577, "eta_seconds": 4566.6804349422455, "data_wait_seconds": 3.325939178466797e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 60500, "total_steps": 72479, "tokens": 31719424000, "loss": 4.377167522907257, "l_mix": 2.712477445602417, "l_a": 2.8002061247825623, "l_c": 2.748760312795639, "z_loss": 42.63670873641968, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.22593539953231812, "tokens_per_second": 1446885.9353725852, "eta_seconds": 4340.664179849625, "data_wait_seconds": 2.703070640563965e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.16243751, "gate_a2b/L1_absmax": 0.69629616, "gate_a2b/L1_rms": 0.17273149, "gate_a2b/L3_absmean": 0.15425953, "gate_a2b/L3_absmax": 0.49976751, "gate_a2b/L3_rms": 0.15994151, "gate_a2b/L5_absmean": 0.22615743, "gate_a2b/L5_absmax": 0.73331136, "gate_a2b/L5_rms": 0.23220854, "gate_c2b/L1_absmean": 0.03444704, "gate_c2b/L1_absmax": 0.63213646, "gate_c2b/L1_rms": 0.06963661, "gate_c2b/L3_absmean": 0.17073159, "gate_c2b/L3_absmax": 0.3922016, "gate_c2b/L3_rms": 0.17436515, "gate_c2b/L5_absmean": 0.48185998, "gate_c2b/L5_absmax": 0.67432278, "gate_c2b/L5_rms": 0.49143741, "gate_b2a/L7_absmean": 0.05042231, "gate_b2a/L7_absmax": 0.33180115, "gate_b2a/L7_rms": 0.05624678, "gate_b2a/L11_absmean": 0.08252376, "gate_b2a/L11_absmax": 0.61723244, "gate_b2a/L11_rms": 0.09185898, "gate_b2a/L14_absmean": 0.214912, "gate_b2a/L14_absmax": 0.82213759, "gate_b2a/L14_rms": 0.2242633, "gate_b2c/L3_absmean": 0.03478281, "gate_b2c/L3_absmax": 0.55394024, "gate_b2c/L3_rms": 0.04547236, "gate_b2c/L5_absmean": 0.03679353, "gate_b2c/L5_absmax": 0.52657759, "gate_b2c/L5_rms": 0.04594146, "gate_b2c/L6_absmean": 0.05361324, "gate_b2c/L6_absmax": 0.34273276, "gate_b2c/L6_rms": 0.05955328}, "streams": {"cos_a2b/round0": 0.354953, "cos_c2b/round0": 0.253837, "cos_b2a/round0": 0.16056, "cos_b2c/round0": 0.147061, "rms_a/round0": 7.59765, "rms_b/round0": 12.945601, "rms_c/round0": 4.831089, "cos_a2b/round1": 0.414284, "cos_c2b/round1": 0.307697, "cos_b2a/round1": 0.250598, "cos_b2c/round1": 0.114928, "rms_a/round1": 14.92867, "rms_b/round1": 30.114479, "rms_c/round1": 10.426627, "cos_a2b/round2": 0.544556, "cos_c2b/round2": 0.517847, "cos_b2a/round2": 0.099782, "cos_b2c/round2": 0.185666, "rms_a/round2": 42.714443, "rms_b/round2": 81.679863, "rms_c/round2": 20.102045}, "alpha_mean": 0.44861347118817463, "alpha_hist": [0.0098, 0.06406, 0.15924, 0.1862, 0.18361, 0.17329, 0.13553, 0.07707, 0.01103, 0.00017], "alpha_by_type": {"special": {"share": 0.00098, "mean": 0.58673, "std": 0.10559}, "whitespace": {"share": 0.02972, "mean": 0.58359, "std": 0.11326}, "digit": {"share": 0.03406, "mean": 0.45588, "std": 0.11637}, "word_start": {"share": 0.59753, "mean": 0.46119, "std": 0.16538}, "word_cont": {"share": 0.23604, "mean": 0.33021, "std": 0.1366}, "punct": {"share": 0.10167, "mean": 0.60635, "std": 0.15607}}, "alpha_p05": 0.17217, "alpha_p50": 0.4335, "alpha_p95": 0.73867}
{"step": 60510, "total_steps": 72479, "tokens": 31724666880, "loss": 4.60893577337265, "l_mix": 2.859480381011963, "l_a": 2.937224566936493, "l_c": 2.8942936062812805, "z_loss": 41.84714412689209, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.25748953223228455, "tokens_per_second": 1359728.9373178391, "eta_seconds": 4615.039733123779, "data_wait_seconds": 1.7017126083374023e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 60520, "total_steps": 72479, "tokens": 31729909760, "loss": 4.660018622875214, "l_mix": 2.89120215177536, "l_a": 2.965792715549469, "l_c": 2.930261790752411, "z_loss": 39.95090055465698, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17490872740745544, "tokens_per_second": 1444696.3128389441, "eta_seconds": 4339.984906363487, "data_wait_seconds": 2.3484230041503906e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 60530, "total_steps": 72479, "tokens": 31735152640, "loss": 4.680316507816315, "l_mix": 2.9047575294971466, "l_a": 2.9819929003715515, "l_c": 2.936536878347397, "z_loss": 39.245197772979736, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18796341121196747, "tokens_per_second": 1472845.2836610407, "eta_seconds": 4253.479561972618, "data_wait_seconds": 1.627206802368164e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 60540, "total_steps": 72479, "tokens": 31740395520, "loss": 4.655730187892914, "l_mix": 2.8884253203868866, "l_a": 2.963314473628998, "l_c": 2.9277017414569855, "z_loss": 41.045888900756836, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1851959377527237, "tokens_per_second": 1318206.2654790187, "eta_seconds": 4748.478744125367, "data_wait_seconds": 2.3245811462402344e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 60550, "total_steps": 72479, "tokens": 31745638400, "loss": 4.775328576564789, "l_mix": 2.9633713960647583, "l_a": 3.039688527584076, "l_c": 3.0001687705516815, "z_loss": 39.47815704345703, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16502565145492554, "tokens_per_second": 1462328.5833883225, "eta_seconds": 4276.898928904533, "data_wait_seconds": 2.092123031616211e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 60560, "total_steps": 72479, "tokens": 31750881280, "loss": 4.273869514465332, "l_mix": 2.6479076147079468, "l_a": 2.7336155474185944, "l_c": 2.6862571835517883, "z_loss": 43.36359691619873, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14128106832504272, "tokens_per_second": 1437495.1213291232, "eta_seconds": 4347.137308001518, "data_wait_seconds": 1.6510486602783203e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 60570, "total_steps": 72479, "tokens": 31756124160, "loss": 4.287931144237518, "l_mix": 2.6569499373435974, "l_a": 2.742415428161621, "l_c": 2.6941882967948914, "z_loss": 43.55777406692505, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14248289167881012, "tokens_per_second": 1237899.5085900535, "eta_seconds": 5043.822821378708, "data_wait_seconds": 1.9103288650512695e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 60580, "total_steps": 72479, "tokens": 31761367040, "loss": 4.319860994815826, "l_mix": 2.6775549948215485, "l_a": 2.7623763978481293, "l_c": 2.7119768857955933, "z_loss": 41.88145112991333, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15528395771980286, "tokens_per_second": 1470679.3219671557, "eta_seconds": 4241.919240188598, "data_wait_seconds": 1.761317253112793e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 60590, "total_steps": 72479, "tokens": 31766609920, "loss": 4.301608383655548, "l_mix": 2.665509283542633, "l_a": 2.7524186074733734, "l_c": 2.7012449204921722, "z_loss": 43.79445791244507, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2410319745540619, "tokens_per_second": 1438736.8609854674, "eta_seconds": 4332.453140687942, "data_wait_seconds": 2.3812055587768555e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 60600, "total_steps": 72479, "tokens": 31771852800, "loss": 4.230899274349213, "l_mix": 2.6216158270835876, "l_a": 2.7080500721931458, "l_c": 2.656227797269821, "z_loss": 42.684744358062744, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16704034805297852, "tokens_per_second": 1408757.9457541036, "eta_seconds": 4420.9277901649475, "data_wait_seconds": 2.0951032638549805e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16178155, "gate_a2b/L1_absmax": 0.70131928, "gate_a2b/L1_rms": 0.17212617, "gate_a2b/L3_absmean": 0.15419231, "gate_a2b/L3_absmax": 0.49808714, "gate_a2b/L3_rms": 0.15991232, "gate_a2b/L5_absmean": 0.22635031, "gate_a2b/L5_absmax": 0.73846829, "gate_a2b/L5_rms": 0.23238571, "gate_c2b/L1_absmean": 0.03469065, "gate_c2b/L1_absmax": 0.63323861, "gate_c2b/L1_rms": 0.06965534, "gate_c2b/L3_absmean": 0.17066805, "gate_c2b/L3_absmax": 0.38966191, "gate_c2b/L3_rms": 0.1742945, "gate_c2b/L5_absmean": 0.48158923, "gate_c2b/L5_absmax": 0.68240738, "gate_c2b/L5_rms": 0.4912239, "gate_b2a/L7_absmean": 0.05102123, "gate_b2a/L7_absmax": 0.33534127, "gate_b2a/L7_rms": 0.05683847, "gate_b2a/L11_absmean": 0.08284046, "gate_b2a/L11_absmax": 0.61609644, "gate_b2a/L11_rms": 0.09212956, "gate_b2a/L14_absmean": 0.21409163, "gate_b2a/L14_absmax": 0.82858509, "gate_b2a/L14_rms": 0.22361107, "gate_b2c/L3_absmean": 0.03477103, "gate_b2c/L3_absmax": 0.561167, "gate_b2c/L3_rms": 0.04568913, "gate_b2c/L5_absmean": 0.03684486, "gate_b2c/L5_absmax": 0.52828646, "gate_b2c/L5_rms": 0.04602944, "gate_b2c/L6_absmean": 0.05326039, "gate_b2c/L6_absmax": 0.3454127, "gate_b2c/L6_rms": 0.0593237}, "streams": {"cos_a2b/round0": 0.358624, "cos_c2b/round0": 0.250803, "cos_b2a/round0": 0.164854, "cos_b2c/round0": 0.146931, "rms_a/round0": 7.647929, "rms_b/round0": 12.982335, "rms_c/round0": 4.943905, "cos_a2b/round1": 0.416554, "cos_c2b/round1": 0.307607, "cos_b2a/round1": 0.256151, "cos_b2c/round1": 0.107787, "rms_a/round1": 14.960575, "rms_b/round1": 30.297754, "rms_c/round1": 10.500408, "cos_a2b/round2": 0.528148, "cos_c2b/round2": 0.533424, "cos_b2a/round2": 0.091412, "cos_b2c/round2": 0.178086, "rms_a/round2": 42.335064, "rms_b/round2": 82.602959, "rms_c/round2": 20.598961}, "alpha_mean": 0.4121169982479813, "alpha_hist": [0.00826, 0.08376, 0.20987, 0.21072, 0.17514, 0.14816, 0.11078, 0.04938, 0.00385, 9e-05], "alpha_by_type": {"special": {"share": 0.00128, "mean": 0.54006, "std": 0.12336}, "whitespace": {"share": 0.02226, "mean": 0.50017, "std": 0.11476}, "digit": {"share": 0.02237, "mean": 0.44651, "std": 0.12435}, "word_start": {"share": 0.60492, "mean": 0.42657, "std": 0.1644}, "word_cont": {"share": 0.24997, "mean": 0.31052, "std": 0.12103}, "punct": {"share": 0.0992, "mean": 0.55081, "std": 0.1597}}, "alpha_p05": 0.17329, "alpha_p50": 0.40263, "alpha_p95": 0.70902}
{"step": 60610, "total_steps": 72479, "tokens": 31777095680, "loss": 4.308026611804962, "l_mix": 2.6691344678401947, "l_a": 2.7571110129356384, "l_c": 2.705862432718277, "z_loss": 43.057130336761475, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17832060158252716, "tokens_per_second": 1451579.2576679739, "eta_seconds": 4286.8994160175325, "data_wait_seconds": 2.6017427444458008e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 60620, "total_steps": 72479, "tokens": 31782338560, "loss": 2.790931820869446, "l_mix": 1.7278569489717484, "l_a": 1.7911183089017868, "l_c": 1.752464547753334, "z_loss": 52.420265197753906, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2070472091436386, "tokens_per_second": 1399605.3495494898, "eta_seconds": 4442.346118497848, "data_wait_seconds": 4.124641418457031e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 60630, "total_steps": 72479, "tokens": 31787581440, "loss": 4.734700858592987, "l_mix": 2.935491144657135, "l_a": 3.010063797235489, "l_c": 2.9873016476631165, "z_loss": 41.473777770996094, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.5842050313949585, "tokens_per_second": 1364736.4975844915, "eta_seconds": 4552.005843615532, "data_wait_seconds": 3.224611282348633e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 60640, "total_steps": 72479, "tokens": 31792824320, "loss": 4.641404628753662, "l_mix": 2.87995982170105, "l_a": 2.959380030632019, "l_c": 2.912102609872818, "z_loss": 40.33383893966675, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1906798630952835, "tokens_per_second": 1269873.0316826794, "eta_seconds": 4887.926176190376, "data_wait_seconds": 2.3096799850463867e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 60650, "total_steps": 72479, "tokens": 31798067200, "loss": 4.670799612998962, "l_mix": 2.897160977125168, "l_a": 2.9775454699993134, "l_c": 2.934582859277725, "z_loss": 41.266079902648926, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13982822000980377, "tokens_per_second": 1428658.5587319934, "eta_seconds": 4340.997164154053, "data_wait_seconds": 1.7851591110229492e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 60660, "total_steps": 72479, "tokens": 31803310080, "loss": 4.417391717433929, "l_mix": 2.7404136061668396, "l_a": 2.8128428608179092, "l_c": 2.777083858847618, "z_loss": 40.41079783439636, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14839160442352295, "tokens_per_second": 1358116.7495909943, "eta_seconds": 4562.612068414688, "data_wait_seconds": 2.8520822525024414e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 60670, "total_steps": 72479, "tokens": 31808552960, "loss": 4.328543782234192, "l_mix": 2.68288916349411, "l_a": 2.7693847715854645, "l_c": 2.716130346059799, "z_loss": 42.10863256454468, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11835436522960663, "tokens_per_second": 1455555.397430267, "eta_seconds": 4253.577021479607, "data_wait_seconds": 2.0205974578857422e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 60680, "total_steps": 72479, "tokens": 31813795840, "loss": 4.274845600128174, "l_mix": 2.6487605273723602, "l_a": 2.7346946001052856, "l_c": 2.685588628053665, "z_loss": 42.45576238632202, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13536731898784637, "tokens_per_second": 1478281.7932612172, "eta_seconds": 4184.637962937355, "data_wait_seconds": 1.6570091247558594e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 60690, "total_steps": 72479, "tokens": 31819038720, "loss": 4.323918104171753, "l_mix": 2.6802114844322205, "l_a": 2.7639766335487366, "l_c": 2.715045243501663, "z_loss": 42.217942237854004, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13086387515068054, "tokens_per_second": 1437742.7713503197, "eta_seconds": 4298.982651948929, "data_wait_seconds": 4.088878631591797e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 60700, "total_steps": 72479, "tokens": 31824281600, "loss": 4.26498955488205, "l_mix": 2.642991065979004, "l_a": 2.7291556298732758, "l_c": 2.677505671977997, "z_loss": 42.08187961578369, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1603984832763672, "tokens_per_second": 1382545.6902011302, "eta_seconds": 4466.824059247971, "data_wait_seconds": 1.677870750427246e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16152819, "gate_a2b/L1_absmax": 0.69978255, "gate_a2b/L1_rms": 0.17189775, "gate_a2b/L3_absmean": 0.15378316, "gate_a2b/L3_absmax": 0.49983054, "gate_a2b/L3_rms": 0.15952213, "gate_a2b/L5_absmean": 0.22582603, "gate_a2b/L5_absmax": 0.73573822, "gate_a2b/L5_rms": 0.23189037, "gate_c2b/L1_absmean": 0.03448461, "gate_c2b/L1_absmax": 0.63310683, "gate_c2b/L1_rms": 0.0695544, "gate_c2b/L3_absmean": 0.17000264, "gate_c2b/L3_absmax": 0.39147064, "gate_c2b/L3_rms": 0.17367774, "gate_c2b/L5_absmean": 0.48126754, "gate_c2b/L5_absmax": 0.67922151, "gate_c2b/L5_rms": 0.49085221, "gate_b2a/L7_absmean": 0.0509053, "gate_b2a/L7_absmax": 0.33465379, "gate_b2a/L7_rms": 0.05670233, "gate_b2a/L11_absmean": 0.08263436, "gate_b2a/L11_absmax": 0.61541158, "gate_b2a/L11_rms": 0.09192517, "gate_b2a/L14_absmean": 0.21497519, "gate_b2a/L14_absmax": 0.82905346, "gate_b2a/L14_rms": 0.22443488, "gate_b2c/L3_absmean": 0.03470748, "gate_b2c/L3_absmax": 0.55801493, "gate_b2c/L3_rms": 0.04558853, "gate_b2c/L5_absmean": 0.03663971, "gate_b2c/L5_absmax": 0.52392823, "gate_b2c/L5_rms": 0.04574051, "gate_b2c/L6_absmean": 0.0534819, "gate_b2c/L6_absmax": 0.34449357, "gate_b2c/L6_rms": 0.05949276}, "streams": {"cos_a2b/round0": 0.357808, "cos_c2b/round0": 0.253021, "cos_b2a/round0": 0.159177, "cos_b2c/round0": 0.144884, "rms_a/round0": 7.670172, "rms_b/round0": 13.059347, "rms_c/round0": 4.949535, "cos_a2b/round1": 0.411277, "cos_c2b/round1": 0.303176, "cos_b2a/round1": 0.253711, "cos_b2c/round1": 0.110154, "rms_a/round1": 14.969214, "rms_b/round1": 30.321375, "rms_c/round1": 10.56207, "cos_a2b/round2": 0.535005, "cos_c2b/round2": 0.523133, "cos_b2a/round2": 0.100173, "cos_b2c/round2": 0.182678, "rms_a/round2": 42.599064, "rms_b/round2": 82.298973, "rms_c/round2": 20.381954}, "alpha_mean": 0.4279902138002001, "alpha_hist": [0.00905, 0.07362, 0.18889, 0.20334, 0.17921, 0.15701, 0.11615, 0.06343, 0.00923, 6e-05], "alpha_by_type": {"special": {"share": 0.00099, "mean": 0.54204, "std": 0.10032}, "whitespace": {"share": 0.02852, "mean": 0.51843, "std": 0.11646}, "digit": {"share": 0.03481, "mean": 0.4233, "std": 0.12743}, "word_start": {"share": 0.58257, "mean": 0.44308, "std": 0.16839}, "word_cont": {"share": 0.24626, "mean": 0.32055, "std": 0.12679}, "punct": {"share": 0.10686, "mean": 0.56963, "std": 0.16427}}, "alpha_p05": 0.18272, "alpha_p50": 0.40686, "alpha_p95": 0.72798}
{"step": 60710, "total_steps": 72479, "tokens": 31829524480, "loss": 4.25418359041214, "l_mix": 2.636578679084778, "l_a": 2.7208359241485596, "l_c": 2.671180099248886, "z_loss": 42.040082931518555, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14736385643482208, "tokens_per_second": 1475151.0155975278, "eta_seconds": 4182.8568104267115, "data_wait_seconds": 2.205371856689453e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 60720, "total_steps": 72479, "tokens": 31834767360, "loss": 4.306850969791412, "l_mix": 2.6689867079257965, "l_a": 2.757329523563385, "l_c": 2.702217936515808, "z_loss": 42.096840381622314, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.155876025557518, "tokens_per_second": 1429732.514473044, "eta_seconds": 4312.067138147354, "data_wait_seconds": 3.3289194107055664e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 60730, "total_steps": 72479, "tokens": 31840010240, "loss": 3.6060763597488403, "l_mix": 2.2367002964019775, "l_a": 2.2961209416389465, "l_c": 2.2684658467769623, "z_loss": 49.344117641448975, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.5493672490119934, "tokens_per_second": 1375054.8034220806, "eta_seconds": 4479.7194240331655, "data_wait_seconds": 2.3752450942993164e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 60740, "total_steps": 72479, "tokens": 31845253120, "loss": 3.747288227081299, "l_mix": 2.3230773359537125, "l_a": 2.39553864300251, "l_c": 2.351830869913101, "z_loss": 52.47174787521362, "lr": 0.0023, "wd": 0.01, "grad_norm": 2.5316975116729736, "tokens_per_second": 1411849.7922965565, "eta_seconds": 4359.257525539398, "data_wait_seconds": 2.0325183868408203e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 60750, "total_steps": 72479, "tokens": 31850496000, "loss": 4.264301776885986, "l_mix": 2.642143577337265, "l_a": 2.728292614221573, "l_c": 2.6789012253284454, "z_loss": 41.72044897079468, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1453450471162796, "tokens_per_second": 1424846.4648935215, "eta_seconds": 4315.815144658089, "data_wait_seconds": 2.0384788513183594e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 60760, "total_steps": 72479, "tokens": 31855738880, "loss": 4.368206560611725, "l_mix": 2.7074387967586517, "l_a": 2.794762074947357, "l_c": 2.741130083799362, "z_loss": 42.8344669342041, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2582920491695404, "tokens_per_second": 1359384.6469865623, "eta_seconds": 4519.788483428955, "data_wait_seconds": 2.3305416107177734e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 60770, "total_steps": 72479, "tokens": 31860981760, "loss": 4.297766208648682, "l_mix": 2.662513494491577, "l_a": 2.7491276264190674, "l_c": 2.7017143070697784, "z_loss": 43.19016981124878, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15903836488723755, "tokens_per_second": 1314168.5808575652, "eta_seconds": 4671.309511899948, "data_wait_seconds": 2.0444393157958984e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 60780, "total_steps": 72479, "tokens": 31866224640, "loss": 4.290758967399597, "l_mix": 2.659413754940033, "l_a": 2.7420921623706818, "l_c": 2.6957247853279114, "z_loss": 40.93522930145264, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14573253691196442, "tokens_per_second": 1431831.4816316976, "eta_seconds": 4283.775982499123, "data_wait_seconds": 1.5616416931152344e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 60790, "total_steps": 72479, "tokens": 31871467520, "loss": 4.296171188354492, "l_mix": 2.6626356840133667, "l_a": 2.7459656596183777, "l_c": 2.699152261018753, "z_loss": 41.781086444854736, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1124071255326271, "tokens_per_second": 1410701.0683891536, "eta_seconds": 4344.224704527855, "data_wait_seconds": 1.8656253814697266e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 60800, "total_steps": 72479, "tokens": 31876710400, "loss": 4.179217755794525, "l_mix": 2.5891806185245514, "l_a": 2.675451338291168, "l_c": 2.624672591686249, "z_loss": 42.98986053466797, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1163530945777893, "tokens_per_second": 1347814.3423777523, "eta_seconds": 4543.028931713104, "data_wait_seconds": 1.6391277313232422e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16093759, "gate_a2b/L1_absmax": 0.70452499, "gate_a2b/L1_rms": 0.17152268, "gate_a2b/L3_absmean": 0.15247689, "gate_a2b/L3_absmax": 0.49976826, "gate_a2b/L3_rms": 0.158228, "gate_a2b/L5_absmean": 0.22512574, "gate_a2b/L5_absmax": 0.74054176, "gate_a2b/L5_rms": 0.23135053, "gate_c2b/L1_absmean": 0.0346323, "gate_c2b/L1_absmax": 0.6366244, "gate_c2b/L1_rms": 0.07009882, "gate_c2b/L3_absmean": 0.17029284, "gate_c2b/L3_absmax": 0.39626133, "gate_c2b/L3_rms": 0.17395239, "gate_c2b/L5_absmean": 0.48134065, "gate_c2b/L5_absmax": 0.69124514, "gate_c2b/L5_rms": 0.49103647, "gate_b2a/L7_absmean": 0.05135502, "gate_b2a/L7_absmax": 0.33985144, "gate_b2a/L7_rms": 0.05731487, "gate_b2a/L11_absmean": 0.08307952, "gate_b2a/L11_absmax": 0.62221819, "gate_b2a/L11_rms": 0.09261049, "gate_b2a/L14_absmean": 0.21340197, "gate_b2a/L14_absmax": 0.84490466, "gate_b2a/L14_rms": 0.22345683, "gate_b2c/L3_absmean": 0.03451462, "gate_b2c/L3_absmax": 0.56386709, "gate_b2c/L3_rms": 0.04558209, "gate_b2c/L5_absmean": 0.03650904, "gate_b2c/L5_absmax": 0.53607303, "gate_b2c/L5_rms": 0.04597347, "gate_b2c/L6_absmean": 0.05298365, "gate_b2c/L6_absmax": 0.35132807, "gate_b2c/L6_rms": 0.05934184}, "streams": {"cos_a2b/round0": 0.358767, "cos_c2b/round0": 0.250904, "cos_b2a/round0": 0.15692, "cos_b2c/round0": 0.14525, "rms_a/round0": 7.713908, "rms_b/round0": 13.196241, "rms_c/round0": 4.945976, "cos_a2b/round1": 0.409511, "cos_c2b/round1": 0.29802, "cos_b2a/round1": 0.248486, "cos_b2c/round1": 0.108648, "rms_a/round1": 15.193984, "rms_b/round1": 30.941294, "rms_c/round1": 10.632181, "cos_a2b/round2": 0.527935, "cos_c2b/round2": 0.525339, "cos_b2a/round2": 0.092955, "cos_b2c/round2": 0.180725, "rms_a/round2": 43.194031, "rms_b/round2": 84.581558, "rms_c/round2": 20.704504}, "alpha_mean": 0.43125314398172065, "alpha_hist": [0.00816, 0.07555, 0.18301, 0.20317, 0.1799, 0.15173, 0.118, 0.07057, 0.00983, 8e-05], "alpha_by_type": {"special": {"share": 0.00101, "mean": 0.55216, "std": 0.10737}, "whitespace": {"share": 0.03262, "mean": 0.52263, "std": 0.12764}, "digit": {"share": 0.05124, "mean": 0.46471, "std": 0.12878}, "word_start": {"share": 0.55237, "mean": 0.44248, "std": 0.17163}, "word_cont": {"share": 0.25606, "mean": 0.32349, "std": 0.12582}, "punct": {"share": 0.1067, "mean": 0.58657, "std": 0.15843}}, "alpha_p05": 0.17838, "alpha_p50": 0.40969, "alpha_p95": 0.72584}
{"step": 60810, "total_steps": 72479, "tokens": 31881953280, "loss": 4.346378922462463, "l_mix": 2.6936609148979187, "l_a": 2.7809828221797943, "l_c": 2.728077381849289, "z_loss": 42.53768730163574, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14002901315689087, "tokens_per_second": 1434563.7455926284, "eta_seconds": 4264.653063201904, "data_wait_seconds": 1.5467405319213867e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 60820, "total_steps": 72479, "tokens": 31887196160, "loss": 4.157687544822693, "l_mix": 2.575638383626938, "l_a": 2.6614255607128143, "l_c": 2.612071454524994, "z_loss": 43.38010311126709, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19185388088226318, "tokens_per_second": 1429245.1238344284, "eta_seconds": 4276.85474665165, "data_wait_seconds": 1.7851591110229492e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 60830, "total_steps": 72479, "tokens": 31892439040, "loss": 4.325248420238495, "l_mix": 2.6807282269001007, "l_a": 2.764960676431656, "l_c": 2.716773211956024, "z_loss": 42.41669988632202, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.138459712266922, "tokens_per_second": 1319021.3861862433, "eta_seconds": 4630.274365496635, "data_wait_seconds": 0.0008471906185150146, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 60840, "total_steps": 72479, "tokens": 31897681920, "loss": 2.818241208791733, "l_mix": 1.744619071483612, "l_a": 1.8091450780630112, "l_c": 1.7695951163768768, "z_loss": 51.74143981933594, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14437474310398102, "tokens_per_second": 1368358.6649490702, "eta_seconds": 4459.494566965102, "data_wait_seconds": 2.613663673400879e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 60850, "total_steps": 72479, "tokens": 31902924800, "loss": 3.0566344261169434, "l_mix": 1.8930203020572662, "l_a": 1.9577212780714035, "l_c": 1.9209923148155212, "z_loss": 50.70641565322876, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2145470231771469, "tokens_per_second": 1354312.361823108, "eta_seconds": 4501.875138902665, "data_wait_seconds": 1.7315149307250977e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 60860, "total_steps": 72479, "tokens": 31908167680, "loss": 4.25067999958992, "l_mix": 2.63417586684227, "l_a": 2.7187490463256836, "l_c": 2.6695975959300995, "z_loss": 42.89350748062134, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11301057785749435, "tokens_per_second": 1401247.329869568, "eta_seconds": 4347.342644047737, "data_wait_seconds": 1.9311904907226562e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 60870, "total_steps": 72479, "tokens": 31913410560, "loss": 4.292961657047272, "l_mix": 2.6596599817276, "l_a": 2.751011461019516, "l_c": 2.693327158689499, "z_loss": 43.52745294570923, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.29182955622673035, "tokens_per_second": 1417828.52190264, "eta_seconds": 4292.80360634327, "data_wait_seconds": 2.0325183868408203e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 60880, "total_steps": 72479, "tokens": 31918653440, "loss": 4.374385476112366, "l_mix": 2.7111216485500336, "l_a": 2.796956956386566, "l_c": 2.747255265712738, "z_loss": 42.866326332092285, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1580481231212616, "tokens_per_second": 1440230.7642822356, "eta_seconds": 4222.390371608734, "data_wait_seconds": 2.16066837310791e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 60890, "total_steps": 72479, "tokens": 31923896320, "loss": 4.339102625846863, "l_mix": 2.6891233026981354, "l_a": 2.7742209136486053, "l_c": 2.725709766149521, "z_loss": 42.161818981170654, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1794315129518509, "tokens_per_second": 1396597.264255573, "eta_seconds": 4350.555301451683, "data_wait_seconds": 1.9162893295288086e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 60900, "total_steps": 72479, "tokens": 31929139200, "loss": 4.372352778911591, "l_mix": 2.7098849415779114, "l_a": 2.7958956956863403, "l_c": 2.7456633746623993, "z_loss": 42.3353009223938, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11896178126335144, "tokens_per_second": 1406660.7950140128, "eta_seconds": 4315.703383159637, "data_wait_seconds": 1.7195940017700195e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.16160972, "gate_a2b/L1_absmax": 0.70035094, "gate_a2b/L1_rms": 0.17201358, "gate_a2b/L3_absmean": 0.15269825, "gate_a2b/L3_absmax": 0.50100493, "gate_a2b/L3_rms": 0.15840867, "gate_a2b/L5_absmean": 0.22454308, "gate_a2b/L5_absmax": 0.73791486, "gate_a2b/L5_rms": 0.23070811, "gate_c2b/L1_absmean": 0.03535391, "gate_c2b/L1_absmax": 0.63572413, "gate_c2b/L1_rms": 0.07005823, "gate_c2b/L3_absmean": 0.16999801, "gate_c2b/L3_absmax": 0.39425278, "gate_c2b/L3_rms": 0.17375609, "gate_c2b/L5_absmean": 0.48157999, "gate_c2b/L5_absmax": 0.67356056, "gate_c2b/L5_rms": 0.49113768, "gate_b2a/L7_absmean": 0.05107122, "gate_b2a/L7_absmax": 0.33819336, "gate_b2a/L7_rms": 0.05704427, "gate_b2a/L11_absmean": 0.08321069, "gate_b2a/L11_absmax": 0.61844397, "gate_b2a/L11_rms": 0.09253164, "gate_b2a/L14_absmean": 0.21493903, "gate_b2a/L14_absmax": 0.83969611, "gate_b2a/L14_rms": 0.22470228, "gate_b2c/L3_absmean": 0.03496607, "gate_b2c/L3_absmax": 0.5592823, "gate_b2c/L3_rms": 0.04571724, "gate_b2c/L5_absmean": 0.03652579, "gate_b2c/L5_absmax": 0.53086615, "gate_b2c/L5_rms": 0.04582736, "gate_b2c/L6_absmean": 0.05309252, "gate_b2c/L6_absmax": 0.34801397, "gate_b2c/L6_rms": 0.05929048}, "streams": {"cos_a2b/round0": 0.358283, "cos_c2b/round0": 0.246541, "cos_b2a/round0": 0.161199, "cos_b2c/round0": 0.145168, "rms_a/round0": 7.717763, "rms_b/round0": 13.260693, "rms_c/round0": 4.9967, "cos_a2b/round1": 0.411717, "cos_c2b/round1": 0.308665, "cos_b2a/round1": 0.255823, "cos_b2c/round1": 0.114573, "rms_a/round1": 15.155633, "rms_b/round1": 30.691368, "rms_c/round1": 10.753778, "cos_a2b/round2": 0.530694, "cos_c2b/round2": 0.530077, "cos_b2a/round2": 0.0947, "cos_b2c/round2": 0.182751, "rms_a/round2": 43.31963, "rms_b/round2": 83.789169, "rms_c/round2": 20.977419}, "alpha_mean": 0.42445022462959514, "alpha_hist": [0.00784, 0.06985, 0.18829, 0.21469, 0.188, 0.15343, 0.1145, 0.0582, 0.00514, 5e-05], "alpha_by_type": {"special": {"share": 0.00124, "mean": 0.5272, "std": 0.10622}, "whitespace": {"share": 0.02617, "mean": 0.50301, "std": 0.11251}, "digit": {"share": 0.02776, "mean": 0.42469, "std": 0.12256}, "word_start": {"share": 0.5799, "mean": 0.44246, "std": 0.16048}, "word_cont": {"share": 0.26755, "mean": 0.31956, "std": 0.12554}, "punct": {"share": 0.0974, "mean": 0.58287, "std": 0.15077}}, "alpha_p05": 0.18097, "alpha_p50": 0.40451, "alpha_p95": 0.71003}
{"step": 60910, "total_steps": 72479, "tokens": 31934382080, "loss": 4.6145453453063965, "l_mix": 2.862203061580658, "l_a": 2.942312002182007, "l_c": 2.8988289535045624, "z_loss": 41.33993911743164, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1258377730846405, "tokens_per_second": 1229684.9034550984, "eta_seconds": 4932.554555201531, "data_wait_seconds": 4.220008850097656e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 60920, "total_steps": 72479, "tokens": 31939624960, "loss": 4.666323125362396, "l_mix": 2.895187258720398, "l_a": 2.973612666130066, "l_c": 2.930173248052597, "z_loss": 41.60581874847412, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16602492332458496, "tokens_per_second": 1394087.5562372713, "eta_seconds": 4347.105004191399, "data_wait_seconds": 2.60770320892334e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 60930, "total_steps": 72479, "tokens": 31944867840, "loss": 4.6725093126297, "l_mix": 2.8995216488838196, "l_a": 2.974213123321533, "l_c": 2.9357452392578125, "z_loss": 40.34627723693848, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1622268557548523, "tokens_per_second": 1463015.7362357555, "eta_seconds": 4138.7129078865055, "data_wait_seconds": 2.9861927032470703e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 60940, "total_steps": 72479, "tokens": 31950110720, "loss": 4.640590131282806, "l_mix": 2.8791114687919617, "l_a": 2.9557386934757233, "l_c": 2.9158568382263184, "z_loss": 40.10622215270996, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17083610594272614, "tokens_per_second": 1420137.8556003084, "eta_seconds": 4259.980260467529, "data_wait_seconds": 2.6911497116088867e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 60950, "total_steps": 72479, "tokens": 31955353600, "loss": 2.7689722776412964, "l_mix": 1.7129496484994888, "l_a": 1.78131565451622, "l_c": 1.738759621977806, "z_loss": 53.42774820327759, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18252065777778625, "tokens_per_second": 1411653.481028772, "eta_seconds": 4281.869759988785, "data_wait_seconds": 3.9011240005493164e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 60960, "total_steps": 72479, "tokens": 31960596480, "loss": 3.0248682498931885, "l_mix": 1.873682588338852, "l_a": 1.9376352429389954, "l_c": 1.8996499329805374, "z_loss": 50.33948564529419, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1841859519481659, "tokens_per_second": 1435984.6607585785, "eta_seconds": 4205.667119598389, "data_wait_seconds": 2.1517276763916016e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 60970, "total_steps": 72479, "tokens": 31965839360, "loss": 4.326865196228027, "l_mix": 2.6817335188388824, "l_a": 2.7669676542282104, "l_c": 2.716804176568985, "z_loss": 41.91085195541382, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12090371549129486, "tokens_per_second": 1330288.9591491274, "eta_seconds": 4535.87963013649, "data_wait_seconds": 2.110004425048828e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 60980, "total_steps": 72479, "tokens": 31971082240, "loss": 4.334007501602173, "l_mix": 2.6865134835243225, "l_a": 2.7693140506744385, "l_c": 2.7223324477672577, "z_loss": 41.86101961135864, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13092920184135437, "tokens_per_second": 1418664.730975562, "eta_seconds": 4249.6211968660355, "data_wait_seconds": 2.154707908630371e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 60990, "total_steps": 72479, "tokens": 31976325120, "loss": 4.299391984939575, "l_mix": 2.6646842062473297, "l_a": 2.7488369047641754, "l_c": 2.7001889050006866, "z_loss": 42.69826650619507, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12884971499443054, "tokens_per_second": 1429504.714391684, "eta_seconds": 4213.72855322361, "data_wait_seconds": 2.5063753128051758e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 61000, "total_steps": 72479, "tokens": 31981568000, "loss": 4.215147137641907, "l_mix": 2.611487925052643, "l_a": 2.7002551555633545, "l_c": 2.645275115966797, "z_loss": 43.22030973434448, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.161091610789299, "tokens_per_second": 1404272.9008861387, "eta_seconds": 4285.706822514534, "data_wait_seconds": 3.039836883544922e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16137104, "gate_a2b/L1_absmax": 0.70087439, "gate_a2b/L1_rms": 0.17171758, "gate_a2b/L3_absmean": 0.15320098, "gate_a2b/L3_absmax": 0.49936658, "gate_a2b/L3_rms": 0.15885219, "gate_a2b/L5_absmean": 0.22440444, "gate_a2b/L5_absmax": 0.73801744, "gate_a2b/L5_rms": 0.23050787, "gate_c2b/L1_absmean": 0.03520328, "gate_c2b/L1_absmax": 0.63433301, "gate_c2b/L1_rms": 0.07000922, "gate_c2b/L3_absmean": 0.16997483, "gate_c2b/L3_absmax": 0.39386845, "gate_c2b/L3_rms": 0.1736396, "gate_c2b/L5_absmean": 0.48147807, "gate_c2b/L5_absmax": 0.68790716, "gate_c2b/L5_rms": 0.49102667, "gate_b2a/L7_absmean": 0.05100995, "gate_b2a/L7_absmax": 0.33654127, "gate_b2a/L7_rms": 0.05684374, "gate_b2a/L11_absmean": 0.08276996, "gate_b2a/L11_absmax": 0.61801243, "gate_b2a/L11_rms": 0.09210005, "gate_b2a/L14_absmean": 0.21506698, "gate_b2a/L14_absmax": 0.8362841, "gate_b2a/L14_rms": 0.22474749, "gate_b2c/L3_absmean": 0.03444592, "gate_b2c/L3_absmax": 0.55919057, "gate_b2c/L3_rms": 0.04540797, "gate_b2c/L5_absmean": 0.03647143, "gate_b2c/L5_absmax": 0.53022587, "gate_b2c/L5_rms": 0.04577619, "gate_b2c/L6_absmean": 0.05319706, "gate_b2c/L6_absmax": 0.3470287, "gate_b2c/L6_rms": 0.05931213}, "streams": {"cos_a2b/round0": 0.357789, "cos_c2b/round0": 0.25366, "cos_b2a/round0": 0.155773, "cos_b2c/round0": 0.14498, "rms_a/round0": 7.734757, "rms_b/round0": 13.07818, "rms_c/round0": 4.946475, "cos_a2b/round1": 0.410919, "cos_c2b/round1": 0.295835, "cos_b2a/round1": 0.244055, "cos_b2c/round1": 0.112266, "rms_a/round1": 15.230116, "rms_b/round1": 30.743616, "rms_c/round1": 10.522895, "cos_a2b/round2": 0.524128, "cos_c2b/round2": 0.525259, "cos_b2a/round2": 0.092839, "cos_b2c/round2": 0.180386, "rms_a/round2": 42.987312, "rms_b/round2": 83.699753, "rms_c/round2": 20.434597}, "alpha_mean": 0.41800418800177397, "alpha_hist": [0.01331, 0.09879, 0.20444, 0.18938, 0.1584, 0.13962, 0.11308, 0.07042, 0.01213, 0.00044], "alpha_by_type": {"special": {"share": 0.00078, "mean": 0.57937, "std": 0.10969}, "whitespace": {"share": 0.03078, "mean": 0.52038, "std": 0.12502}, "digit": {"share": 0.04889, "mean": 0.45884, "std": 0.12123}, "word_start": {"share": 0.5473, "mean": 0.43419, "std": 0.18052}, "word_cont": {"share": 0.26506, "mean": 0.29799, "std": 0.12377}, "punct": {"share": 0.10719, "mean": 0.58294, "std": 0.17001}}, "alpha_p05": 0.15508, "alpha_p50": 0.40922, "alpha_p95": 0.72604}
{"step": 61010, "total_steps": 72479, "tokens": 31986810880, "loss": 4.240174829959869, "l_mix": 2.627491056919098, "l_a": 2.7122855484485626, "l_c": 2.663326472043991, "z_loss": 42.28232479095459, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1712142676115036, "tokens_per_second": 1432474.3087583315, "eta_seconds": 4197.673239398003, "data_wait_seconds": 3.2454729080200195e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 61020, "total_steps": 72479, "tokens": 31992053760, "loss": 4.688372611999512, "l_mix": 2.9102074205875397, "l_a": 2.983555853366852, "l_c": 2.9436612129211426, "z_loss": 40.45108985900879, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18824678659439087, "tokens_per_second": 1446867.6571269156, "eta_seconds": 4152.291443109512, "data_wait_seconds": 2.9414892196655273e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 61030, "total_steps": 72479, "tokens": 31997296640, "loss": 4.679729104042053, "l_mix": 2.903081238269806, "l_a": 2.9819274246692657, "l_c": 2.94023197889328, "z_loss": 41.688976764678955, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1461593061685562, "tokens_per_second": 1465252.3026186428, "eta_seconds": 4096.614147114753, "data_wait_seconds": 2.5272369384765625e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 61040, "total_steps": 72479, "tokens": 32002539520, "loss": 4.588863909244537, "l_mix": 2.846790462732315, "l_a": 2.9210202395915985, "l_c": 2.885891079902649, "z_loss": 41.23802328109741, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17103137075901031, "tokens_per_second": 1381923.9568733682, "eta_seconds": 4339.841133928299, "data_wait_seconds": 1.6301870346069336e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 61050, "total_steps": 72479, "tokens": 32007782400, "loss": 4.650271236896515, "l_mix": 2.8853874504566193, "l_a": 2.961555451154709, "l_c": 2.921390026807785, "z_loss": 41.45548343658447, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14621227979660034, "tokens_per_second": 1372259.7018964253, "eta_seconds": 4366.584214138985, "data_wait_seconds": 2.580881118774414e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 61060, "total_steps": 72479, "tokens": 32013025280, "loss": 4.589254438877106, "l_mix": 2.8463760316371918, "l_a": 2.9263675808906555, "l_c": 2.8832268118858337, "z_loss": 42.50205183029175, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15826988220214844, "tokens_per_second": 1376349.5424903526, "eta_seconds": 4349.799587368965, "data_wait_seconds": 2.5928020477294922e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 61070, "total_steps": 72479, "tokens": 32018268160, "loss": 2.8362038135528564, "l_mix": 1.7562205940485, "l_a": 1.8201589733362198, "l_c": 1.779785007238388, "z_loss": 52.076308727264404, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1992139220237732, "tokens_per_second": 1427943.947755077, "eta_seconds": 4188.961199355126, "data_wait_seconds": 2.3037195205688477e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 61080, "total_steps": 72479, "tokens": 32023511040, "loss": 4.302845895290375, "l_mix": 2.666033625602722, "l_a": 2.7524511516094208, "l_c": 2.703589290380478, "z_loss": 43.068912982940674, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13590142130851746, "tokens_per_second": 1412576.8730478107, "eta_seconds": 4230.820301556588, "data_wait_seconds": 1.7702579498291016e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 61090, "total_steps": 72479, "tokens": 32028753920, "loss": 4.338774561882019, "l_mix": 2.689339280128479, "l_a": 2.7721555829048157, "l_c": 2.725961923599243, "z_loss": 42.666316986083984, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11905346810817719, "tokens_per_second": 1305455.9917582015, "eta_seconds": 4573.969608855247, "data_wait_seconds": 2.0295381546020508e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 61100, "total_steps": 72479, "tokens": 32033996800, "loss": 4.274023950099945, "l_mix": 2.648864448070526, "l_a": 2.733451187610626, "l_c": 2.6837474405765533, "z_loss": 42.57718324661255, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14641112089157104, "tokens_per_second": 1414917.639700406, "eta_seconds": 4216.4101885557175, "data_wait_seconds": 1.5616416931152344e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16243473, "gate_a2b/L1_absmax": 0.69895524, "gate_a2b/L1_rms": 0.17264585, "gate_a2b/L3_absmean": 0.1523339, "gate_a2b/L3_absmax": 0.50343859, "gate_a2b/L3_rms": 0.15804923, "gate_a2b/L5_absmean": 0.22448647, "gate_a2b/L5_absmax": 0.73703402, "gate_a2b/L5_rms": 0.23058875, "gate_c2b/L1_absmean": 0.03518503, "gate_c2b/L1_absmax": 0.63390988, "gate_c2b/L1_rms": 0.07003017, "gate_c2b/L3_absmean": 0.16983478, "gate_c2b/L3_absmax": 0.39533398, "gate_c2b/L3_rms": 0.17351094, "gate_c2b/L5_absmean": 0.48063976, "gate_c2b/L5_absmax": 0.67958444, "gate_c2b/L5_rms": 0.49018937, "gate_b2a/L7_absmean": 0.0513633, "gate_b2a/L7_absmax": 0.33300933, "gate_b2a/L7_rms": 0.05723639, "gate_b2a/L11_absmean": 0.08280565, "gate_b2a/L11_absmax": 0.61270684, "gate_b2a/L11_rms": 0.09206092, "gate_b2a/L14_absmean": 0.21515365, "gate_b2a/L14_absmax": 0.82766032, "gate_b2a/L14_rms": 0.22481628, "gate_b2c/L3_absmean": 0.03495027, "gate_b2c/L3_absmax": 0.5545249, "gate_b2c/L3_rms": 0.04562245, "gate_b2c/L5_absmean": 0.03677854, "gate_b2c/L5_absmax": 0.52238083, "gate_b2c/L5_rms": 0.04581586, "gate_b2c/L6_absmean": 0.05311903, "gate_b2c/L6_absmax": 0.34690991, "gate_b2c/L6_rms": 0.05929846}, "streams": {"cos_a2b/round0": 0.355857, "cos_c2b/round0": 0.250938, "cos_b2a/round0": 0.166762, "cos_b2c/round0": 0.14911, "rms_a/round0": 7.672689, "rms_b/round0": 13.084872, "rms_c/round0": 4.905606, "cos_a2b/round1": 0.415456, "cos_c2b/round1": 0.30556, "cos_b2a/round1": 0.255172, "cos_b2c/round1": 0.116561, "rms_a/round1": 15.07156, "rms_b/round1": 30.377115, "rms_c/round1": 10.372625, "cos_a2b/round2": 0.543459, "cos_c2b/round2": 0.520629, "cos_b2a/round2": 0.100022, "cos_b2c/round2": 0.186468, "rms_a/round2": 42.854378, "rms_b/round2": 82.962997, "rms_c/round2": 20.077448}, "alpha_mean": 0.43730162994766886, "alpha_hist": [0.00693, 0.06825, 0.18469, 0.20265, 0.1757, 0.14835, 0.12395, 0.07637, 0.01295, 0.00015], "alpha_by_type": {"special": {"share": 0.00087, "mean": 0.56231, "std": 0.1002}, "whitespace": {"share": 0.02318, "mean": 0.53248, "std": 0.11271}, "digit": {"share": 0.02718, "mean": 0.45304, "std": 0.11746}, "word_start": {"share": 0.60573, "mean": 0.4498, "std": 0.17273}, "word_cont": {"share": 0.24086, "mean": 0.32586, "std": 0.12296}, "punct": {"share": 0.10219, "mean": 0.59903, "std": 0.1617}}, "alpha_p05": 0.18477, "alpha_p50": 0.43158, "alpha_p95": 0.72526}
{"step": 61110, "total_steps": 72479, "tokens": 32039239680, "loss": 4.235199570655823, "l_mix": 2.624520868062973, "l_a": 2.7100559771060944, "l_c": 2.6588729321956635, "z_loss": 42.0937614440918, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.149393230676651, "tokens_per_second": 1476722.9262953154, "eta_seconds": 4036.3904195308687, "data_wait_seconds": 2.0503997802734375e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 61120, "total_steps": 72479, "tokens": 32044482560, "loss": 4.2927194237709045, "l_mix": 2.6603972017765045, "l_a": 2.7454120218753815, "l_c": 2.6956617534160614, "z_loss": 42.75532913208008, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11666365712881088, "tokens_per_second": 1319095.2071653407, "eta_seconds": 4514.751747751236, "data_wait_seconds": 1.6301870346069336e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 61130, "total_steps": 72479, "tokens": 32049725440, "loss": 2.8960098028182983, "l_mix": 1.7934896051883698, "l_a": 1.8568773716688156, "l_c": 1.8181897550821304, "z_loss": 50.0220685005188, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13303318619728088, "tokens_per_second": 1299935.6397096522, "eta_seconds": 4577.260850644112, "data_wait_seconds": 3.3855438232421875e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 61140, "total_steps": 72479, "tokens": 32054968320, "loss": 4.7508784532547, "l_mix": 2.948702424764633, "l_a": 3.022108644247055, "l_c": 2.9851444363594055, "z_loss": 42.31036424636841, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12483727931976318, "tokens_per_second": 1481524.2975850264, "eta_seconds": 4012.6926312923433, "data_wait_seconds": 2.6494264602661133e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 61150, "total_steps": 72479, "tokens": 32060211200, "loss": 4.707998275756836, "l_mix": 2.9217187762260437, "l_a": 2.994938224554062, "l_c": 2.959326833486557, "z_loss": 40.534481048583984, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13640698790550232, "tokens_per_second": 1409519.9699767088, "eta_seconds": 4213.958566403389, "data_wait_seconds": 2.5600194931030273e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 61160, "total_steps": 72479, "tokens": 32065454080, "loss": 4.775031745433807, "l_mix": 2.963966429233551, "l_a": 3.039729982614517, "l_c": 2.9971542358398438, "z_loss": 39.4382483959198, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15295302867889404, "tokens_per_second": 1265358.4963534009, "eta_seconds": 4689.908740568161, "data_wait_seconds": 2.3871660232543945e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 61170, "total_steps": 72479, "tokens": 32070696960, "loss": 4.529743432998657, "l_mix": 2.8095976412296295, "l_a": 2.889679491519928, "l_c": 2.8441396355628967, "z_loss": 41.343679428100586, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1706773042678833, "tokens_per_second": 1466124.583370336, "eta_seconds": 4044.1126622200013, "data_wait_seconds": 2.0205974578857422e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 61180, "total_steps": 72479, "tokens": 32075939840, "loss": 4.639697849750519, "l_mix": 2.8792546689510345, "l_a": 2.9543983042240143, "l_c": 2.913745194673538, "z_loss": 40.78326988220215, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1508544236421585, "tokens_per_second": 1422522.1727732264, "eta_seconds": 4164.385079813003, "data_wait_seconds": 1.7076730728149414e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 61190, "total_steps": 72479, "tokens": 32081182720, "loss": 4.305445730686188, "l_mix": 2.6678375601768494, "l_a": 2.7550948560237885, "l_c": 2.703598767518997, "z_loss": 42.77416944503784, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13275445997714996, "tokens_per_second": 1388104.0898845652, "eta_seconds": 4263.864125990867, "data_wait_seconds": 2.238154411315918e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 61200, "total_steps": 72479, "tokens": 32086425600, "loss": 4.255658566951752, "l_mix": 2.6375383734703064, "l_a": 2.7225802540779114, "l_c": 2.671153575181961, "z_loss": 41.3405065536499, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14966236054897308, "tokens_per_second": 1365893.2518518404, "eta_seconds": 4329.360544085503, "data_wait_seconds": 1.755356788635254e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16130191, "gate_a2b/L1_absmax": 0.69860971, "gate_a2b/L1_rms": 0.1717016, "gate_a2b/L3_absmean": 0.15279274, "gate_a2b/L3_absmax": 0.50145656, "gate_a2b/L3_rms": 0.15848844, "gate_a2b/L5_absmean": 0.22439098, "gate_a2b/L5_absmax": 0.73767817, "gate_a2b/L5_rms": 0.23048079, "gate_c2b/L1_absmean": 0.03568361, "gate_c2b/L1_absmax": 0.63313657, "gate_c2b/L1_rms": 0.07005204, "gate_c2b/L3_absmean": 0.16973852, "gate_c2b/L3_absmax": 0.39191854, "gate_c2b/L3_rms": 0.17336734, "gate_c2b/L5_absmean": 0.47987938, "gate_c2b/L5_absmax": 0.67940569, "gate_c2b/L5_rms": 0.48937991, "gate_b2a/L7_absmean": 0.05054555, "gate_b2a/L7_absmax": 0.33519703, "gate_b2a/L7_rms": 0.05657473, "gate_b2a/L11_absmean": 0.08292405, "gate_b2a/L11_absmax": 0.61475557, "gate_b2a/L11_rms": 0.09221543, "gate_b2a/L14_absmean": 0.21481188, "gate_b2a/L14_absmax": 0.83079356, "gate_b2a/L14_rms": 0.22449321, "gate_b2c/L3_absmean": 0.0346853, "gate_b2c/L3_absmax": 0.55498898, "gate_b2c/L3_rms": 0.04548308, "gate_b2c/L5_absmean": 0.03672298, "gate_b2c/L5_absmax": 0.52214527, "gate_b2c/L5_rms": 0.04574136, "gate_b2c/L6_absmean": 0.05341439, "gate_b2c/L6_absmax": 0.34555668, "gate_b2c/L6_rms": 0.05948868}, "streams": {"cos_a2b/round0": 0.358475, "cos_c2b/round0": 0.25082, "cos_b2a/round0": 0.156593, "cos_b2c/round0": 0.146386, "rms_a/round0": 7.726361, "rms_b/round0": 13.100479, "rms_c/round0": 4.925004, "cos_a2b/round1": 0.412275, "cos_c2b/round1": 0.301247, "cos_b2a/round1": 0.252616, "cos_b2c/round1": 0.109666, "rms_a/round1": 15.1436, "rms_b/round1": 30.566942, "rms_c/round1": 10.540484, "cos_a2b/round2": 0.53488, "cos_c2b/round2": 0.519791, "cos_b2a/round2": 0.089827, "cos_b2c/round2": 0.18273, "rms_a/round2": 43.304947, "rms_b/round2": 83.793587, "rms_c/round2": 20.400167}, "alpha_mean": 0.4229843835953915, "alpha_hist": [0.0072, 0.07176, 0.20114, 0.20889, 0.17839, 0.15274, 0.1161, 0.0554, 0.0083, 6e-05], "alpha_by_type": {"special": {"share": 0.00084, "mean": 0.5426, "std": 0.10341}, "whitespace": {"share": 0.02644, "mean": 0.51823, "std": 0.11467}, "digit": {"share": 0.04784, "mean": 0.41709, "std": 0.11292}, "word_start": {"share": 0.56436, "mean": 0.43952, "std": 0.16497}, "word_cont": {"share": 0.25285, "mean": 0.31294, "std": 0.12108}, "punct": {"share": 0.10767, "mean": 0.57305, "std": 0.16074}}, "alpha_p05": 0.1833, "alpha_p50": 0.40875, "alpha_p95": 0.71879}
{"step": 61210, "total_steps": 72479, "tokens": 32091668480, "loss": 4.279145240783691, "l_mix": 2.651775360107422, "l_a": 2.7378274500370026, "l_c": 2.6867386996746063, "z_loss": 41.39149761199951, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16949057579040527, "tokens_per_second": 1337546.157050953, "eta_seconds": 4417.194457817078, "data_wait_seconds": 2.1785497665405273e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 61220, "total_steps": 72479, "tokens": 32096911360, "loss": 4.232721567153931, "l_mix": 2.624106913805008, "l_a": 2.700701743364334, "l_c": 2.6613471806049347, "z_loss": 42.619596004486084, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16939470171928406, "tokens_per_second": 1442000.0503297942, "eta_seconds": 4093.5911137104035, "data_wait_seconds": 1.6689300537109375e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 61230, "total_steps": 72479, "tokens": 32102154240, "loss": 4.30248361825943, "l_mix": 2.6659701466560364, "l_a": 2.7540169060230255, "l_c": 2.701027810573578, "z_loss": 42.88135242462158, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1433132141828537, "tokens_per_second": 1460905.063003605, "eta_seconds": 4037.0287305831907, "data_wait_seconds": 2.4080276489257812e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 61240, "total_steps": 72479, "tokens": 32107397120, "loss": 2.9448330104351044, "l_mix": 1.823703020811081, "l_a": 1.888214036822319, "l_c": 1.848885715007782, "z_loss": 52.45456123352051, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14419831335544586, "tokens_per_second": 1430570.4503946323, "eta_seconds": 4118.967248606682, "data_wait_seconds": 5.65648078918457e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 61250, "total_steps": 72479, "tokens": 32112640000, "loss": 3.0768086910247803, "l_mix": 1.8929401487112045, "l_a": 2.0035476237535477, "l_c": 1.9426806420087814, "z_loss": 45.230966567993164, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.2526697814464569, "tokens_per_second": 1367319.9776903493, "eta_seconds": 4305.670982694625, "data_wait_seconds": 2.1189451217651367e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 61260, "total_steps": 72479, "tokens": 32117882880, "loss": 3.09468150138855, "l_mix": 1.90562042593956, "l_a": 2.0140029937028885, "l_c": 1.9495338797569275, "z_loss": 43.371957302093506, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.23194822669029236, "tokens_per_second": 1395315.6946431801, "eta_seconds": 4215.524196124077, "data_wait_seconds": 2.9742717742919922e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 61270, "total_steps": 72479, "tokens": 32123125760, "loss": 4.335343658924103, "l_mix": 2.6868121922016144, "l_a": 2.771440237760544, "l_c": 2.7236646115779877, "z_loss": 43.046578884124756, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12671910226345062, "tokens_per_second": 1453897.6544574716, "eta_seconds": 4042.0618149995803, "data_wait_seconds": 2.1845102310180664e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 61280, "total_steps": 72479, "tokens": 32128368640, "loss": 4.270690619945526, "l_mix": 2.6464240849018097, "l_a": 2.730249136686325, "l_c": 2.683972656726837, "z_loss": 43.807015895843506, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15270942449569702, "tokens_per_second": 1349449.5148304712, "eta_seconds": 4351.034438467026, "data_wait_seconds": 1.9848346710205078e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 61290, "total_steps": 72479, "tokens": 32133611520, "loss": 4.281554281711578, "l_mix": 2.654387950897217, "l_a": 2.7351717352867126, "l_c": 2.6887157559394836, "z_loss": 42.784953594207764, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1605946272611618, "tokens_per_second": 1429679.5312313084, "eta_seconds": 4103.198167037964, "data_wait_seconds": 1.728534698486328e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 61300, "total_steps": 72479, "tokens": 32138854400, "loss": 4.234015643596649, "l_mix": 2.622688740491867, "l_a": 2.710376501083374, "l_c": 2.660712778568268, "z_loss": 43.79351472854614, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17862334847450256, "tokens_per_second": 1468581.3110626012, "eta_seconds": 3990.93704097271, "data_wait_seconds": 1.9371509552001953e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.16157542, "gate_a2b/L1_absmax": 0.69635653, "gate_a2b/L1_rms": 0.17179069, "gate_a2b/L3_absmean": 0.15318727, "gate_a2b/L3_absmax": 0.49937147, "gate_a2b/L3_rms": 0.15883924, "gate_a2b/L5_absmean": 0.22381078, "gate_a2b/L5_absmax": 0.73689979, "gate_a2b/L5_rms": 0.22994229, "gate_c2b/L1_absmean": 0.03518681, "gate_c2b/L1_absmax": 0.63063735, "gate_c2b/L1_rms": 0.06961788, "gate_c2b/L3_absmean": 0.17014265, "gate_c2b/L3_absmax": 0.38915297, "gate_c2b/L3_rms": 0.17372507, "gate_c2b/L5_absmean": 0.47965685, "gate_c2b/L5_absmax": 0.67243475, "gate_c2b/L5_rms": 0.48911774, "gate_b2a/L7_absmean": 0.05088209, "gate_b2a/L7_absmax": 0.33443484, "gate_b2a/L7_rms": 0.05684702, "gate_b2a/L11_absmean": 0.08289792, "gate_b2a/L11_absmax": 0.61608821, "gate_b2a/L11_rms": 0.09224515, "gate_b2a/L14_absmean": 0.21556938, "gate_b2a/L14_absmax": 0.82789475, "gate_b2a/L14_rms": 0.22509828, "gate_b2c/L3_absmean": 0.03469532, "gate_b2c/L3_absmax": 0.55399573, "gate_b2c/L3_rms": 0.04542783, "gate_b2c/L5_absmean": 0.03691602, "gate_b2c/L5_absmax": 0.52126867, "gate_b2c/L5_rms": 0.04588013, "gate_b2c/L6_absmean": 0.05361154, "gate_b2c/L6_absmax": 0.34649852, "gate_b2c/L6_rms": 0.05968629}, "streams": {"cos_a2b/round0": 0.356101, "cos_c2b/round0": 0.24701, "cos_b2a/round0": 0.159953, "cos_b2c/round0": 0.1451, "rms_a/round0": 7.767334, "rms_b/round0": 13.136859, "rms_c/round0": 4.952277, "cos_a2b/round1": 0.413383, "cos_c2b/round1": 0.302896, "cos_b2a/round1": 0.252621, "cos_b2c/round1": 0.115504, "rms_a/round1": 15.360568, "rms_b/round1": 30.953499, "rms_c/round1": 10.667522, "cos_a2b/round2": 0.529611, "cos_c2b/round2": 0.525704, "cos_b2a/round2": 0.096523, "cos_b2c/round2": 0.184313, "rms_a/round2": 43.40699, "rms_b/round2": 83.817429, "rms_c/round2": 20.879324}, "alpha_mean": 0.41959887658630635, "alpha_hist": [0.01085, 0.0975, 0.19435, 0.19154, 0.17177, 0.1461, 0.10994, 0.06654, 0.01126, 0.00014], "alpha_by_type": {"special": {"share": 0.00148, "mean": 0.55628, "std": 0.1068}, "whitespace": {"share": 0.02901, "mean": 0.52374, "std": 0.12077}, "digit": {"share": 0.03772, "mean": 0.43591, "std": 0.12116}, "word_start": {"share": 0.57475, "mean": 0.43107, "std": 0.1767}, "word_cont": {"share": 0.24854, "mean": 0.31444, "std": 0.13304}, "punct": {"share": 0.10851, "mean": 0.56431, "std": 0.17239}}, "alpha_p05": 0.13916, "alpha_p50": 0.4031, "alpha_p95": 0.73105}
{"step": 61310, "total_steps": 72479, "tokens": 32144097280, "loss": 4.246610462665558, "l_mix": 2.63175567984581, "l_a": 2.7161199152469635, "l_c": 2.666729062795639, "z_loss": 43.675657749176025, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16504141688346863, "tokens_per_second": 1417962.731866212, "eta_seconds": 4129.708447480201, "data_wait_seconds": 1.5854835510253906e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 61320, "total_steps": 72479, "tokens": 32149340160, "loss": 4.213302731513977, "l_mix": 2.6104688346385956, "l_a": 2.6970848739147186, "l_c": 2.645694464445114, "z_loss": 42.969350814819336, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.10295291990041733, "tokens_per_second": 1384489.5591149384, "eta_seconds": 4225.766639757157, "data_wait_seconds": 1.8328428268432617e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 61330, "total_steps": 72479, "tokens": 32154583040, "loss": 4.269253194332123, "l_mix": 2.645286947488785, "l_a": 2.7320918440818787, "l_c": 2.6811288595199585, "z_loss": 42.872840881347656, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17147088050842285, "tokens_per_second": 1381730.5829490467, "eta_seconds": 4230.410026478768, "data_wait_seconds": 3.102421760559082e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 61340, "total_steps": 72479, "tokens": 32159825920, "loss": 4.289823234081268, "l_mix": 2.6589419543743134, "l_a": 2.744278699159622, "l_c": 2.691992372274399, "z_loss": 42.6989164352417, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.15535074472427368, "tokens_per_second": 1388621.9550819963, "eta_seconds": 4205.639994835854, "data_wait_seconds": 1.8358230590820312e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 61350, "total_steps": 72479, "tokens": 32165068800, "loss": 3.012556493282318, "l_mix": 1.8651429116725922, "l_a": 1.9318891316652298, "l_c": 1.892822504043579, "z_loss": 52.526020526885986, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.25340789556503296, "tokens_per_second": 1418195.8312010102, "eta_seconds": 4114.242210865021, "data_wait_seconds": 1.9878149032592773e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 61360, "total_steps": 72479, "tokens": 32170311680, "loss": 3.086515873670578, "l_mix": 1.9002097398042679, "l_a": 2.009498119354248, "l_c": 1.944855660200119, "z_loss": 45.1902379989624, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1651354879140854, "tokens_per_second": 1450410.1540775716, "eta_seconds": 4019.2481110334397, "data_wait_seconds": 1.773238182067871e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 61370, "total_steps": 72479, "tokens": 32175554560, "loss": 3.096953511238098, "l_mix": 1.9057709872722626, "l_a": 2.014945834875107, "l_c": 1.9556622058153152, "z_loss": 47.10129976272583, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1514347642660141, "tokens_per_second": 1442459.7513065634, "eta_seconds": 4037.7663132190705, "data_wait_seconds": 3.314018249511719e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 61380, "total_steps": 72479, "tokens": 32180797440, "loss": 4.279335379600525, "l_mix": 2.652725428342819, "l_a": 2.735185295343399, "l_c": 2.6868476569652557, "z_loss": 42.315614223480225, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13564051687717438, "tokens_per_second": 1358676.9453508717, "eta_seconds": 4282.896336698532, "data_wait_seconds": 3.0517578125e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 61390, "total_steps": 72479, "tokens": 32186040320, "loss": 4.318206429481506, "l_mix": 2.6764198541641235, "l_a": 2.7591691315174103, "l_c": 2.713452488183975, "z_loss": 42.12557792663574, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1739007532596588, "tokens_per_second": 1393928.2266732794, "eta_seconds": 4170.824236679077, "data_wait_seconds": 1.6391277313232422e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 61400, "total_steps": 72479, "tokens": 32191283200, "loss": 4.266737401485443, "l_mix": 2.643776386976242, "l_a": 2.7306696474552155, "l_c": 2.679199993610382, "z_loss": 42.19516563415527, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.18271300196647644, "tokens_per_second": 1371083.0821746432, "eta_seconds": 4236.495094656944, "data_wait_seconds": 1.7255544662475586e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.16154565, "gate_a2b/L1_absmax": 0.69577986, "gate_a2b/L1_rms": 0.17182024, "gate_a2b/L3_absmean": 0.15270589, "gate_a2b/L3_absmax": 0.49980807, "gate_a2b/L3_rms": 0.15837522, "gate_a2b/L5_absmean": 0.22370005, "gate_a2b/L5_absmax": 0.73586619, "gate_a2b/L5_rms": 0.22978896, "gate_c2b/L1_absmean": 0.0358516, "gate_c2b/L1_absmax": 0.6305868, "gate_c2b/L1_rms": 0.06969693, "gate_c2b/L3_absmean": 0.16981582, "gate_c2b/L3_absmax": 0.39055151, "gate_c2b/L3_rms": 0.17340586, "gate_c2b/L5_absmean": 0.47945234, "gate_c2b/L5_absmax": 0.67611527, "gate_c2b/L5_rms": 0.48893547, "gate_b2a/L7_absmean": 0.05084093, "gate_b2a/L7_absmax": 0.33351427, "gate_b2a/L7_rms": 0.05680183, "gate_b2a/L11_absmean": 0.08296283, "gate_b2a/L11_absmax": 0.61411399, "gate_b2a/L11_rms": 0.09225568, "gate_b2a/L14_absmean": 0.21540743, "gate_b2a/L14_absmax": 0.82732379, "gate_b2a/L14_rms": 0.22494929, "gate_b2c/L3_absmean": 0.03496396, "gate_b2c/L3_absmax": 0.55419981, "gate_b2c/L3_rms": 0.04560309, "gate_b2c/L5_absmean": 0.0365616, "gate_b2c/L5_absmax": 0.52273667, "gate_b2c/L5_rms": 0.04564358, "gate_b2c/L6_absmean": 0.05332391, "gate_b2c/L6_absmax": 0.34466264, "gate_b2c/L6_rms": 0.05933549}, "streams": {"cos_a2b/round0": 0.3563, "cos_c2b/round0": 0.24746, "cos_b2a/round0": 0.163969, "cos_b2c/round0": 0.14655, "rms_a/round0": 7.713703, "rms_b/round0": 13.2049, "rms_c/round0": 5.003441, "cos_a2b/round1": 0.417144, "cos_c2b/round1": 0.30741, "cos_b2a/round1": 0.250584, "cos_b2c/round1": 0.113086, "rms_a/round1": 15.3223, "rms_b/round1": 30.649782, "rms_c/round1": 10.611884, "cos_a2b/round2": 0.53684, "cos_c2b/round2": 0.524335, "cos_b2a/round2": 0.096679, "cos_b2c/round2": 0.186499, "rms_a/round2": 43.288132, "rms_b/round2": 83.234848, "rms_c/round2": 20.678421}, "alpha_mean": 0.43782569384922, "alpha_hist": [0.00824, 0.07298, 0.17564, 0.20108, 0.1739, 0.15501, 0.12727, 0.07326, 0.01247, 0.00014], "alpha_by_type": {"special": {"share": 0.00116, "mean": 0.57533, "std": 0.10517}, "whitespace": {"share": 0.01913, "mean": 0.54426, "std": 0.1185}, "digit": {"share": 0.02112, "mean": 0.47379, "std": 0.12024}, "word_start": {"share": 0.62244, "mean": 0.45165, "std": 0.16982}, "word_cont": {"share": 0.2439, "mean": 0.32525, "std": 0.13171}, "punct": {"share": 0.09225, "mean": 0.61012, "std": 0.15271}}, "alpha_p05": 0.17189, "alpha_p50": 0.40404, "alpha_p95": 0.73716}
{"step": 61410, "total_steps": 72479, "tokens": 32196526080, "loss": 4.318771481513977, "l_mix": 2.6766285598278046, "l_a": 2.76110902428627, "l_c": 2.7127005457878113, "z_loss": 42.00827360153198, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17090655863285065, "tokens_per_second": 1359371.2016837974, "eta_seconds": 4269.1384552001955, "data_wait_seconds": 1.7464160919189453e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 61420, "total_steps": 72479, "tokens": 32201768960, "loss": 4.582525074481964, "l_mix": 2.8431249856948853, "l_a": 2.9186929762363434, "l_c": 2.879307359457016, "z_loss": 40.91924858093262, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1189248189330101, "tokens_per_second": 1391302.0282628092, "eta_seconds": 4167.3920358181, "data_wait_seconds": 2.434849739074707e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 61430, "total_steps": 72479, "tokens": 32207011840, "loss": 4.6602590680122375, "l_mix": 2.8915185034275055, "l_a": 2.9669887125492096, "l_c": 2.9288129210472107, "z_loss": 40.31319618225098, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.13937978446483612, "tokens_per_second": 1436237.887823888, "eta_seconds": 4033.355589008331, "data_wait_seconds": 1.996755599975586e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 61440, "total_steps": 72479, "tokens": 32212254720, "loss": 4.577175557613373, "l_mix": 2.8393874168395996, "l_a": 2.916378766298294, "l_c": 2.8762481212615967, "z_loss": 41.75961637496948, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1034015342593193, "tokens_per_second": 1260415.0828833086, "eta_seconds": 4591.832730817795, "data_wait_seconds": 2.467632293701172e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 61450, "total_steps": 72479, "tokens": 32217497600, "loss": 4.811494588851929, "l_mix": 2.986114650964737, "l_a": 3.06221279501915, "l_c": 3.0223866403102875, "z_loss": 39.829039096832275, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1252402365207672, "tokens_per_second": 1353990.9001887748, "eta_seconds": 4270.613894963264, "data_wait_seconds": 1.722574234008789e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 61460, "total_steps": 72479, "tokens": 32222740480, "loss": 4.6684921979904175, "l_mix": 2.8967712819576263, "l_a": 2.9724238216876984, "l_c": 2.9333123862743378, "z_loss": 39.928030490875244, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12895900011062622, "tokens_per_second": 1438866.3028286216, "eta_seconds": 4015.0564792871473, "data_wait_seconds": 3.2842159271240234e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 61470, "total_steps": 72479, "tokens": 32227983360, "loss": 3.017217993736267, "l_mix": 1.8681512027978897, "l_a": 1.9327281713485718, "l_c": 1.8974943906068802, "z_loss": 51.271825313568115, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17371515929698944, "tokens_per_second": 1425564.8169085886, "eta_seconds": 4048.8419211387636, "data_wait_seconds": 1.6421079635620117e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 61480, "total_steps": 72479, "tokens": 32233226240, "loss": 3.0980976819992065, "l_mix": 1.9086787849664688, "l_a": 2.013801336288452, "l_c": 1.9509281814098358, "z_loss": 43.864909172058105, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.24082469940185547, "tokens_per_second": 1442789.6679978594, "eta_seconds": 3996.870673465729, "data_wait_seconds": 1.6927719116210938e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 61490, "total_steps": 72479, "tokens": 32238469120, "loss": 4.237800180912018, "l_mix": 2.625809371471405, "l_a": 2.7121667861938477, "l_c": 2.661135673522949, "z_loss": 43.01182460784912, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11393885314464569, "tokens_per_second": 1432322.9704670373, "eta_seconds": 4022.417395234108, "data_wait_seconds": 3.2454729080200195e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 61500, "total_steps": 72479, "tokens": 32243712000, "loss": 4.285863280296326, "l_mix": 2.6559709906578064, "l_a": 2.7409363985061646, "l_c": 2.6920372247695923, "z_loss": 43.16882038116455, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1726096123456955, "tokens_per_second": 1403206.6637713737, "eta_seconds": 4102.145536088944, "data_wait_seconds": 2.467632293701172e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16076098, "gate_a2b/L1_absmax": 0.69776565, "gate_a2b/L1_rms": 0.17106012, "gate_a2b/L3_absmean": 0.15233193, "gate_a2b/L3_absmax": 0.50030625, "gate_a2b/L3_rms": 0.15804732, "gate_a2b/L5_absmean": 0.22360145, "gate_a2b/L5_absmax": 0.73082238, "gate_a2b/L5_rms": 0.22966373, "gate_c2b/L1_absmean": 0.03639417, "gate_c2b/L1_absmax": 0.6341809, "gate_c2b/L1_rms": 0.07011899, "gate_c2b/L3_absmean": 0.16948706, "gate_c2b/L3_absmax": 0.38993499, "gate_c2b/L3_rms": 0.17308268, "gate_c2b/L5_absmean": 0.47979289, "gate_c2b/L5_absmax": 0.68206376, "gate_c2b/L5_rms": 0.48934713, "gate_b2a/L7_absmean": 0.05124878, "gate_b2a/L7_absmax": 0.33489624, "gate_b2a/L7_rms": 0.05713354, "gate_b2a/L11_absmean": 0.08295756, "gate_b2a/L11_absmax": 0.61355299, "gate_b2a/L11_rms": 0.09216093, "gate_b2a/L14_absmean": 0.21547866, "gate_b2a/L14_absmax": 0.83091754, "gate_b2a/L14_rms": 0.22506323, "gate_b2c/L3_absmean": 0.03497774, "gate_b2c/L3_absmax": 0.55290157, "gate_b2c/L3_rms": 0.04556187, "gate_b2c/L5_absmean": 0.03681352, "gate_b2c/L5_absmax": 0.52010304, "gate_b2c/L5_rms": 0.04575891, "gate_b2c/L6_absmean": 0.05323568, "gate_b2c/L6_absmax": 0.34659183, "gate_b2c/L6_rms": 0.05935319}, "streams": {"cos_a2b/round0": 0.351193, "cos_c2b/round0": 0.242255, "cos_b2a/round0": 0.160208, "cos_b2c/round0": 0.145092, "rms_a/round0": 7.693652, "rms_b/round0": 13.158219, "rms_c/round0": 4.966512, "cos_a2b/round1": 0.41476, "cos_c2b/round1": 0.309023, "cos_b2a/round1": 0.251602, "cos_b2c/round1": 0.108292, "rms_a/round1": 14.89278, "rms_b/round1": 30.154551, "rms_c/round1": 10.413573, "cos_a2b/round2": 0.53947, "cos_c2b/round2": 0.520296, "cos_b2a/round2": 0.097373, "cos_b2c/round2": 0.181948, "rms_a/round2": 42.622375, "rms_b/round2": 82.563904, "rms_c/round2": 19.648472}, "alpha_mean": 0.45014466832071776, "alpha_hist": [0.00635, 0.05035, 0.15797, 0.19685, 0.19437, 0.17554, 0.14651, 0.06587, 0.0061, 8e-05], "alpha_by_type": {"special": {"share": 0.00046, "mean": 0.52751, "std": 0.10893}, "whitespace": {"share": 0.02353, "mean": 0.47904, "std": 0.12737}, "digit": {"share": 0.02026, "mean": 0.46851, "std": 0.10877}, "word_start": {"share": 0.56828, "mean": 0.47067, "std": 0.16161}, "word_cont": {"share": 0.27722, "mean": 0.34509, "std": 0.11927}, "punct": {"share": 0.11024, "mean": 0.59863, "std": 0.14676}}, "alpha_p05": 0.19714, "alpha_p50": 0.43254, "alpha_p95": 0.72272}
{"step": 61510, "total_steps": 72479, "tokens": 32248954880, "loss": 4.2990663051605225, "l_mix": 2.6636217534542084, "l_a": 2.74764421582222, "l_c": 2.703837662935257, "z_loss": 43.11884164810181, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.19900831580162048, "tokens_per_second": 1315134.1123892516, "eta_seconds": 4372.873471856117, "data_wait_seconds": 1.7434358596801758e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 61520, "total_steps": 72479, "tokens": 32254197760, "loss": 4.2559274435043335, "l_mix": 2.6375403106212616, "l_a": 2.7227850258350372, "l_c": 2.671838492155075, "z_loss": 42.77714157104492, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.17781096696853638, "tokens_per_second": 1413829.092535132, "eta_seconds": 4063.9085886240005, "data_wait_seconds": 2.4616718292236328e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 61530, "total_steps": 72479, "tokens": 32259440640, "loss": 3.106070399284363, "l_mix": 1.91059809923172, "l_a": 2.0243842005729675, "l_c": 1.9605231285095215, "z_loss": 46.60388231277466, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16565914452075958, "tokens_per_second": 1399856.5118019897, "eta_seconds": 4100.726941370964, "data_wait_seconds": 1.7464160919189453e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 61540, "total_steps": 72479, "tokens": 32264683520, "loss": 3.1323061287403107, "l_mix": 1.9281370341777802, "l_a": 2.0393472760915756, "l_c": 1.9745496809482574, "z_loss": 44.595505714416504, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16182692348957062, "tokens_per_second": 1299932.1048633913, "eta_seconds": 4411.912291836738, "data_wait_seconds": 3.421306610107422e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 61550, "total_steps": 72479, "tokens": 32269926400, "loss": 4.52113401889801, "l_mix": 2.80542129278183, "l_a": 2.8787828385829926, "l_c": 2.8402595818042755, "z_loss": 41.673226833343506, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.11416205763816833, "tokens_per_second": 1269675.1407118905, "eta_seconds": 4512.920957708358, "data_wait_seconds": 2.1845102310180664e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 61560, "total_steps": 72479, "tokens": 32275169280, "loss": 4.675228118896484, "l_mix": 2.901163548231125, "l_a": 2.976272404193878, "l_c": 2.9372761845588684, "z_loss": 40.2405686378479, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1328180581331253, "tokens_per_second": 1421614.5022746883, "eta_seconds": 4026.900867176056, "data_wait_seconds": 3.93986701965332e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 61570, "total_steps": 72479, "tokens": 32280412160, "loss": 4.615014016628265, "l_mix": 2.863262712955475, "l_a": 2.939922422170639, "l_c": 2.8992486596107483, "z_loss": 41.491150856018066, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.1180320605635643, "tokens_per_second": 1321393.956463444, "eta_seconds": 4328.35171072483, "data_wait_seconds": 1.710653305053711e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 61580, "total_steps": 72479, "tokens": 32285655040, "loss": 4.6288933753967285, "l_mix": 2.871503382921219, "l_a": 2.9500421285629272, "l_c": 2.907924383878708, "z_loss": 41.81801795959473, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.12093445658683777, "tokens_per_second": 1409741.8062121067, "eta_seconds": 4053.376928186417, "data_wait_seconds": 2.3037195205688477e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 61590, "total_steps": 72479, "tokens": 32290897920, "loss": 3.0511209070682526, "l_mix": 1.8783030658960342, "l_a": 1.9863301068544388, "l_c": 1.923062488436699, "z_loss": 44.79209613800049, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.16221486032009125, "tokens_per_second": 1445283.8701072251, "eta_seconds": 3950.0697060823445, "data_wait_seconds": 2.250075340270996e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 61600, "total_steps": 72479, "tokens": 32296140800, "loss": 4.261847674846649, "l_mix": 2.640088826417923, "l_a": 2.7281605899333954, "l_c": 2.67770192027092, "z_loss": 43.83430242538452, "lr": 0.0023, "wd": 0.01, "grad_norm": 0.14906249940395355, "tokens_per_second": 1379155.378524915, "eta_seconds": 4135.668279886246, "data_wait_seconds": 1.8596649169921875e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16065067, "gate_a2b/L1_absmax": 0.69576406, "gate_a2b/L1_rms": 0.17090531, "gate_a2b/L3_absmean": 0.15231641, "gate_a2b/L3_absmax": 0.49942347, "gate_a2b/L3_rms": 0.15800509, "gate_a2b/L5_absmean": 0.22311781, "gate_a2b/L5_absmax": 0.73285687, "gate_a2b/L5_rms": 0.22925329, "gate_c2b/L1_absmean": 0.03649532, "gate_c2b/L1_absmax": 0.63152725, "gate_c2b/L1_rms": 0.07007364, "gate_c2b/L3_absmean": 0.16921811, "gate_c2b/L3_absmax": 0.38969067, "gate_c2b/L3_rms": 0.17284088, "gate_c2b/L5_absmean": 0.47956657, "gate_c2b/L5_absmax": 0.68434894, "gate_c2b/L5_rms": 0.48916027, "gate_b2a/L7_absmean": 0.05075555, "gate_b2a/L7_absmax": 0.33364391, "gate_b2a/L7_rms": 0.05667469, "gate_b2a/L11_absmean": 0.08302151, "gate_b2a/L11_absmax": 0.61298788, "gate_b2a/L11_rms": 0.09221405, "gate_b2a/L14_absmean": 0.21568567, "gate_b2a/L14_absmax": 0.83007842, "gate_b2a/L14_rms": 0.22523087, "gate_b2c/L3_absmean": 0.03485938, "gate_b2c/L3_absmax": 0.55308515, "gate_b2c/L3_rms": 0.04552576, "gate_b2c/L5_absmean": 0.0366294, "gate_b2c/L5_absmax": 0.52149487, "gate_b2c/L5_rms": 0.04564971, "gate_b2c/L6_absmean": 0.0533646, "gate_b2c/L6_absmax": 0.3459588, "gate_b2c/L6_rms": 0.05943368}, "streams": {"cos_a2b/round0": 0.353604, "cos_c2b/round0": 0.242476, "cos_b2a/round0": 0.154963, "cos_b2c/round0": 0.144132, "rms_a/round0": 7.653002, "rms_b/round0": 13.105969, "rms_c/round0": 4.973268, "cos_a2b/round1": 0.410996, "cos_c2b/round1": 0.297407, "cos_b2a/round1": 0.243676, "cos_b2c/round1": 0.103757, "rms_a/round1": 15.000478, "rms_b/round1": 30.46072, "rms_c/round1": 10.401121, "cos_a2b/round2": 0.525438, "cos_c2b/round2": 0.524003, "cos_b2a/round2": 0.097912, "cos_b2c/round2": 0.175453, "rms_a/round2": 42.466705, "rms_b/round2": 83.201019, "rms_c/round2": 20.0158}, "alpha_mean": 0.42021556624187895, "alpha_hist": [0.00687, 0.07211, 0.20383, 0.2173, 0.17773, 0.14714, 0.10909, 0.05846, 0.00746, 2e-05], "alpha_by_type": {"special": {"share": 0.00044, "mean": 0.56717, "std": 0.11502}, "whitespace": {"share": 0.02191, "mean": 0.52472, "std": 0.13221}, "digit": {"share": 0.03677, "mean": 0.43364, "std": 0.11107}, "word_start": {"share": 0.57518, "mean": 0.43741, "std": 0.16476}, "word_cont": {"share": 0.26163, "mean": 0.31406, "std": 0.11827}, "punct": {"share": 0.10406, "mean": 0.56469, "std": 0.16207}}, "alpha_p05": 0.18097, "alpha_p50": 0.40169, "alpha_p95": 0.71503}
{"step": 61610, "total_steps": 72479, "tokens": 32301383680, "loss": 4.244611859321594, "l_mix": 2.6297340393066406, "l_a": 2.7150458693504333, "l_c": 2.667879641056061, "z_loss": 43.30481815338135, "lr": 0.002299999951970452, "wd": 0.01, "grad_norm": 0.22864747047424316, "tokens_per_second": 1478347.881730731, "eta_seconds": 3854.631472349167, "data_wait_seconds": 1.6242265701293945e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 61620, "total_steps": 72479, "tokens": 32306626560, "loss": 4.1860977709293365, "l_mix": 2.5937699377536774, "l_a": 2.6762440502643585, "l_c": 2.6315152645111084, "z_loss": 43.66201114654541, "lr": 0.0022999941884295555, "wd": 0.01, "grad_norm": 0.11980541795492172, "tokens_per_second": 1445347.6110030343, "eta_seconds": 3939.0132509708405, "data_wait_seconds": 1.6421079635620117e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 61630, "total_steps": 72479, "tokens": 32311869440, "loss": 4.2811920046806335, "l_mix": 2.652874916791916, "l_a": 2.7374325692653656, "l_c": 2.690290629863739, "z_loss": 43.19528818130493, "lr": 0.002299978819034238, "wd": 0.01, "grad_norm": 0.16118352115154266, "tokens_per_second": 1321782.1904818865, "eta_seconds": 4303.281246304512, "data_wait_seconds": 2.6851892471313477e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 61640, "total_steps": 72479, "tokens": 32317112320, "loss": 3.063768744468689, "l_mix": 1.8868753015995026, "l_a": 1.9907503873109818, "l_c": 1.9322276711463928, "z_loss": 45.736802101135254, "lr": 0.0022999538439128784, "wd": 0.01, "grad_norm": 0.2741925120353699, "tokens_per_second": 1389770.032812509, "eta_seconds": 4088.991342329979, "data_wait_seconds": 2.6971101760864258e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 61650, "total_steps": 72479, "tokens": 32322355200, "loss": 3.1229890286922455, "l_mix": 1.9215989112854004, "l_a": 2.0351477563381195, "l_c": 1.9694857597351074, "z_loss": 46.54895305633545, "lr": 0.0022999192632740936, "wd": 0.01, "grad_norm": 0.1779690533876419, "tokens_per_second": 1470642.4389427349, "eta_seconds": 3860.5677366971972, "data_wait_seconds": 2.047419548034668e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 61660, "total_steps": 72479, "tokens": 32327598080, "loss": 3.0915558636188507, "l_mix": 1.9028251767158508, "l_a": 2.0160861760377884, "l_c": 1.9463491290807724, "z_loss": 46.377037525177, "lr": 0.002299875077406734, "wd": 0.01, "grad_norm": 0.16752777993679047, "tokens_per_second": 1421777.2826910112, "eta_seconds": 3989.564287638664, "data_wait_seconds": 2.1398067474365234e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 61670, "total_steps": 72479, "tokens": 32332840960, "loss": 3.097202032804489, "l_mix": 1.9061132222414017, "l_a": 2.016551196575165, "l_c": 1.9537445604801178, "z_loss": 44.42560958862305, "lr": 0.002299821286679883, "wd": 0.01, "grad_norm": 0.16328544914722443, "tokens_per_second": 1369097.971061846, "eta_seconds": 4139.242853164673, "data_wait_seconds": 3.394484519958496e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 61680, "total_steps": 72479, "tokens": 32338083840, "loss": 3.074556887149811, "l_mix": 1.891486570239067, "l_a": 2.005647838115692, "l_c": 1.937919706106186, "z_loss": 45.46964168548584, "lr": 0.0022997578915428516, "wd": 0.01, "grad_norm": 0.20993253588676453, "tokens_per_second": 1296081.3524698694, "eta_seconds": 4368.3879111528395, "data_wait_seconds": 1.749396324157715e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 61690, "total_steps": 72479, "tokens": 32343326720, "loss": 4.734231948852539, "l_mix": 2.9376331567764282, "l_a": 3.0120633244514465, "l_c": 2.9765990376472473, "z_loss": 41.14209604263306, "lr": 0.002299684892525177, "wd": 0.01, "grad_norm": 0.13807687163352966, "tokens_per_second": 1392156.056290877, "eta_seconds": 4063.1531260013585, "data_wait_seconds": 4.3004751205444336e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 61700, "total_steps": 72479, "tokens": 32348569600, "loss": 3.0992659628391266, "l_mix": 1.9065271764993668, "l_a": 2.0205083042383194, "l_c": 1.955287605524063, "z_loss": 46.50533628463745, "lr": 0.002299602290236617, "wd": 0.01, "grad_norm": 0.17976604402065277, "tokens_per_second": 1435751.9581921808, "eta_seconds": 3936.1258187770845, "data_wait_seconds": 1.755356788635254e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.16113445, "gate_a2b/L1_absmax": 0.69458824, "gate_a2b/L1_rms": 0.1714054, "gate_a2b/L3_absmean": 0.15260874, "gate_a2b/L3_absmax": 0.49929857, "gate_a2b/L3_rms": 0.15831208, "gate_a2b/L5_absmean": 0.22309045, "gate_a2b/L5_absmax": 0.73287714, "gate_a2b/L5_rms": 0.22920911, "gate_c2b/L1_absmean": 0.03600771, "gate_c2b/L1_absmax": 0.63132328, "gate_c2b/L1_rms": 0.06990649, "gate_c2b/L3_absmean": 0.16916521, "gate_c2b/L3_absmax": 0.38873857, "gate_c2b/L3_rms": 0.172739, "gate_c2b/L5_absmean": 0.4785327, "gate_c2b/L5_absmax": 0.67664033, "gate_c2b/L5_rms": 0.48805907, "gate_b2a/L7_absmean": 0.05120379, "gate_b2a/L7_absmax": 0.33645716, "gate_b2a/L7_rms": 0.05716437, "gate_b2a/L11_absmean": 0.08281235, "gate_b2a/L11_absmax": 0.61618787, "gate_b2a/L11_rms": 0.09208844, "gate_b2a/L14_absmean": 0.21510455, "gate_b2a/L14_absmax": 0.82904732, "gate_b2a/L14_rms": 0.22466624, "gate_b2c/L3_absmean": 0.03472303, "gate_b2c/L3_absmax": 0.553002, "gate_b2c/L3_rms": 0.04542121, "gate_b2c/L5_absmean": 0.03662388, "gate_b2c/L5_absmax": 0.52254301, "gate_b2c/L5_rms": 0.04569142, "gate_b2c/L6_absmean": 0.05326251, "gate_b2c/L6_absmax": 0.346017, "gate_b2c/L6_rms": 0.05934466}, "streams": {"cos_a2b/round0": 0.352619, "cos_c2b/round0": 0.210601, "cos_b2a/round0": 0.145366, "cos_b2c/round0": 0.123238, "rms_a/round0": 8.052324, "rms_b/round0": 13.488308, "rms_c/round0": 5.076562, "cos_a2b/round1": 0.404421, "cos_c2b/round1": 0.305699, "cos_b2a/round1": 0.238494, "cos_b2c/round1": 0.117763, "rms_a/round1": 16.114958, "rms_b/round1": 32.411331, "rms_c/round1": 11.49908, "cos_a2b/round2": 0.509575, "cos_c2b/round2": 0.513145, "cos_b2a/round2": 0.089918, "cos_b2c/round2": 0.169309, "rms_a/round2": 44.310875, "rms_b/round2": 84.986595, "rms_c/round2": 22.192387}, "alpha_mean": 0.39773218317125725, "alpha_hist": [0.04912, 0.18637, 0.18123, 0.14001, 0.11469, 0.10228, 0.09547, 0.08438, 0.04404, 0.00241], "alpha_by_type": {"special": {"share": 0.00105, "mean": 0.65212, "std": 0.11908}, "whitespace": {"share": 0.03854, "mean": 0.60138, "std": 0.16742}, "digit": {"share": 0.00897, "mean": 0.38674, "std": 0.18804}, "word_start": {"share": 0.56302, "mean": 0.40517, "std": 0.21643}, "word_cont": {"share": 0.26472, "mean": 0.25682, "std": 0.14045}, "punct": {"share": 0.12369, "mean": 0.60064, "std": 0.19638}}, "alpha_p05": 0.10195, "alpha_p50": 0.34511, "alpha_p95": 0.79341}
{"step": 61710, "total_steps": 72479, "tokens": 32353812480, "loss": 4.194601118564606, "l_mix": 2.5994958579540253, "l_a": 2.6826429069042206, "l_c": 2.6343744695186615, "z_loss": 42.93011713027954, "lr": 0.002299510085367145, "wd": 0.01, "grad_norm": 0.11573732644319534, "tokens_per_second": 1418825.188746919, "eta_seconds": 3979.389086675644, "data_wait_seconds": 1.8596649169921875e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 61720, "total_steps": 72479, "tokens": 32359055360, "loss": 4.215381681919098, "l_mix": 2.6112114787101746, "l_a": 2.701583296060562, "l_c": 2.6456505954265594, "z_loss": 43.569167613983154, "lr": 0.002299408278686945, "wd": 0.01, "grad_norm": 0.14207525551319122, "tokens_per_second": 1474323.513275107, "eta_seconds": 3826.035833525658, "data_wait_seconds": 1.5676021575927734e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 61730, "total_steps": 72479, "tokens": 32364298240, "loss": 4.265902757644653, "l_mix": 2.643503576517105, "l_a": 2.7294342517852783, "l_c": 2.6785627901554108, "z_loss": 42.69811487197876, "lr": 0.0022992968710464036, "wd": 0.01, "grad_norm": 0.1799657940864563, "tokens_per_second": 1384812.063060362, "eta_seconds": 4069.5570628881455, "data_wait_seconds": 3.9517879486083984e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 61740, "total_steps": 72479, "tokens": 32369541120, "loss": 4.293846547603607, "l_mix": 2.6612541377544403, "l_a": 2.744456857442856, "l_c": 2.6975176334381104, "z_loss": 42.77884340286255, "lr": 0.002299175863376104, "wd": 0.01, "grad_norm": 0.11997151374816895, "tokens_per_second": 1410880.0061182398, "eta_seconds": 3990.650379610062, "data_wait_seconds": 1.6361474990844727e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 61750, "total_steps": 72479, "tokens": 32374784000, "loss": 3.1460572481155396, "l_mix": 1.9369853138923645, "l_a": 2.0454962700605392, "l_c": 1.9847431927919388, "z_loss": 45.965155601501465, "lr": 0.0022990452566868194, "wd": 0.01, "grad_norm": 0.15723150968551636, "tokens_per_second": 1474497.699165553, "eta_seconds": 3814.9167375326156, "data_wait_seconds": 1.919269561767578e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 61760, "total_steps": 72479, "tokens": 32380026880, "loss": 3.0919729471206665, "l_mix": 1.9012978076934814, "l_a": 2.0145488530397415, "l_c": 1.9543682485818863, "z_loss": 44.89773178100586, "lr": 0.002298905052069501, "wd": 0.01, "grad_norm": 0.18732737004756927, "tokens_per_second": 1299757.768587167, "eta_seconds": 4323.761863803864, "data_wait_seconds": 3.24249267578125e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 61770, "total_steps": 72479, "tokens": 32385269760, "loss": 3.1081338226795197, "l_mix": 1.912627175450325, "l_a": 2.0233681350946426, "l_c": 1.9616540223360062, "z_loss": 44.01436710357666, "lr": 0.0022987552506952727, "wd": 0.01, "grad_norm": 0.1950579732656479, "tokens_per_second": 1395209.10661055, "eta_seconds": 4024.1997886896133, "data_wait_seconds": 1.990795135498047e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 61780, "total_steps": 72479, "tokens": 32390512640, "loss": 3.0448335111141205, "l_mix": 1.873986005783081, "l_a": 1.9840474128723145, "l_c": 1.9187774360179901, "z_loss": 46.66085243225098, "lr": 0.002298595853815419, "wd": 0.01, "grad_norm": 0.2086823433637619, "tokens_per_second": 1426004.9422331443, "eta_seconds": 3933.6170204401014, "data_wait_seconds": 1.7344951629638672e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 61790, "total_steps": 72479, "tokens": 32395755520, "loss": 3.0885065495967865, "l_mix": 1.8999542146921158, "l_a": 2.0136756747961044, "l_c": 1.9481652826070786, "z_loss": 47.020334243774414, "lr": 0.002298426862761376, "wd": 0.01, "grad_norm": 0.15203729271888733, "tokens_per_second": 1441763.1255782726, "eta_seconds": 3886.9869346618652, "data_wait_seconds": 4.869699478149414e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 61800, "total_steps": 72479, "tokens": 32400998400, "loss": 3.1174356639385223, "l_mix": 1.9192463010549545, "l_a": 2.0283998996019363, "l_c": 1.965564340353012, "z_loss": 43.64577627182007, "lr": 0.002298248278944719, "wd": 0.01, "grad_norm": 0.18859292566776276, "tokens_per_second": 1356277.884616203, "eta_seconds": 4128.115348267555, "data_wait_seconds": 1.6421079635620117e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.16070147, "gate_a2b/L1_absmax": 0.69648188, "gate_a2b/L1_rms": 0.17090435, "gate_a2b/L3_absmean": 0.15210871, "gate_a2b/L3_absmax": 0.50017112, "gate_a2b/L3_rms": 0.1578276, "gate_a2b/L5_absmean": 0.22309998, "gate_a2b/L5_absmax": 0.73428905, "gate_a2b/L5_rms": 0.22922187, "gate_c2b/L1_absmean": 0.03653892, "gate_c2b/L1_absmax": 0.63049418, "gate_c2b/L1_rms": 0.07026479, "gate_c2b/L3_absmean": 0.16905533, "gate_c2b/L3_absmax": 0.39441681, "gate_c2b/L3_rms": 0.17268868, "gate_c2b/L5_absmean": 0.47822103, "gate_c2b/L5_absmax": 0.6697461, "gate_c2b/L5_rms": 0.4876914, "gate_b2a/L7_absmean": 0.05111714, "gate_b2a/L7_absmax": 0.33575976, "gate_b2a/L7_rms": 0.05699074, "gate_b2a/L11_absmean": 0.08314948, "gate_b2a/L11_absmax": 0.61448359, "gate_b2a/L11_rms": 0.09239089, "gate_b2a/L14_absmean": 0.2159223, "gate_b2a/L14_absmax": 0.8301242, "gate_b2a/L14_rms": 0.22555466, "gate_b2c/L3_absmean": 0.03472966, "gate_b2c/L3_absmax": 0.56068391, "gate_b2c/L3_rms": 0.04566897, "gate_b2c/L5_absmean": 0.0367816, "gate_b2c/L5_absmax": 0.53162175, "gate_b2c/L5_rms": 0.04608373, "gate_b2c/L6_absmean": 0.05348573, "gate_b2c/L6_absmax": 0.34642118, "gate_b2c/L6_rms": 0.05952128}, "streams": {"cos_a2b/round0": 0.354414, "cos_c2b/round0": 0.208829, "cos_b2a/round0": 0.147832, "cos_b2c/round0": 0.122669, "rms_a/round0": 8.120697, "rms_b/round0": 13.415805, "rms_c/round0": 5.121611, "cos_a2b/round1": 0.408823, "cos_c2b/round1": 0.311532, "cos_b2a/round1": 0.24274, "cos_b2c/round1": 0.11956, "rms_a/round1": 16.000479, "rms_b/round1": 32.255577, "rms_c/round1": 11.405481, "cos_a2b/round2": 0.518789, "cos_c2b/round2": 0.512521, "cos_b2a/round2": 0.089987, "cos_b2c/round2": 0.17555, "rms_a/round2": 44.230923, "rms_b/round2": 85.179367, "rms_c/round2": 22.016953}, "alpha_mean": 0.4181035111713669, "alpha_hist": [0.02448, 0.14528, 0.2003, 0.1582, 0.12311, 0.1111, 0.10291, 0.08685, 0.04393, 0.00385], "alpha_by_type": {"special": {"share": 0.0011, "mean": 0.58957, "std": 0.10675}, "whitespace": {"share": 0.03174, "mean": 0.64102, "std": 0.1675}, "digit": {"share": 0.00894, "mean": 0.42784, "std": 0.1713}, "word_start": {"share": 0.57808, "mean": 0.42446, "std": 0.20709}, "word_cont": {"share": 0.26047, "mean": 0.28644, "std": 0.13864}, "punct": {"share": 0.11967, "mean": 0.61253, "std": 0.1849}}, "alpha_p05": 0.12897, "alpha_p50": 0.36841, "alpha_p95": 0.80409}
{"step": 61810, "total_steps": 72479, "tokens": 32406241280, "loss": 4.258847713470459, "l_mix": 2.6396058797836304, "l_a": 2.724317640066147, "l_c": 2.6731549501419067, "z_loss": 43.81683397293091, "lr": 0.0022980601038571505, "wd": 0.01, "grad_norm": 0.2312649041414261, "tokens_per_second": 1481424.890364265, "eta_seconds": 3775.8435870647427, "data_wait_seconds": 2.002716064453125e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 61820, "total_steps": 72479, "tokens": 32411484160, "loss": 4.325721740722656, "l_mix": 2.681040972471237, "l_a": 2.7620403170585632, "l_c": 2.7202287912368774, "z_loss": 42.31495189666748, "lr": 0.0022978623390704908, "wd": 0.01, "grad_norm": 0.2056545466184616, "tokens_per_second": 1440039.7781746632, "eta_seconds": 3880.716266798973, "data_wait_seconds": 2.008676528930664e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 61830, "total_steps": 72479, "tokens": 32416727040, "loss": 4.225089609622955, "l_mix": 2.61879363656044, "l_a": 2.7015380859375, "l_c": 2.652781456708908, "z_loss": 42.70842933654785, "lr": 0.0022976549862366593, "wd": 0.01, "grad_norm": 0.12458913028240204, "tokens_per_second": 1401339.5718637442, "eta_seconds": 3984.1470433712, "data_wait_seconds": 3.018975257873535e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 61840, "total_steps": 72479, "tokens": 32421969920, "loss": 4.3389429450035095, "l_mix": 2.689384013414383, "l_a": 2.774044394493103, "l_c": 2.7244853079319, "z_loss": 42.78771638870239, "lr": 0.002297438047087667, "wd": 0.01, "grad_norm": 0.1211095079779625, "tokens_per_second": 1449952.3515697035, "eta_seconds": 3846.9540229797362, "data_wait_seconds": 3.275275230407715e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 61850, "total_steps": 72479, "tokens": 32427212800, "loss": 4.213379859924316, "l_mix": 2.6101434230804443, "l_a": 2.696974217891693, "l_c": 2.6471472084522247, "z_loss": 43.95076274871826, "lr": 0.0022972115234355973, "wd": 0.01, "grad_norm": 0.12298573553562164, "tokens_per_second": 1408856.5046787125, "eta_seconds": 3955.4469411849973, "data_wait_seconds": 2.053380012512207e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 61860, "total_steps": 72479, "tokens": 32432455680, "loss": 3.0931559801101685, "l_mix": 1.9026905298233032, "l_a": 2.01552477478981, "l_c": 1.952693223953247, "z_loss": 46.33334827423096, "lr": 0.002296975417172593, "wd": 0.01, "grad_norm": 0.18412752449512482, "tokens_per_second": 1395899.6503825702, "eta_seconds": 3988.4058073043825, "data_wait_seconds": 3.4362077713012695e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 61870, "total_steps": 72479, "tokens": 32437698560, "loss": 3.080673426389694, "l_mix": 1.897849202156067, "l_a": 2.0034569203853607, "l_c": 1.9392903447151184, "z_loss": 44.93269491195679, "lr": 0.0022967297302708407, "wd": 0.01, "grad_norm": 0.26772359013557434, "tokens_per_second": 1332221.7197897837, "eta_seconds": 4175.109375095368, "data_wait_seconds": 2.35140323638916e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 61880, "total_steps": 72479, "tokens": 32442941440, "loss": 3.0914363265037537, "l_mix": 1.901122584939003, "l_a": 2.0142880976200104, "l_c": 1.9534242004156113, "z_loss": 46.05888080596924, "lr": 0.002296474464782552, "wd": 0.01, "grad_norm": 0.2401609569787979, "tokens_per_second": 1424314.0569255843, "eta_seconds": 3901.476984643936, "data_wait_seconds": 2.619624137878418e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 61890, "total_steps": 72479, "tokens": 32448184320, "loss": 3.107360690832138, "l_mix": 1.9125023484230042, "l_a": 2.0254391729831696, "l_c": 1.957421824336052, "z_loss": 45.73744201660156, "lr": 0.0022962096228399493, "wd": 0.01, "grad_norm": 0.16503918170928955, "tokens_per_second": 1466638.825202044, "eta_seconds": 3785.3120595216756, "data_wait_seconds": 2.0563602447509766e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 61900, "total_steps": 72479, "tokens": 32453427200, "loss": 3.1162248253822327, "l_mix": 1.9183364510536194, "l_a": 2.0293363481760025, "l_c": 1.9636248797178268, "z_loss": 46.46432685852051, "lr": 0.0022959352066552474, "wd": 0.01, "grad_norm": 0.1875777542591095, "tokens_per_second": 1448128.0331521009, "eta_seconds": 3830.0776071071623, "data_wait_seconds": 1.7821788787841797e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16027533, "gate_a2b/L1_absmax": 0.694767, "gate_a2b/L1_rms": 0.17057709, "gate_a2b/L3_absmean": 0.15187289, "gate_a2b/L3_absmax": 0.49852464, "gate_a2b/L3_rms": 0.15755297, "gate_a2b/L5_absmean": 0.22217953, "gate_a2b/L5_absmax": 0.73267847, "gate_a2b/L5_rms": 0.22822124, "gate_c2b/L1_absmean": 0.03633496, "gate_c2b/L1_absmax": 0.6297316, "gate_c2b/L1_rms": 0.06981561, "gate_c2b/L3_absmean": 0.16922955, "gate_c2b/L3_absmax": 0.39163181, "gate_c2b/L3_rms": 0.17284915, "gate_c2b/L5_absmean": 0.47793636, "gate_c2b/L5_absmax": 0.67292094, "gate_c2b/L5_rms": 0.48739341, "gate_b2a/L7_absmean": 0.05107294, "gate_b2a/L7_absmax": 0.33576089, "gate_b2a/L7_rms": 0.05699536, "gate_b2a/L11_absmean": 0.08300184, "gate_b2a/L11_absmax": 0.61443502, "gate_b2a/L11_rms": 0.09224784, "gate_b2a/L14_absmean": 0.21584278, "gate_b2a/L14_absmax": 0.82920194, "gate_b2a/L14_rms": 0.22540364, "gate_b2c/L3_absmean": 0.03472938, "gate_b2c/L3_absmax": 0.55819398, "gate_b2c/L3_rms": 0.04559671, "gate_b2c/L5_absmean": 0.03661391, "gate_b2c/L5_absmax": 0.52208209, "gate_b2c/L5_rms": 0.04567561, "gate_b2c/L6_absmean": 0.05322718, "gate_b2c/L6_absmax": 0.34478179, "gate_b2c/L6_rms": 0.05924807}, "streams": {"cos_a2b/round0": 0.352557, "cos_c2b/round0": 0.211277, "cos_b2a/round0": 0.146651, "cos_b2c/round0": 0.124103, "rms_a/round0": 8.095611, "rms_b/round0": 13.444968, "rms_c/round0": 5.069757, "cos_a2b/round1": 0.406611, "cos_c2b/round1": 0.304213, "cos_b2a/round1": 0.237329, "cos_b2c/round1": 0.117564, "rms_a/round1": 16.228552, "rms_b/round1": 32.325447, "rms_c/round1": 11.400568, "cos_a2b/round2": 0.516343, "cos_c2b/round2": 0.511959, "cos_b2a/round2": 0.08852, "cos_b2c/round2": 0.1725, "rms_a/round2": 44.545219, "rms_b/round2": 85.125832, "rms_c/round2": 22.139244}, "alpha_mean": 0.43146645854623245, "alpha_hist": [0.02632, 0.1369, 0.18317, 0.15446, 0.12233, 0.11723, 0.10547, 0.09869, 0.05161, 0.00381], "alpha_by_type": {"special": {"share": 0.00114, "mean": 0.64943, "std": 0.12273}, "whitespace": {"share": 0.03934, "mean": 0.63068, "std": 0.15929}, "digit": {"share": 0.00967, "mean": 0.3933, "std": 0.17821}, "word_start": {"share": 0.56522, "mean": 0.4361, "std": 0.21118}, "word_cont": {"share": 0.25526, "mean": 0.29053, "std": 0.14314}, "punct": {"share": 0.12936, "mean": 0.62966, "std": 0.17986}}, "alpha_p05": 0.11241, "alpha_p50": 0.40875, "alpha_p95": 0.82276}
{"step": 61910, "total_steps": 72479, "tokens": 32458670080, "loss": 3.1268047988414764, "l_mix": 1.925161987543106, "l_a": 2.0356418192386627, "l_c": 1.969834104180336, "z_loss": 45.865283489227295, "lr": 0.0022956512185206327, "wd": 0.01, "grad_norm": 0.16781313717365265, "tokens_per_second": 1383935.4892449714, "eta_seconds": 4003.943764042854, "data_wait_seconds": 1.71661376953125e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 61920, "total_steps": 72479, "tokens": 32463912960, "loss": 4.233785688877106, "l_mix": 2.6234821677207947, "l_a": 2.709343373775482, "l_c": 2.6583344638347626, "z_loss": 44.0256085395813, "lr": 0.0022953576608082463, "wd": 0.01, "grad_norm": 0.14318765699863434, "tokens_per_second": 1449328.801478392, "eta_seconds": 3819.669481730461, "data_wait_seconds": 1.8805265426635742e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 61930, "total_steps": 72479, "tokens": 32469155840, "loss": 4.2758177518844604, "l_mix": 2.649810492992401, "l_a": 2.735955834388733, "l_c": 2.684068351984024, "z_loss": 42.468735218048096, "lr": 0.0022950545359701644, "wd": 0.01, "grad_norm": 0.1466970443725586, "tokens_per_second": 1410836.7383082714, "eta_seconds": 3920.1659283637996, "data_wait_seconds": 2.709031105041504e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 61940, "total_steps": 72479, "tokens": 32474398720, "loss": 4.390662133693695, "l_mix": 2.721574902534485, "l_a": 2.8063502609729767, "l_c": 2.757273405790329, "z_loss": 42.46369981765747, "lr": 0.0022947418465383757, "wd": 0.01, "grad_norm": 0.15221190452575684, "tokens_per_second": 1474465.1721143299, "eta_seconds": 3747.4409952163696, "data_wait_seconds": 2.5928020477294922e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 61950, "total_steps": 72479, "tokens": 32479641600, "loss": 4.35760635137558, "l_mix": 2.700809419155121, "l_a": 2.788771837949753, "l_c": 2.7338844537734985, "z_loss": 42.16581153869629, "lr": 0.0022944195951247626, "wd": 0.01, "grad_norm": 0.174491748213768, "tokens_per_second": 1327392.1375827703, "eta_seconds": 4158.702011036873, "data_wait_seconds": 1.8894672393798828e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 61960, "total_steps": 72479, "tokens": 32484884480, "loss": 3.0969503223896027, "l_mix": 1.906778872013092, "l_a": 2.0151117146015167, "l_c": 1.9521264284849167, "z_loss": 45.81454086303711, "lr": 0.0022940877844210786, "wd": 0.01, "grad_norm": 0.18656300008296967, "tokens_per_second": 1439229.9942778286, "eta_seconds": 3831.9000395536423, "data_wait_seconds": 2.1576881408691406e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 61970, "total_steps": 72479, "tokens": 32490127360, "loss": 4.6823490858078, "l_mix": 2.905530631542206, "l_a": 2.9812545478343964, "l_c": 2.9414733052253723, "z_loss": 40.31059551239014, "lr": 0.0022937464171989236, "wd": 0.01, "grad_norm": 0.1701645404100418, "tokens_per_second": 1392910.8085707892, "eta_seconds": 3955.5602254629134, "data_wait_seconds": 3.141164779663086e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 61980, "total_steps": 72479, "tokens": 32495370240, "loss": 3.0809032917022705, "l_mix": 1.8960179090499878, "l_a": 2.0081862658262253, "l_c": 1.9414317309856415, "z_loss": 45.869933128356934, "lr": 0.002293395496309725, "wd": 0.01, "grad_norm": 0.15594720840454102, "tokens_per_second": 1393519.5092581885, "eta_seconds": 3950.070074677467, "data_wait_seconds": 3.0219554901123047e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 61990, "total_steps": 72479, "tokens": 32500613120, "loss": 3.106460601091385, "l_mix": 1.9110691994428635, "l_a": 2.0235906541347504, "l_c": 1.9610471427440643, "z_loss": 46.771525859832764, "lr": 0.0022930350246847106, "wd": 0.01, "grad_norm": 0.1509372591972351, "tokens_per_second": 1419119.7456272505, "eta_seconds": 3875.118254780769, "data_wait_seconds": 2.8431415557861328e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 62000, "total_steps": 72479, "tokens": 32505856000, "loss": 3.0893255174160004, "l_mix": 1.9023923128843307, "l_a": 2.009295552968979, "l_c": 1.9471482038497925, "z_loss": 45.60626411437988, "lr": 0.0022926650053348846, "wd": 0.01, "grad_norm": 0.18651065230369568, "tokens_per_second": 1406766.3501925792, "eta_seconds": 3905.4203644037243, "data_wait_seconds": 2.7179718017578125e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16023161, "gate_a2b/L1_absmax": 0.69561237, "gate_a2b/L1_rms": 0.17065285, "gate_a2b/L3_absmean": 0.15239166, "gate_a2b/L3_absmax": 0.49375251, "gate_a2b/L3_rms": 0.15806717, "gate_a2b/L5_absmean": 0.22216038, "gate_a2b/L5_absmax": 0.73022598, "gate_a2b/L5_rms": 0.22821109, "gate_c2b/L1_absmean": 0.03643673, "gate_c2b/L1_absmax": 0.63024628, "gate_c2b/L1_rms": 0.06984742, "gate_c2b/L3_absmean": 0.16927004, "gate_c2b/L3_absmax": 0.3888751, "gate_c2b/L3_rms": 0.17288569, "gate_c2b/L5_absmean": 0.47760698, "gate_c2b/L5_absmax": 0.68403447, "gate_c2b/L5_rms": 0.48702523, "gate_b2a/L7_absmean": 0.05102579, "gate_b2a/L7_absmax": 0.34127581, "gate_b2a/L7_rms": 0.05697805, "gate_b2a/L11_absmean": 0.08319278, "gate_b2a/L11_absmax": 0.62061405, "gate_b2a/L11_rms": 0.09252163, "gate_b2a/L14_absmean": 0.21558496, "gate_b2a/L14_absmax": 0.8307364, "gate_b2a/L14_rms": 0.22513726, "gate_b2c/L3_absmean": 0.03454988, "gate_b2c/L3_absmax": 0.56288576, "gate_b2c/L3_rms": 0.0454969, "gate_b2c/L5_absmean": 0.03667329, "gate_b2c/L5_absmax": 0.53388053, "gate_b2c/L5_rms": 0.04604138, "gate_b2c/L6_absmean": 0.05308313, "gate_b2c/L6_absmax": 0.34371573, "gate_b2c/L6_rms": 0.05912114}, "streams": {"cos_a2b/round0": 0.354797, "cos_c2b/round0": 0.212702, "cos_b2a/round0": 0.149091, "cos_b2c/round0": 0.126829, "rms_a/round0": 8.079858, "rms_b/round0": 13.314111, "rms_c/round0": 5.089863, "cos_a2b/round1": 0.411495, "cos_c2b/round1": 0.306228, "cos_b2a/round1": 0.242308, "cos_b2c/round1": 0.119424, "rms_a/round1": 16.017729, "rms_b/round1": 32.12347, "rms_c/round1": 11.240076, "cos_a2b/round2": 0.517406, "cos_c2b/round2": 0.509855, "cos_b2a/round2": 0.090419, "cos_b2c/round2": 0.174626, "rms_a/round2": 43.961487, "rms_b/round2": 84.435448, "rms_c/round2": 21.869884}, "alpha_mean": 0.41353632463926715, "alpha_hist": [0.02815, 0.15285, 0.19276, 0.15393, 0.12764, 0.11418, 0.10321, 0.08591, 0.03966, 0.00171], "alpha_by_type": {"special": {"share": 0.00108, "mean": 0.6285, "std": 0.11632}, "whitespace": {"share": 0.039, "mean": 0.59357, "std": 0.16003}, "digit": {"share": 0.01164, "mean": 0.40133, "std": 0.16665}, "word_start": {"share": 0.56473, "mean": 0.41486, "std": 0.20786}, "word_cont": {"share": 0.25909, "mean": 0.2872, "std": 0.13831}, "punct": {"share": 0.12445, "mean": 0.61337, "std": 0.18033}}, "alpha_p05": 0.12831, "alpha_p50": 0.3803, "alpha_p95": 0.79787}
{"step": 62010, "total_steps": 72479, "tokens": 32511098880, "loss": 3.038627803325653, "l_mix": 1.8699008077383041, "l_a": 1.9820885956287384, "l_c": 1.9136677086353302, "z_loss": 45.66957187652588, "lr": 0.0022922854413510037, "wd": 0.01, "grad_norm": 0.15323981642723083, "tokens_per_second": 1421134.2829425815, "eta_seconds": 3862.246613764763, "data_wait_seconds": 2.9146671295166016e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 62020, "total_steps": 72479, "tokens": 32516341760, "loss": 3.065134346485138, "l_mix": 1.8863808810710907, "l_a": 1.9943330585956573, "l_c": 1.9348449409008026, "z_loss": 46.45792245864868, "lr": 0.002291896335903551, "wd": 0.01, "grad_norm": 0.1508716344833374, "tokens_per_second": 1403394.9900873115, "eta_seconds": 3907.3306023836135, "data_wait_seconds": 2.110004425048828e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 62030, "total_steps": 72479, "tokens": 32521584640, "loss": 4.2213481068611145, "l_mix": 2.6163282692432404, "l_a": 2.7003842294216156, "l_c": 2.6496817767620087, "z_loss": 43.01982355117798, "lr": 0.0022914976922427076, "wd": 0.01, "grad_norm": 0.10684995353221893, "tokens_per_second": 1460195.165770572, "eta_seconds": 3751.7486979961395, "data_wait_seconds": 2.2590160369873047e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 62040, "total_steps": 72479, "tokens": 32526827520, "loss": 4.307302474975586, "l_mix": 2.66912978887558, "l_a": 2.7546569406986237, "l_c": 2.705918252468109, "z_loss": 42.84375238418579, "lr": 0.002291089513698328, "wd": 0.01, "grad_norm": 0.14433521032333374, "tokens_per_second": 1397196.141995307, "eta_seconds": 3917.1611397266392, "data_wait_seconds": 2.2679567337036133e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 62050, "total_steps": 72479, "tokens": 32532070400, "loss": 4.38178825378418, "l_mix": 2.7169012427330017, "l_a": 2.7995246052742004, "l_c": 2.750098556280136, "z_loss": 42.18406057357788, "lr": 0.0022906718036799106, "wd": 0.01, "grad_norm": 0.12749508023262024, "tokens_per_second": 1235612.5595749717, "eta_seconds": 4425.173173928261, "data_wait_seconds": 3.87728214263916e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 62060, "total_steps": 72479, "tokens": 32537313280, "loss": 4.256382524967194, "l_mix": 2.637220859527588, "l_a": 2.727360427379608, "l_c": 2.6698446571826935, "z_loss": 43.1518988609314, "lr": 0.00229024456567657, "wd": 0.01, "grad_norm": 0.1595308482646942, "tokens_per_second": 1456410.476608103, "eta_seconds": 3750.698556303978, "data_wait_seconds": 1.7434358596801758e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 62070, "total_steps": 72479, "tokens": 32542556160, "loss": 3.1133182048797607, "l_mix": 1.9150242805480957, "l_a": 2.030873864889145, "l_c": 1.963439255952835, "z_loss": 45.83804368972778, "lr": 0.0022898078032570087, "wd": 0.01, "grad_norm": 0.20442694425582886, "tokens_per_second": 1366556.6880136845, "eta_seconds": 3993.4777970552445, "data_wait_seconds": 1.8268823623657227e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 62080, "total_steps": 72479, "tokens": 32547799040, "loss": 3.0912828743457794, "l_mix": 1.90255868434906, "l_a": 2.0126127898693085, "l_c": 1.9498010873794556, "z_loss": 44.85451412200928, "lr": 0.002289361520069484, "wd": 0.01, "grad_norm": 0.2027754783630371, "tokens_per_second": 1387795.8157748622, "eta_seconds": 3928.58290104866, "data_wait_seconds": 2.0265579223632812e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 62090, "total_steps": 72479, "tokens": 32553041920, "loss": 3.099724233150482, "l_mix": 1.9070929884910583, "l_a": 2.021188050508499, "l_c": 1.954249083995819, "z_loss": 46.85763931274414, "lr": 0.0022889057198417825, "wd": 0.01, "grad_norm": 0.18198081851005554, "tokens_per_second": 1461066.1906826023, "eta_seconds": 3727.981707286835, "data_wait_seconds": 2.2798776626586914e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 62100, "total_steps": 72479, "tokens": 32558284800, "loss": 4.6059863567352295, "l_mix": 2.8577209413051605, "l_a": 2.9318670630455017, "l_c": 2.8956838250160217, "z_loss": 40.993581771850586, "lr": 0.0022884404063811844, "wd": 0.01, "grad_norm": 0.19380725920200348, "tokens_per_second": 1370592.3941540208, "eta_seconds": 3970.243213963509, "data_wait_seconds": 1.7017126083374023e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.1603056, "gate_a2b/L1_absmax": 0.69275266, "gate_a2b/L1_rms": 0.17055401, "gate_a2b/L3_absmean": 0.15200911, "gate_a2b/L3_absmax": 0.49944338, "gate_a2b/L3_rms": 0.15762983, "gate_a2b/L5_absmean": 0.2219115, "gate_a2b/L5_absmax": 0.73171806, "gate_a2b/L5_rms": 0.22793846, "gate_c2b/L1_absmean": 0.03625461, "gate_c2b/L1_absmax": 0.62880492, "gate_c2b/L1_rms": 0.06995093, "gate_c2b/L3_absmean": 0.1689903, "gate_c2b/L3_absmax": 0.39217076, "gate_c2b/L3_rms": 0.17260246, "gate_c2b/L5_absmean": 0.47749844, "gate_c2b/L5_absmax": 0.67423016, "gate_c2b/L5_rms": 0.48688874, "gate_b2a/L7_absmean": 0.05065759, "gate_b2a/L7_absmax": 0.33970991, "gate_b2a/L7_rms": 0.0567309, "gate_b2a/L11_absmean": 0.08249341, "gate_b2a/L11_absmax": 0.61692262, "gate_b2a/L11_rms": 0.09183521, "gate_b2a/L14_absmean": 0.21549851, "gate_b2a/L14_absmax": 0.82874799, "gate_b2a/L14_rms": 0.22505131, "gate_b2c/L3_absmean": 0.03470966, "gate_b2c/L3_absmax": 0.55947042, "gate_b2c/L3_rms": 0.04558641, "gate_b2c/L5_absmean": 0.03665583, "gate_b2c/L5_absmax": 0.52486289, "gate_b2c/L5_rms": 0.04576358, "gate_b2c/L6_absmean": 0.05321169, "gate_b2c/L6_absmax": 0.34271604, "gate_b2c/L6_rms": 0.05917454}, "streams": {"cos_a2b/round0": 0.355723, "cos_c2b/round0": 0.249966, "cos_b2a/round0": 0.171184, "cos_b2c/round0": 0.147861, "rms_a/round0": 7.692888, "rms_b/round0": 13.001073, "rms_c/round0": 4.914931, "cos_a2b/round1": 0.413807, "cos_c2b/round1": 0.311751, "cos_b2a/round1": 0.25436, "cos_b2c/round1": 0.115008, "rms_a/round1": 14.993044, "rms_b/round1": 30.037886, "rms_c/round1": 10.582271, "cos_a2b/round2": 0.543929, "cos_c2b/round2": 0.522051, "cos_b2a/round2": 0.105885, "cos_b2c/round2": 0.188452, "rms_a/round2": 43.026718, "rms_b/round2": 82.427849, "rms_c/round2": 20.668516}, "alpha_mean": 0.44742358986569286, "alpha_hist": [0.00627, 0.05299, 0.15814, 0.18782, 0.21489, 0.1805, 0.12694, 0.06516, 0.00728, 2e-05], "alpha_by_type": {"special": {"share": 0.00058, "mean": 0.5475, "std": 0.17431}, "whitespace": {"share": 0.06964, "mean": 0.51347, "std": 0.09833}, "digit": {"share": 0.01663, "mean": 0.42092, "std": 0.14847}, "word_start": {"share": 0.58224, "mean": 0.45876, "std": 0.15931}, "word_cont": {"share": 0.22482, "mean": 0.33921, "std": 0.1261}, "punct": {"share": 0.10608, "mean": 0.57479, "std": 0.16104}}, "alpha_p05": 0.18596, "alpha_p50": 0.43374, "alpha_p95": 0.7076}
{"step": 62110, "total_steps": 72479, "tokens": 32563527680, "loss": 3.0924909114837646, "l_mix": 1.9043782651424408, "l_a": 2.012244552373886, "l_c": 1.9481309205293655, "z_loss": 46.40932083129883, "lr": 0.002287965583574433, "wd": 0.01, "grad_norm": 0.23101545870304108, "tokens_per_second": 1417436.916700582, "eta_seconds": 3835.3327812671664, "data_wait_seconds": 3.147125244140625e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 62120, "total_steps": 72479, "tokens": 32568770560, "loss": 3.0686398446559906, "l_mix": 1.8880086094141006, "l_a": 1.9993484616279602, "l_c": 1.9360890537500381, "z_loss": 46.37906312942505, "lr": 0.002287481255387705, "wd": 0.01, "grad_norm": 0.18711450695991516, "tokens_per_second": 1419847.459731221, "eta_seconds": 3825.128787446022, "data_wait_seconds": 2.1457672119140625e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 62130, "total_steps": 72479, "tokens": 32574013440, "loss": 3.123392939567566, "l_mix": 1.9223239868879318, "l_a": 2.034188434481621, "l_c": 1.9693745374679565, "z_loss": 45.7574896812439, "lr": 0.0022869874258665737, "wd": 0.01, "grad_norm": 0.1492597758769989, "tokens_per_second": 1467602.9609040446, "eta_seconds": 3697.08746612072, "data_wait_seconds": 1.800060272216797e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 62140, "total_steps": 72479, "tokens": 32579256320, "loss": 3.1126836836338043, "l_mix": 1.9145475924015045, "l_a": 2.031911939382553, "l_c": 1.9618747979402542, "z_loss": 46.63217306137085, "lr": 0.0022864840991359764, "wd": 0.01, "grad_norm": 0.1513780653476715, "tokens_per_second": 1379434.1253162816, "eta_seconds": 3929.592238235474, "data_wait_seconds": 1.856684684753418e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 62150, "total_steps": 72479, "tokens": 32584499200, "loss": 4.304328799247742, "l_mix": 2.667280375957489, "l_a": 2.7539563179016113, "l_c": 2.702871650457382, "z_loss": 43.73532962799072, "lr": 0.0022859712794001813, "wd": 0.01, "grad_norm": 0.19022811949253082, "tokens_per_second": 1455761.7005870782, "eta_seconds": 3719.9568788051606, "data_wait_seconds": 2.059340476989746e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 62160, "total_steps": 72479, "tokens": 32589742080, "loss": 4.2140777707099915, "l_mix": 2.6117960810661316, "l_a": 2.6942111253738403, "l_c": 2.646727591753006, "z_loss": 44.39113473892212, "lr": 0.00228544897094275, "wd": 0.01, "grad_norm": 0.14838328957557678, "tokens_per_second": 1427325.4680150065, "eta_seconds": 3790.3953886032104, "data_wait_seconds": 3.2961368560791016e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 62170, "total_steps": 72479, "tokens": 32594984960, "loss": 4.337387323379517, "l_mix": 2.6881443858146667, "l_a": 2.77238792181015, "l_c": 2.7250886857509613, "z_loss": 42.1747465133667, "lr": 0.0022849171781265045, "wd": 0.01, "grad_norm": 0.15890708565711975, "tokens_per_second": 1409181.8831187307, "eta_seconds": 3835.477206134796, "data_wait_seconds": 2.327561378479004e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 62180, "total_steps": 72479, "tokens": 32600227840, "loss": 3.1111219823360443, "l_mix": 1.9148693233728409, "l_a": 2.0235954970121384, "l_c": 1.9639133214950562, "z_loss": 46.127933502197266, "lr": 0.0022843759053934876, "wd": 0.01, "grad_norm": 0.17155878245830536, "tokens_per_second": 1408575.4856856551, "eta_seconds": 3833.40627951622, "data_wait_seconds": 3.7789344787597656e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 62190, "total_steps": 72479, "tokens": 32605470720, "loss": 3.111576557159424, "l_mix": 1.914935901761055, "l_a": 2.027490884065628, "l_c": 1.9613110721111298, "z_loss": 46.05233860015869, "lr": 0.002283825157264929, "wd": 0.01, "grad_norm": 0.18963414430618286, "tokens_per_second": 1378586.5553260304, "eta_seconds": 3912.992775940895, "data_wait_seconds": 2.6345252990722656e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 62200, "total_steps": 72479, "tokens": 32610713600, "loss": 3.1338628828525543, "l_mix": 1.9288534671068192, "l_a": 2.0412829369306564, "l_c": 1.9754151552915573, "z_loss": 46.47531509399414, "lr": 0.0022832649383412048, "wd": 0.01, "grad_norm": 0.19776023924350739, "tokens_per_second": 1335486.4796948687, "eta_seconds": 4035.350738430023, "data_wait_seconds": 1.749396324157715e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.16077273, "gate_a2b/L1_absmax": 0.6923641, "gate_a2b/L1_rms": 0.17088468, "gate_a2b/L3_absmean": 0.15208657, "gate_a2b/L3_absmax": 0.49886873, "gate_a2b/L3_rms": 0.15772375, "gate_a2b/L5_absmean": 0.22198272, "gate_a2b/L5_absmax": 0.732126, "gate_a2b/L5_rms": 0.22803494, "gate_c2b/L1_absmean": 0.03676511, "gate_c2b/L1_absmax": 0.62828845, "gate_c2b/L1_rms": 0.0700919, "gate_c2b/L3_absmean": 0.16892453, "gate_c2b/L3_absmax": 0.39113861, "gate_c2b/L3_rms": 0.17256108, "gate_c2b/L5_absmean": 0.47673854, "gate_c2b/L5_absmax": 0.67160273, "gate_c2b/L5_rms": 0.48615494, "gate_b2a/L7_absmean": 0.05086279, "gate_b2a/L7_absmax": 0.33781064, "gate_b2a/L7_rms": 0.05688982, "gate_b2a/L11_absmean": 0.08268166, "gate_b2a/L11_absmax": 0.61704952, "gate_b2a/L11_rms": 0.09198147, "gate_b2a/L14_absmean": 0.21578227, "gate_b2a/L14_absmax": 0.82670963, "gate_b2a/L14_rms": 0.22527984, "gate_b2c/L3_absmean": 0.03491312, "gate_b2c/L3_absmax": 0.55877912, "gate_b2c/L3_rms": 0.04573254, "gate_b2c/L5_absmean": 0.03662796, "gate_b2c/L5_absmax": 0.52446759, "gate_b2c/L5_rms": 0.04574282, "gate_b2c/L6_absmean": 0.05314333, "gate_b2c/L6_absmax": 0.34377959, "gate_b2c/L6_rms": 0.0591637}, "streams": {"cos_a2b/round0": 0.353938, "cos_c2b/round0": 0.213055, "cos_b2a/round0": 0.149651, "cos_b2c/round0": 0.125256, "rms_a/round0": 8.056771, "rms_b/round0": 13.432232, "rms_c/round0": 5.112029, "cos_a2b/round1": 0.410877, "cos_c2b/round1": 0.307261, "cos_b2a/round1": 0.240994, "cos_b2c/round1": 0.120067, "rms_a/round1": 16.173473, "rms_b/round1": 32.281242, "rms_c/round1": 11.312138, "cos_a2b/round2": 0.522157, "cos_c2b/round2": 0.507783, "cos_b2a/round2": 0.087553, "cos_b2c/round2": 0.1727, "rms_a/round2": 44.562813, "rms_b/round2": 84.531952, "rms_c/round2": 21.90785}, "alpha_mean": 0.4264124004462815, "alpha_hist": [0.02708, 0.14687, 0.18555, 0.1517, 0.1192, 0.11258, 0.10658, 0.09586, 0.0519, 0.00269], "alpha_by_type": {"special": {"share": 0.0011, "mean": 0.6971, "std": 0.09707}, "whitespace": {"share": 0.03175, "mean": 0.61525, "std": 0.16144}, "digit": {"share": 0.01231, "mean": 0.42791, "std": 0.15763}, "word_start": {"share": 0.58392, "mean": 0.4358, "std": 0.21232}, "word_cont": {"share": 0.25816, "mean": 0.28665, "std": 0.14513}, "punct": {"share": 0.11275, "mean": 0.64186, "std": 0.18083}}, "alpha_p05": 0.12637, "alpha_p50": 0.43975, "alpha_p95": 0.81435}
{"step": 62210, "total_steps": 72479, "tokens": 32615956480, "loss": 4.679215669631958, "l_mix": 2.903903216123581, "l_a": 2.9781765043735504, "l_c": 2.939531624317169, "z_loss": 39.46474266052246, "lr": 0.002282695253301801, "wd": 0.01, "grad_norm": 0.17593173682689667, "tokens_per_second": 1457218.373528235, "eta_seconds": 3694.6511036396028, "data_wait_seconds": 4.634261131286621e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 62220, "total_steps": 72479, "tokens": 32621199360, "loss": 4.592941045761108, "l_mix": 2.8494695127010345, "l_a": 2.9300970137119293, "l_c": 2.881474733352661, "z_loss": 42.01848220825195, "lr": 0.0022821161069052735, "wd": 0.01, "grad_norm": 0.20374882221221924, "tokens_per_second": 1398860.2339360667, "eta_seconds": 3845.0378826379774, "data_wait_seconds": 2.092123031616211e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 62230, "total_steps": 72479, "tokens": 32626442240, "loss": 4.642633497714996, "l_mix": 2.879381477832794, "l_a": 2.9586222767829895, "l_c": 2.9188840985298157, "z_loss": 42.189942359924316, "lr": 0.002281527503989208, "wd": 0.01, "grad_norm": 0.1662956327199936, "tokens_per_second": 1409958.9209074627, "eta_seconds": 3811.052671337128, "data_wait_seconds": 2.6702880859375e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 62240, "total_steps": 72479, "tokens": 32631685120, "loss": 4.622683763504028, "l_mix": 2.8677529096603394, "l_a": 2.945018917322159, "l_c": 2.9047502875328064, "z_loss": 42.069082260131836, "lr": 0.0022809294494701812, "wd": 0.01, "grad_norm": 0.12522795796394348, "tokens_per_second": 1460829.4617235838, "eta_seconds": 3674.7512099504474, "data_wait_seconds": 3.069639205932617e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 62250, "total_steps": 72479, "tokens": 32636928000, "loss": 3.094124287366867, "l_mix": 1.9054338485002518, "l_a": 2.0139412581920624, "l_c": 1.9483601599931717, "z_loss": 45.09246301651001, "lr": 0.002280321948343718, "wd": 0.01, "grad_norm": 0.18147508800029755, "tokens_per_second": 1422144.4337325213, "eta_seconds": 3771.0248163223264, "data_wait_seconds": 2.193450927734375e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 62260, "total_steps": 72479, "tokens": 32642170880, "loss": 4.255085051059723, "l_mix": 2.6371546387672424, "l_a": 2.7205825746059418, "l_c": 2.6725188195705414, "z_loss": 42.459630489349365, "lr": 0.00227970500568425, "wd": 0.01, "grad_norm": 0.1356668770313263, "tokens_per_second": 1329389.5328590302, "eta_seconds": 4030.195017766953, "data_wait_seconds": 3.096461296081543e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 62270, "total_steps": 72479, "tokens": 32647413760, "loss": 4.301150977611542, "l_mix": 2.665478229522705, "l_a": 2.752883404493332, "l_c": 2.699358731508255, "z_loss": 44.028772830963135, "lr": 0.002279078626645074, "wd": 0.01, "grad_norm": 0.17317257821559906, "tokens_per_second": 1246455.309354296, "eta_seconds": 4294.142077803612, "data_wait_seconds": 2.7626752853393555e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 62280, "total_steps": 72479, "tokens": 32652656640, "loss": 4.248604357242584, "l_mix": 2.6329990923404694, "l_a": 2.7173045873641968, "l_c": 2.6680460274219513, "z_loss": 43.64665603637695, "lr": 0.002278442816458309, "wd": 0.01, "grad_norm": 0.12167911231517792, "tokens_per_second": 1446777.1294995262, "eta_seconds": 3695.948189234734, "data_wait_seconds": 2.422928810119629e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 62290, "total_steps": 72479, "tokens": 32657899520, "loss": 3.122523844242096, "l_mix": 1.921682208776474, "l_a": 2.035359710454941, "l_c": 1.9674453139305115, "z_loss": 45.71714782714844, "lr": 0.002277797580434851, "wd": 0.01, "grad_norm": 0.1841970533132553, "tokens_per_second": 1335036.2538659635, "eta_seconds": 4001.36731607914, "data_wait_seconds": 2.9265880584716797e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 62300, "total_steps": 72479, "tokens": 32663142400, "loss": 3.1119880080223083, "l_mix": 1.9169320315122604, "l_a": 2.0208509266376495, "l_c": 1.9626687914133072, "z_loss": 45.36591196060181, "lr": 0.0022771429239643306, "wd": 0.01, "grad_norm": 0.24603016674518585, "tokens_per_second": 1329715.096618895, "eta_seconds": 4013.4368373870852, "data_wait_seconds": 2.232193946838379e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16035876, "gate_a2b/L1_absmax": 0.68877071, "gate_a2b/L1_rms": 0.17045325, "gate_a2b/L3_absmean": 0.15184365, "gate_a2b/L3_absmax": 0.49735606, "gate_a2b/L3_rms": 0.15754162, "gate_a2b/L5_absmean": 0.22182854, "gate_a2b/L5_absmax": 0.73096222, "gate_a2b/L5_rms": 0.22790389, "gate_c2b/L1_absmean": 0.03732171, "gate_c2b/L1_absmax": 0.62582648, "gate_c2b/L1_rms": 0.07006778, "gate_c2b/L3_absmean": 0.16911221, "gate_c2b/L3_absmax": 0.38884649, "gate_c2b/L3_rms": 0.1726599, "gate_c2b/L5_absmean": 0.47669011, "gate_c2b/L5_absmax": 0.67077303, "gate_c2b/L5_rms": 0.48608124, "gate_b2a/L7_absmean": 0.05149492, "gate_b2a/L7_absmax": 0.3374581, "gate_b2a/L7_rms": 0.05737345, "gate_b2a/L11_absmean": 0.08288461, "gate_b2a/L11_absmax": 0.61561, "gate_b2a/L11_rms": 0.09210522, "gate_b2a/L14_absmean": 0.21567234, "gate_b2a/L14_absmax": 0.82599419, "gate_b2a/L14_rms": 0.22518447, "gate_b2c/L3_absmean": 0.03507454, "gate_b2c/L3_absmax": 0.5566957, "gate_b2c/L3_rms": 0.04573607, "gate_b2c/L5_absmean": 0.03663583, "gate_b2c/L5_absmax": 0.52681869, "gate_b2c/L5_rms": 0.04580167, "gate_b2c/L6_absmean": 0.05333814, "gate_b2c/L6_absmax": 0.34312913, "gate_b2c/L6_rms": 0.05928222}, "streams": {"cos_a2b/round0": 0.354431, "cos_c2b/round0": 0.21322, "cos_b2a/round0": 0.151141, "cos_b2c/round0": 0.126096, "rms_a/round0": 8.086935, "rms_b/round0": 13.458093, "rms_c/round0": 5.124082, "cos_a2b/round1": 0.40939, "cos_c2b/round1": 0.307685, "cos_b2a/round1": 0.244069, "cos_b2c/round1": 0.116684, "rms_a/round1": 16.078293, "rms_b/round1": 32.241982, "rms_c/round1": 11.393303, "cos_a2b/round2": 0.516143, "cos_c2b/round2": 0.514097, "cos_b2a/round2": 0.089532, "cos_b2c/round2": 0.176465, "rms_a/round2": 44.513355, "rms_b/round2": 85.300224, "rms_c/round2": 22.190088}, "alpha_mean": 0.4145906731778979, "alpha_hist": [0.02423, 0.14464, 0.20209, 0.15829, 0.12648, 0.11424, 0.10597, 0.08701, 0.03548, 0.00157], "alpha_by_type": {"special": {"share": 0.00105, "mean": 0.63218, "std": 0.09345}, "whitespace": {"share": 0.03462, "mean": 0.60951, "std": 0.16569}, "digit": {"share": 0.01105, "mean": 0.36926, "std": 0.17186}, "word_start": {"share": 0.56647, "mean": 0.41968, "std": 0.20096}, "word_cont": {"share": 0.2655, "mean": 0.28706, "std": 0.14053}, "punct": {"share": 0.12131, "mean": 0.61655, "std": 0.17407}}, "alpha_p05": 0.12295, "alpha_p50": 0.37296, "alpha_p95": 0.78531}
{"step": 62310, "total_steps": 72479, "tokens": 32668385280, "loss": 3.13348188996315, "l_mix": 1.9272240847349167, "l_a": 2.043582946062088, "l_c": 1.97727632522583, "z_loss": 47.646302700042725, "lr": 0.0022764788525150665, "wd": 0.01, "grad_norm": 0.2462736815214157, "tokens_per_second": 1479163.8901316395, "eta_seconds": 3604.390769386291, "data_wait_seconds": 2.7924776077270508e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 62320, "total_steps": 72479, "tokens": 32673628160, "loss": 2.901490181684494, "l_mix": 1.7974072992801666, "l_a": 1.8576811105012894, "l_c": 1.8225950747728348, "z_loss": 52.593515396118164, "lr": 0.0022758053716340197, "wd": 0.01, "grad_norm": 0.15397939085960388, "tokens_per_second": 1409137.545526228, "eta_seconds": 3779.7884308099747, "data_wait_seconds": 2.002716064453125e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 62330, "total_steps": 72479, "tokens": 32678871040, "loss": 4.206237375736237, "l_mix": 2.606590986251831, "l_a": 2.691798835992813, "l_c": 2.6403555870056152, "z_loss": 43.408854484558105, "lr": 0.002275122486946749, "wd": 0.01, "grad_norm": 0.17395652830600739, "tokens_per_second": 1434097.7462286856, "eta_seconds": 3710.3460527658463, "data_wait_seconds": 2.6911497116088867e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 62340, "total_steps": 72479, "tokens": 32684113920, "loss": 4.2518423199653625, "l_mix": 2.6341792941093445, "l_a": 2.7228426039218903, "l_c": 2.6693672835826874, "z_loss": 44.263617515563965, "lr": 0.002274430204157362, "wd": 0.01, "grad_norm": 0.220635786652565, "tokens_per_second": 1416251.6315394035, "eta_seconds": 3753.397993421555, "data_wait_seconds": 3.555417060852051e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 62350, "total_steps": 72479, "tokens": 32689356800, "loss": 4.251931726932526, "l_mix": 2.6351040601730347, "l_a": 2.72015118598938, "l_c": 2.669274091720581, "z_loss": 42.18741750717163, "lr": 0.0022737285290484687, "wd": 0.01, "grad_norm": 0.19023463129997253, "tokens_per_second": 1266145.563486107, "eta_seconds": 4194.235880255699, "data_wait_seconds": 1.6033649444580078e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 62360, "total_steps": 72479, "tokens": 32694599680, "loss": 4.30497008562088, "l_mix": 2.6676777601242065, "l_a": 2.75387379527092, "l_c": 2.7037669718265533, "z_loss": 43.73633670806885, "lr": 0.002273017467481133, "wd": 0.01, "grad_norm": 0.1448887586593628, "tokens_per_second": 1336373.2244637213, "eta_seconds": 3969.9016523838045, "data_wait_seconds": 1.5854835510253906e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 62370, "total_steps": 72479, "tokens": 32699842560, "loss": 4.279608249664307, "l_mix": 2.6525204479694366, "l_a": 2.7346938848495483, "l_c": 2.688932031393051, "z_loss": 44.34510850906372, "lr": 0.002272297025394822, "wd": 0.01, "grad_norm": 0.14517444372177124, "tokens_per_second": 1478277.9175795165, "eta_seconds": 3585.271300458908, "data_wait_seconds": 2.810359001159668e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 62380, "total_steps": 72479, "tokens": 32705085440, "loss": 4.264467835426331, "l_mix": 2.6436608731746674, "l_a": 2.726759433746338, "l_c": 2.6759301126003265, "z_loss": 41.79347467422485, "lr": 0.00227156720880736, "wd": 0.01, "grad_norm": 0.17073960602283478, "tokens_per_second": 1406350.1605722806, "eta_seconds": 3764.9119404554367, "data_wait_seconds": 4.172325134277344e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 62390, "total_steps": 72479, "tokens": 32710328320, "loss": 4.205628722906113, "l_mix": 2.60599547624588, "l_a": 2.6920635402202606, "l_c": 2.6400473415851593, "z_loss": 43.427842140197754, "lr": 0.0022708280238148745, "wd": 0.01, "grad_norm": 0.17411521077156067, "tokens_per_second": 1337570.1573991391, "eta_seconds": 3954.5900472879407, "data_wait_seconds": 2.3752450942993164e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 62400, "total_steps": 72479, "tokens": 32715571200, "loss": 3.068201184272766, "l_mix": 1.8879203647375107, "l_a": 2.0002303570508957, "l_c": 1.9340387284755707, "z_loss": 46.0363883972168, "lr": 0.0022700794765917478, "wd": 0.01, "grad_norm": 0.1614009290933609, "tokens_per_second": 1350089.7751409046, "eta_seconds": 3914.03508810997, "data_wait_seconds": 1.8328428268432617e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.1601069, "gate_a2b/L1_absmax": 0.6893084, "gate_a2b/L1_rms": 0.17022771, "gate_a2b/L3_absmean": 0.15202184, "gate_a2b/L3_absmax": 0.49890572, "gate_a2b/L3_rms": 0.15773903, "gate_a2b/L5_absmean": 0.22165322, "gate_a2b/L5_absmax": 0.73190331, "gate_a2b/L5_rms": 0.2276985, "gate_c2b/L1_absmean": 0.03714308, "gate_c2b/L1_absmax": 0.62672961, "gate_c2b/L1_rms": 0.07002963, "gate_c2b/L3_absmean": 0.16853854, "gate_c2b/L3_absmax": 0.38965487, "gate_c2b/L3_rms": 0.17214641, "gate_c2b/L5_absmean": 0.47624072, "gate_c2b/L5_absmax": 0.667427, "gate_c2b/L5_rms": 0.48572657, "gate_b2a/L7_absmean": 0.05100938, "gate_b2a/L7_absmax": 0.33694246, "gate_b2a/L7_rms": 0.05701505, "gate_b2a/L11_absmean": 0.08281221, "gate_b2a/L11_absmax": 0.6169979, "gate_b2a/L11_rms": 0.09210683, "gate_b2a/L14_absmean": 0.2152577, "gate_b2a/L14_absmax": 0.82570916, "gate_b2a/L14_rms": 0.22478075, "gate_b2c/L3_absmean": 0.03492392, "gate_b2c/L3_absmax": 0.55744767, "gate_b2c/L3_rms": 0.04572296, "gate_b2c/L5_absmean": 0.03668417, "gate_b2c/L5_absmax": 0.52832043, "gate_b2c/L5_rms": 0.04588203, "gate_b2c/L6_absmean": 0.05313857, "gate_b2c/L6_absmax": 0.34393477, "gate_b2c/L6_rms": 0.05910617}, "streams": {"cos_a2b/round0": 0.354132, "cos_c2b/round0": 0.212951, "cos_b2a/round0": 0.148538, "cos_b2c/round0": 0.126097, "rms_a/round0": 8.080328, "rms_b/round0": 13.445403, "rms_c/round0": 5.086002, "cos_a2b/round1": 0.409594, "cos_c2b/round1": 0.304256, "cos_b2a/round1": 0.240539, "cos_b2c/round1": 0.117292, "rms_a/round1": 16.179043, "rms_b/round1": 32.292038, "rms_c/round1": 11.431875, "cos_a2b/round2": 0.51917, "cos_c2b/round2": 0.51249, "cos_b2a/round2": 0.091263, "cos_b2c/round2": 0.171061, "rms_a/round2": 44.872776, "rms_b/round2": 85.420692, "rms_c/round2": 22.40571}, "alpha_mean": 0.42435342903274886, "alpha_hist": [0.02655, 0.14468, 0.1797, 0.15929, 0.12802, 0.11649, 0.10393, 0.09261, 0.04617, 0.00256], "alpha_by_type": {"special": {"share": 0.00111, "mean": 0.70899, "std": 0.10622}, "whitespace": {"share": 0.03377, "mean": 0.63888, "std": 0.15643}, "digit": {"share": 0.0146, "mean": 0.38644, "std": 0.1429}, "word_start": {"share": 0.57518, "mean": 0.42967, "std": 0.20933}, "word_cont": {"share": 0.25745, "mean": 0.29573, "std": 0.14511}, "punct": {"share": 0.11789, "mean": 0.61988, "std": 0.18393}}, "alpha_p05": 0.12809, "alpha_p50": 0.37708, "alpha_p95": 0.79944}
{"step": 62410, "total_steps": 72479, "tokens": 32720814080, "loss": 3.1353525519371033, "l_mix": 1.9298978000879288, "l_a": 2.039743036031723, "l_c": 1.9784393161535263, "z_loss": 44.81238842010498, "lr": 0.002269321573390564, "wd": 0.01, "grad_norm": 0.20846743881702423, "tokens_per_second": 1226310.8173478707, "eta_seconds": 4304.826963377, "data_wait_seconds": 1.8924474716186523e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 62420, "total_steps": 72479, "tokens": 32726056960, "loss": 3.0846842229366302, "l_mix": 1.8978592306375504, "l_a": 2.0121613442897797, "l_c": 1.9439218491315842, "z_loss": 47.329670429229736, "lr": 0.002268554320542058, "wd": 0.01, "grad_norm": 0.20013904571533203, "tokens_per_second": 1320383.2995267145, "eta_seconds": 3994.1530568361286, "data_wait_seconds": 2.1696090698242188e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 62430, "total_steps": 72479, "tokens": 32731299840, "loss": 2.9213134050369263, "l_mix": 1.8081248253583908, "l_a": 1.8766113817691803, "l_c": 1.8340169489383698, "z_loss": 52.905768394470215, "lr": 0.002267777724455061, "wd": 0.01, "grad_norm": 0.13439196348190308, "tokens_per_second": 1427819.6156004816, "eta_seconds": 3689.940980243683, "data_wait_seconds": 2.4080276489257812e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 62440, "total_steps": 72479, "tokens": 32736542720, "loss": 4.16997355222702, "l_mix": 2.584653526544571, "l_a": 2.666684627532959, "l_c": 2.6177152693271637, "z_loss": 44.16523551940918, "lr": 0.0022669917916164485, "wd": 0.01, "grad_norm": 0.1908193826675415, "tokens_per_second": 1332271.8421065547, "eta_seconds": 3950.640601754188, "data_wait_seconds": 1.582503318786621e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 62450, "total_steps": 72479, "tokens": 32741785600, "loss": 4.376436114311218, "l_mix": 2.712337553501129, "l_a": 2.7978346943855286, "l_c": 2.749160051345825, "z_loss": 43.22364568710327, "lr": 0.0022661965285910857, "wd": 0.01, "grad_norm": 0.15252147614955902, "tokens_per_second": 1454706.7107127002, "eta_seconds": 3614.532271885872, "data_wait_seconds": 1.5109777450561523e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 62460, "total_steps": 72479, "tokens": 32747028480, "loss": 4.329017698764801, "l_mix": 2.6841840147972107, "l_a": 2.7673870027065277, "l_c": 2.7153915464878082, "z_loss": 41.9043083190918, "lr": 0.002265391942021771, "wd": 0.01, "grad_norm": 0.23465965688228607, "tokens_per_second": 1401462.9075771435, "eta_seconds": 3748.1130921125414, "data_wait_seconds": 1.6450881958007812e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 62470, "total_steps": 72479, "tokens": 32752271360, "loss": 4.3141472935676575, "l_mix": 2.67282372713089, "l_a": 2.758560538291931, "l_c": 2.7125179171562195, "z_loss": 43.38108539581299, "lr": 0.0022645780386291847, "wd": 0.01, "grad_norm": 0.27586978673934937, "tokens_per_second": 1399791.2847026328, "eta_seconds": 3748.8435950040816, "data_wait_seconds": 2.16066837310791e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 62480, "total_steps": 72479, "tokens": 32757514240, "loss": 4.175462305545807, "l_mix": 2.5875478982925415, "l_a": 2.6747641563415527, "l_c": 2.618283599615097, "z_loss": 44.33566617965698, "lr": 0.0022637548252118275, "wd": 0.01, "grad_norm": 0.34743139147758484, "tokens_per_second": 1419201.4411241247, "eta_seconds": 3693.877105879784, "data_wait_seconds": 3.153085708618164e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 62490, "total_steps": 72479, "tokens": 32762757120, "loss": 4.545204699039459, "l_mix": 2.819033622741699, "l_a": 2.8968844413757324, "l_c": 2.8570191264152527, "z_loss": 43.455782890319824, "lr": 0.0022629223086459674, "wd": 0.01, "grad_norm": 0.21579132974147797, "tokens_per_second": 1456376.1384081799, "eta_seconds": 3595.989177441597, "data_wait_seconds": 2.142786979675293e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 62500, "total_steps": 72479, "tokens": 32768000000, "loss": 4.567667126655579, "l_mix": 2.833089768886566, "l_a": 2.909883290529251, "l_c": 2.872040957212448, "z_loss": 41.97205352783203, "lr": 0.0022620804958855822, "wd": 0.01, "grad_norm": 0.1556355208158493, "tokens_per_second": 1405149.3069102454, "eta_seconds": 3723.355181026459, "data_wait_seconds": 4.291534423828125e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16039006, "gate_a2b/L1_absmax": 0.69010895, "gate_a2b/L1_rms": 0.17047881, "gate_a2b/L3_absmean": 0.15209976, "gate_a2b/L3_absmax": 0.49802113, "gate_a2b/L3_rms": 0.15779616, "gate_a2b/L5_absmean": 0.22161655, "gate_a2b/L5_absmax": 0.73238719, "gate_a2b/L5_rms": 0.22769426, "gate_c2b/L1_absmean": 0.03664696, "gate_c2b/L1_absmax": 0.62833083, "gate_c2b/L1_rms": 0.06988837, "gate_c2b/L3_absmean": 0.16848417, "gate_c2b/L3_absmax": 0.38861305, "gate_c2b/L3_rms": 0.17202459, "gate_c2b/L5_absmean": 0.47574082, "gate_c2b/L5_absmax": 0.66594064, "gate_c2b/L5_rms": 0.48502076, "gate_b2a/L7_absmean": 0.05103076, "gate_b2a/L7_absmax": 0.33575147, "gate_b2a/L7_rms": 0.05699906, "gate_b2a/L11_absmean": 0.08285289, "gate_b2a/L11_absmax": 0.61684185, "gate_b2a/L11_rms": 0.09207971, "gate_b2a/L14_absmean": 0.215363, "gate_b2a/L14_absmax": 0.82327378, "gate_b2a/L14_rms": 0.22486156, "gate_b2c/L3_absmean": 0.03490257, "gate_b2c/L3_absmax": 0.55823994, "gate_b2c/L3_rms": 0.0456984, "gate_b2c/L5_absmean": 0.03645527, "gate_b2c/L5_absmax": 0.52555132, "gate_b2c/L5_rms": 0.04562668, "gate_b2c/L6_absmean": 0.05339897, "gate_b2c/L6_absmax": 0.34332794, "gate_b2c/L6_rms": 0.05933313}, "streams": {"cos_a2b/round0": 0.353007, "cos_c2b/round0": 0.24453, "cos_b2a/round0": 0.161517, "cos_b2c/round0": 0.145904, "rms_a/round0": 7.761928, "rms_b/round0": 13.112025, "rms_c/round0": 4.943713, "cos_a2b/round1": 0.412244, "cos_c2b/round1": 0.309037, "cos_b2a/round1": 0.25204, "cos_b2c/round1": 0.112348, "rms_a/round1": 15.139628, "rms_b/round1": 30.471712, "rms_c/round1": 10.656921, "cos_a2b/round2": 0.531768, "cos_c2b/round2": 0.524948, "cos_b2a/round2": 0.095938, "cos_b2c/round2": 0.186243, "rms_a/round2": 43.512054, "rms_b/round2": 83.625519, "rms_c/round2": 20.739218}, "alpha_mean": 0.43802296251601547, "alpha_hist": [0.00835, 0.0677, 0.16814, 0.19382, 0.18842, 0.17542, 0.12769, 0.06343, 0.00702, 3e-05], "alpha_by_type": {"special": {"share": 0.0006, "mean": 0.56271, "std": 0.14511}, "whitespace": {"share": 0.05428, "mean": 0.54244, "std": 0.11976}, "digit": {"share": 0.03101, "mean": 0.43753, "std": 0.1249}, "word_start": {"share": 0.55783, "mean": 0.45681, "std": 0.16404}, "word_cont": {"share": 0.24283, "mean": 0.31825, "std": 0.12824}, "punct": {"share": 0.11346, "mean": 0.55152, "std": 0.15575}}, "alpha_p05": 0.18097, "alpha_p50": 0.43518, "alpha_p95": 0.7068}
{"step": 62510, "total_steps": 72479, "tokens": 32773242880, "loss": 4.437097549438477, "l_mix": 2.7521459460258484, "l_a": 2.8303486704826355, "l_c": 2.7861562371253967, "z_loss": 41.986008167266846, "lr": 0.002261229393962298, "wd": 0.01, "grad_norm": 0.12902024388313293, "tokens_per_second": 1419020.7528151358, "eta_seconds": 3683.263307905197, "data_wait_seconds": 1.8864870071411133e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 62520, "total_steps": 72479, "tokens": 32778485760, "loss": 3.104814440011978, "l_mix": 1.9103657752275467, "l_a": 2.025161385536194, "l_c": 1.956333950161934, "z_loss": 48.5932354927063, "lr": 0.0022603690099853354, "wd": 0.01, "grad_norm": 0.2631068229675293, "tokens_per_second": 1453646.4262602846, "eta_seconds": 3591.9217339754105, "data_wait_seconds": 1.7851591110229492e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 62530, "total_steps": 72479, "tokens": 32783728640, "loss": 3.1200127601623535, "l_mix": 1.9209530055522919, "l_a": 2.0287636071443558, "l_c": 1.9681021571159363, "z_loss": 44.810115814208984, "lr": 0.0022594993511414465, "wd": 0.01, "grad_norm": 0.1952257603406906, "tokens_per_second": 1439709.6096176037, "eta_seconds": 3623.0509799718857, "data_wait_seconds": 2.110004425048828e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 62540, "total_steps": 72479, "tokens": 32788971520, "loss": 3.0379313230514526, "l_mix": 1.8818988054990768, "l_a": 1.945817306637764, "l_c": 1.9076243042945862, "z_loss": 52.724215030670166, "lr": 0.002258620424694856, "wd": 0.01, "grad_norm": 0.18221360445022583, "tokens_per_second": 1407164.9570563266, "eta_seconds": 3703.118391251564, "data_wait_seconds": 1.7434358596801758e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 62550, "total_steps": 72479, "tokens": 32794214400, "loss": 4.347920596599579, "l_mix": 2.694990962743759, "l_a": 2.7799699902534485, "l_c": 2.7297952473163605, "z_loss": 41.95722198486328, "lr": 0.0022577322379871995, "wd": 0.01, "grad_norm": 0.13053736090660095, "tokens_per_second": 1462296.1048155315, "eta_seconds": 3559.918907570839, "data_wait_seconds": 2.2083520889282227e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 62560, "total_steps": 72479, "tokens": 32799457280, "loss": 4.287421405315399, "l_mix": 2.6576105654239655, "l_a": 2.741586357355118, "l_c": 2.69111630320549, "z_loss": 41.80837154388428, "lr": 0.0022568347984374655, "wd": 0.01, "grad_norm": 0.1943201720714569, "tokens_per_second": 1448023.9032385217, "eta_seconds": 3591.385929727554, "data_wait_seconds": 1.5527009963989258e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 62570, "total_steps": 72479, "tokens": 32804700160, "loss": 4.31687331199646, "l_mix": 2.675934374332428, "l_a": 2.761824667453766, "l_c": 2.707971513271332, "z_loss": 42.56762218475342, "lr": 0.00225592811354193, "wd": 0.01, "grad_norm": 0.1465243697166443, "tokens_per_second": 1462867.413028496, "eta_seconds": 3551.3606672286987, "data_wait_seconds": 1.5676021575927734e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 62580, "total_steps": 72479, "tokens": 32809943040, "loss": 4.263194173574448, "l_mix": 2.6418191492557526, "l_a": 2.7246946692466736, "l_c": 2.679888665676117, "z_loss": 42.56860542297363, "lr": 0.0022550121908740937, "wd": 0.01, "grad_norm": 0.20286224782466888, "tokens_per_second": 1331781.6770533235, "eta_seconds": 3896.9802644252773, "data_wait_seconds": 1.5527009963989258e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 62590, "total_steps": 72479, "tokens": 32815185920, "loss": 4.270866572856903, "l_mix": 2.647681027650833, "l_a": 2.730852872133255, "l_c": 2.679765284061432, "z_loss": 41.96268701553345, "lr": 0.002254087038084622, "wd": 0.01, "grad_norm": 0.22744013369083405, "tokens_per_second": 1321425.3212411676, "eta_seconds": 3923.554323244095, "data_wait_seconds": 3.4928321838378906e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 62600, "total_steps": 72479, "tokens": 32820428800, "loss": 4.540443122386932, "l_mix": 2.8157840371131897, "l_a": 2.895180493593216, "l_c": 2.8536830246448517, "z_loss": 42.83002281188965, "lr": 0.0022531526629012786, "wd": 0.01, "grad_norm": 0.15430530905723572, "tokens_per_second": 1407930.0359981942, "eta_seconds": 3678.763162636757, "data_wait_seconds": 3.8623809814453125e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.16003922, "gate_a2b/L1_absmax": 0.68711114, "gate_a2b/L1_rms": 0.17009728, "gate_a2b/L3_absmean": 0.15221068, "gate_a2b/L3_absmax": 0.49706092, "gate_a2b/L3_rms": 0.15784486, "gate_a2b/L5_absmean": 0.22164455, "gate_a2b/L5_absmax": 0.73121715, "gate_a2b/L5_rms": 0.22767895, "gate_c2b/L1_absmean": 0.03733344, "gate_c2b/L1_absmax": 0.62595266, "gate_c2b/L1_rms": 0.07003349, "gate_c2b/L3_absmean": 0.16864608, "gate_c2b/L3_absmax": 0.38699993, "gate_c2b/L3_rms": 0.17216271, "gate_c2b/L5_absmean": 0.47611523, "gate_c2b/L5_absmax": 0.67315024, "gate_c2b/L5_rms": 0.48548514, "gate_b2a/L7_absmean": 0.05086815, "gate_b2a/L7_absmax": 0.33500427, "gate_b2a/L7_rms": 0.05682168, "gate_b2a/L11_absmean": 0.08287349, "gate_b2a/L11_absmax": 0.61672127, "gate_b2a/L11_rms": 0.09209669, "gate_b2a/L14_absmean": 0.21593165, "gate_b2a/L14_absmax": 0.82251453, "gate_b2a/L14_rms": 0.22537251, "gate_b2c/L3_absmean": 0.0349384, "gate_b2c/L3_absmax": 0.55702484, "gate_b2c/L3_rms": 0.04561563, "gate_b2c/L5_absmean": 0.03696207, "gate_b2c/L5_absmax": 0.52601784, "gate_b2c/L5_rms": 0.04605291, "gate_b2c/L6_absmean": 0.05344805, "gate_b2c/L6_absmax": 0.34291908, "gate_b2c/L6_rms": 0.05933434}, "streams": {"cos_a2b/round0": 0.352449, "cos_c2b/round0": 0.241751, "cos_b2a/round0": 0.164094, "cos_b2c/round0": 0.144278, "rms_a/round0": 7.729627, "rms_b/round0": 13.195088, "rms_c/round0": 4.929741, "cos_a2b/round1": 0.40963, "cos_c2b/round1": 0.311322, "cos_b2a/round1": 0.251502, "cos_b2c/round1": 0.116335, "rms_a/round1": 15.186844, "rms_b/round1": 30.459734, "rms_c/round1": 10.718268, "cos_a2b/round2": 0.542015, "cos_c2b/round2": 0.516808, "cos_b2a/round2": 0.098394, "cos_b2c/round2": 0.186076, "rms_a/round2": 43.532738, "rms_b/round2": 82.572639, "rms_c/round2": 20.819592}, "alpha_mean": 0.44500631247221634, "alpha_hist": [0.00999, 0.06197, 0.15733, 0.18538, 0.19289, 0.18616, 0.13683, 0.06325, 0.00586, 0.00035], "alpha_by_type": {"special": {"share": 0.00075, "mean": 0.51881, "std": 0.16351}, "whitespace": {"share": 0.06384, "mean": 0.5428, "std": 0.12611}, "digit": {"share": 0.0278, "mean": 0.45432, "std": 0.12564}, "word_start": {"share": 0.56111, "mean": 0.45895, "std": 0.16117}, "word_cont": {"share": 0.24049, "mean": 0.32878, "std": 0.13307}, "punct": {"share": 0.106, "mean": 0.57302, "std": 0.1518}}, "alpha_p05": 0.17526, "alpha_p50": 0.44734, "alpha_p95": 0.71203}
{"step": 62610, "total_steps": 72479, "tokens": 32825671680, "loss": 4.701297640800476, "l_mix": 2.9176725447177887, "l_a": 2.99141988158226, "l_c": 2.9539965987205505, "z_loss": 41.80431842803955, "lr": 0.00225220907312886, "wd": 0.01, "grad_norm": 0.18576468527317047, "tokens_per_second": 1436664.5428681727, "eta_seconds": 3601.5354438066483, "data_wait_seconds": 3.6835670471191406e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 62620, "total_steps": 72479, "tokens": 32830914560, "loss": 4.6629268527030945, "l_mix": 2.8926643431186676, "l_a": 2.970599979162216, "l_c": 2.9302748143672943, "z_loss": 41.973326206207275, "lr": 0.0022512562766491342, "wd": 0.01, "grad_norm": 0.14449822902679443, "tokens_per_second": 1354790.7089522008, "eta_seconds": 3815.316534018517, "data_wait_seconds": 1.704692840576172e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 62630, "total_steps": 72479, "tokens": 32836157440, "loss": 4.548614740371704, "l_mix": 2.8219365775585175, "l_a": 2.897174835205078, "l_c": 2.858418583869934, "z_loss": 40.0670599937439, "lr": 0.0022502942814207703, "wd": 0.01, "grad_norm": 0.18793222308158875, "tokens_per_second": 1393737.485603744, "eta_seconds": 3704.939104628563, "data_wait_seconds": 3.2007694244384766e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 62640, "total_steps": 72479, "tokens": 32841400320, "loss": 4.541196525096893, "l_mix": 2.816417247056961, "l_a": 2.8983622789382935, "l_c": 2.8509017527103424, "z_loss": 41.495816707611084, "lr": 0.0022493230954792756, "wd": 0.01, "grad_norm": 0.14743748307228088, "tokens_per_second": 1474826.609475208, "eta_seconds": 3497.6787093877792, "data_wait_seconds": 1.5944242477416992e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 62650, "total_steps": 72479, "tokens": 32846643200, "loss": 4.601989388465881, "l_mix": 2.8553938269615173, "l_a": 2.930878698825836, "l_c": 2.8911063075065613, "z_loss": 42.554834842681885, "lr": 0.002248342726936927, "wd": 0.01, "grad_norm": 0.23576290905475616, "tokens_per_second": 1439623.0856223633, "eta_seconds": 3579.5666264772417, "data_wait_seconds": 2.1070241928100586e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 62660, "total_steps": 72479, "tokens": 32851886080, "loss": 4.2797810435295105, "l_mix": 2.6519093215465546, "l_a": 2.737016499042511, "l_c": 2.68922221660614, "z_loss": 42.624298095703125, "lr": 0.0022473531839827016, "wd": 0.01, "grad_norm": 0.17511680722236633, "tokens_per_second": 1326822.692417575, "eta_seconds": 3879.9335445642473, "data_wait_seconds": 3.311038017272949e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 62670, "total_steps": 72479, "tokens": 32857128960, "loss": 4.185373485088348, "l_mix": 2.5933134257793427, "l_a": 2.678379565477371, "l_c": 2.628487229347229, "z_loss": 43.03656721115112, "lr": 0.0022463544748822124, "wd": 0.01, "grad_norm": 0.20990127325057983, "tokens_per_second": 1189477.9726598214, "eta_seconds": 4323.52772409916, "data_wait_seconds": 1.5854835510253906e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 62680, "total_steps": 72479, "tokens": 32862371840, "loss": 4.38459050655365, "l_mix": 2.7173631489276886, "l_a": 2.8050685822963715, "l_c": 2.7523558139801025, "z_loss": 43.213791847229004, "lr": 0.002245346607977635, "wd": 0.01, "grad_norm": 0.12065233290195465, "tokens_per_second": 1282954.9398505278, "eta_seconds": 4004.4259953498836, "data_wait_seconds": 3.567337989807129e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 62690, "total_steps": 72479, "tokens": 32867614720, "loss": 4.155251920223236, "l_mix": 2.574025124311447, "l_a": 2.660309076309204, "l_c": 2.610446661710739, "z_loss": 43.934814453125, "lr": 0.0022443295916876405, "wd": 0.01, "grad_norm": 0.17117932438850403, "tokens_per_second": 1332489.5668515489, "eta_seconds": 3851.6288304805757, "data_wait_seconds": 1.8417835235595703e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 62700, "total_steps": 72479, "tokens": 32872857600, "loss": 4.123880565166473, "l_mix": 2.5553121864795685, "l_a": 2.639050215482712, "l_c": 2.5895107984542847, "z_loss": 43.98755121231079, "lr": 0.0022433034345073245, "wd": 0.01, "grad_norm": 0.16735519468784332, "tokens_per_second": 1350408.8063359847, "eta_seconds": 3796.6372315883636, "data_wait_seconds": 2.2977590560913086e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.15970297, "gate_a2b/L1_absmax": 0.69036585, "gate_a2b/L1_rms": 0.16985771, "gate_a2b/L3_absmean": 0.15195973, "gate_a2b/L3_absmax": 0.49831492, "gate_a2b/L3_rms": 0.15765087, "gate_a2b/L5_absmean": 0.22162989, "gate_a2b/L5_absmax": 0.73523879, "gate_a2b/L5_rms": 0.22767003, "gate_c2b/L1_absmean": 0.03758053, "gate_c2b/L1_absmax": 0.62868363, "gate_c2b/L1_rms": 0.07023136, "gate_c2b/L3_absmean": 0.1678298, "gate_c2b/L3_absmax": 0.38741878, "gate_c2b/L3_rms": 0.17138763, "gate_c2b/L5_absmean": 0.47479293, "gate_c2b/L5_absmax": 0.66863763, "gate_c2b/L5_rms": 0.48409012, "gate_b2a/L7_absmean": 0.05110319, "gate_b2a/L7_absmax": 0.33593056, "gate_b2a/L7_rms": 0.05708309, "gate_b2a/L11_absmean": 0.082834, "gate_b2a/L11_absmax": 0.61467856, "gate_b2a/L11_rms": 0.0920255, "gate_b2a/L14_absmean": 0.21536088, "gate_b2a/L14_absmax": 0.82177007, "gate_b2a/L14_rms": 0.22482052, "gate_b2c/L3_absmean": 0.03512427, "gate_b2c/L3_absmax": 0.5572983, "gate_b2c/L3_rms": 0.04582477, "gate_b2c/L5_absmean": 0.03666476, "gate_b2c/L5_absmax": 0.52918893, "gate_b2c/L5_rms": 0.04592171, "gate_b2c/L6_absmean": 0.05336815, "gate_b2c/L6_absmax": 0.34235671, "gate_b2c/L6_rms": 0.05927413}, "streams": {"cos_a2b/round0": 0.357555, "cos_c2b/round0": 0.266941, "cos_b2a/round0": 0.17714, "cos_b2c/round0": 0.161432, "rms_a/round0": 7.817413, "rms_b/round0": 13.013128, "rms_c/round0": 4.950345, "cos_a2b/round1": 0.426966, "cos_c2b/round1": 0.31229, "cos_b2a/round1": 0.259471, "cos_b2c/round1": 0.107946, "rms_a/round1": 15.472295, "rms_b/round1": 30.001162, "rms_c/round1": 10.5838, "cos_a2b/round2": 0.543454, "cos_c2b/round2": 0.53874, "cos_b2a/round2": 0.110875, "cos_b2c/round2": 0.187381, "rms_a/round2": 44.458603, "rms_b/round2": 84.868004, "rms_c/round2": 20.521223}, "alpha_mean": 0.42950902092619003, "alpha_hist": [0.0063, 0.05692, 0.17563, 0.21797, 0.22543, 0.15031, 0.09921, 0.06378, 0.00444, 0.0], "alpha_by_type": {"special": {"share": 0.00037, "mean": 0.45721, "std": 0.11872}, "whitespace": {"share": 0.03621, "mean": 0.47959, "std": 0.07505}, "digit": {"share": 0.03014, "mean": 0.33378, "std": 0.10862}, "word_start": {"share": 0.63969, "mean": 0.4341, "std": 0.15195}, "word_cont": {"share": 0.19455, "mean": 0.32303, "std": 0.1174}, "punct": {"share": 0.09904, "mean": 0.61971, "std": 0.13118}}, "alpha_p05": 0.19652, "alpha_p50": 0.41087, "alpha_p95": 0.71642}
{"step": 62710, "total_steps": 72479, "tokens": 32878100480, "loss": 2.917845517396927, "l_mix": 1.8056006580591202, "l_a": 1.875256597995758, "l_c": 1.8322261273860931, "z_loss": 53.00034713745117, "lr": 0.0022422681450081365, "wd": 0.01, "grad_norm": 0.14118127524852753, "tokens_per_second": 1455074.2186218284, "eta_seconds": 3519.9369258642196, "data_wait_seconds": 3.096461296081543e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 62720, "total_steps": 72479, "tokens": 32883343360, "loss": 4.608468174934387, "l_mix": 2.8603292405605316, "l_a": 2.931364059448242, "l_c": 2.8957655131816864, "z_loss": 40.27238893508911, "lr": 0.0022412237318378066, "wd": 0.01, "grad_norm": 0.24804195761680603, "tokens_per_second": 1442849.5916083339, "eta_seconds": 3546.1260977983475, "data_wait_seconds": 1.9788742065429688e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 62730, "total_steps": 72479, "tokens": 32888586240, "loss": 4.59052038192749, "l_mix": 2.8477656841278076, "l_a": 2.9251544177532196, "l_c": 2.8840273916721344, "z_loss": 41.62840747833252, "lr": 0.0022401702037202756, "wd": 0.01, "grad_norm": 0.19185234606266022, "tokens_per_second": 1392935.248738612, "eta_seconds": 3669.433820867538, "data_wait_seconds": 1.6182661056518555e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 62740, "total_steps": 72479, "tokens": 32893829120, "loss": 4.56060391664505, "l_mix": 2.829572856426239, "l_a": 2.905489981174469, "l_c": 2.8646136224269867, "z_loss": 42.6210560798645, "lr": 0.0022391075694556204, "wd": 0.01, "grad_norm": 0.24480877816677094, "tokens_per_second": 1446092.779879124, "eta_seconds": 3530.9220148563386, "data_wait_seconds": 2.1338462829589844e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 62750, "total_steps": 72479, "tokens": 32899072000, "loss": 4.4244884848594666, "l_mix": 2.744726777076721, "l_a": 2.8175290524959564, "l_c": 2.78167662024498, "z_loss": 42.96447992324829, "lr": 0.0022380358379199814, "wd": 0.01, "grad_norm": 0.16395780444145203, "tokens_per_second": 1466124.974366086, "eta_seconds": 3479.1017417907715, "data_wait_seconds": 3.960728645324707e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 62760, "total_steps": 72479, "tokens": 32904314880, "loss": 4.65229457616806, "l_mix": 2.8870924711227417, "l_a": 2.9621796905994415, "l_c": 2.9218268990516663, "z_loss": 41.49618434906006, "lr": 0.0022369550180654893, "wd": 0.01, "grad_norm": 0.12000340968370438, "tokens_per_second": 1430042.3126459534, "eta_seconds": 3563.2197921276097, "data_wait_seconds": 1.6033649444580078e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 62770, "total_steps": 72479, "tokens": 32909557760, "loss": 4.267358243465424, "l_mix": 2.644270747900009, "l_a": 2.730413019657135, "l_c": 2.679878681898117, "z_loss": 43.02744388580322, "lr": 0.0022358651189201876, "wd": 0.01, "grad_norm": 0.13551543653011322, "tokens_per_second": 1382974.7828642495, "eta_seconds": 3680.6977647542953, "data_wait_seconds": 1.58846378326416e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 62780, "total_steps": 72479, "tokens": 32914800640, "loss": 4.195055991411209, "l_mix": 2.5998217463493347, "l_a": 2.6840392649173737, "l_c": 2.6334076523780823, "z_loss": 43.02496576309204, "lr": 0.002234766149587959, "wd": 0.01, "grad_norm": 0.1196846216917038, "tokens_per_second": 1441390.5945978118, "eta_seconds": 3527.891281557083, "data_wait_seconds": 1.9878149032592773e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 62790, "total_steps": 72479, "tokens": 32920043520, "loss": 4.178860366344452, "l_mix": 2.5889817774295807, "l_a": 2.6748752892017365, "l_c": 2.6247198581695557, "z_loss": 44.414307594299316, "lr": 0.0022336581192484496, "wd": 0.01, "grad_norm": 0.15023484826087952, "tokens_per_second": 1364374.4294937034, "eta_seconds": 3723.1908794164656, "data_wait_seconds": 1.4960765838623047e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 62800, "total_steps": 72479, "tokens": 32925286400, "loss": 4.159212052822113, "l_mix": 2.576674997806549, "l_a": 2.664764851331711, "l_c": 2.610358476638794, "z_loss": 44.40399885177612, "lr": 0.0022325410371569917, "wd": 0.01, "grad_norm": 0.179474875330925, "tokens_per_second": 1392869.3416560884, "eta_seconds": 3643.2588472127914, "data_wait_seconds": 1.9550323486328125e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16004539, "gate_a2b/L1_absmax": 0.68848425, "gate_a2b/L1_rms": 0.17012182, "gate_a2b/L3_absmean": 0.15176216, "gate_a2b/L3_absmax": 0.49806657, "gate_a2b/L3_rms": 0.15747005, "gate_a2b/L5_absmean": 0.22142565, "gate_a2b/L5_absmax": 0.73393422, "gate_a2b/L5_rms": 0.22746795, "gate_c2b/L1_absmean": 0.03764873, "gate_c2b/L1_absmax": 0.62770247, "gate_c2b/L1_rms": 0.07026336, "gate_c2b/L3_absmean": 0.16802192, "gate_c2b/L3_absmax": 0.38880184, "gate_c2b/L3_rms": 0.17162482, "gate_c2b/L5_absmean": 0.47486955, "gate_c2b/L5_absmax": 0.66856843, "gate_c2b/L5_rms": 0.48422402, "gate_b2a/L7_absmean": 0.05146959, "gate_b2a/L7_absmax": 0.33546966, "gate_b2a/L7_rms": 0.05726269, "gate_b2a/L11_absmean": 0.08312762, "gate_b2a/L11_absmax": 0.61577147, "gate_b2a/L11_rms": 0.0923216, "gate_b2a/L14_absmean": 0.2155045, "gate_b2a/L14_absmax": 0.82204759, "gate_b2a/L14_rms": 0.22495618, "gate_b2c/L3_absmean": 0.03507621, "gate_b2c/L3_absmax": 0.55663365, "gate_b2c/L3_rms": 0.04578529, "gate_b2c/L5_absmean": 0.03697799, "gate_b2c/L5_absmax": 0.52841508, "gate_b2c/L5_rms": 0.04613523, "gate_b2c/L6_absmean": 0.05334147, "gate_b2c/L6_absmax": 0.34185866, "gate_b2c/L6_rms": 0.05926384}, "streams": {"cos_a2b/round0": 0.357344, "cos_c2b/round0": 0.241451, "cos_b2a/round0": 0.153389, "cos_b2c/round0": 0.14162, "rms_a/round0": 7.801015, "rms_b/round0": 13.165493, "rms_c/round0": 5.012531, "cos_a2b/round1": 0.398411, "cos_c2b/round1": 0.297022, "cos_b2a/round1": 0.248316, "cos_b2c/round1": 0.104175, "rms_a/round1": 15.122297, "rms_b/round1": 30.866816, "rms_c/round1": 10.71138, "cos_a2b/round2": 0.52662, "cos_c2b/round2": 0.524498, "cos_b2a/round2": 0.08575, "cos_b2c/round2": 0.178058, "rms_a/round2": 43.502926, "rms_b/round2": 84.428787, "rms_c/round2": 20.564512}, "alpha_mean": 0.41169478635013235, "alpha_hist": [0.01334, 0.07561, 0.19608, 0.22209, 0.19283, 0.15044, 0.09712, 0.0464, 0.00603, 8e-05], "alpha_by_type": {"special": {"share": 0.00075, "mean": 0.56381, "std": 0.10532}, "whitespace": {"share": 0.03053, "mean": 0.52905, "std": 0.12577}, "digit": {"share": 0.05533, "mean": 0.43931, "std": 0.12492}, "word_start": {"share": 0.51141, "mean": 0.41992, "std": 0.16276}, "word_cont": {"share": 0.25339, "mean": 0.31674, "std": 0.12106}, "punct": {"share": 0.14859, "mean": 0.51013, "std": 0.17094}}, "alpha_p05": 0.17078, "alpha_p50": 0.4031, "alpha_p95": 0.69596}
{"step": 62810, "total_steps": 72479, "tokens": 32930529280, "loss": 4.29993736743927, "l_mix": 2.664850115776062, "l_a": 2.74871164560318, "l_c": 2.7015792429447174, "z_loss": 43.80416202545166, "lr": 0.0022314149126445256, "wd": 0.01, "grad_norm": 0.1302049458026886, "tokens_per_second": 1433799.3700686365, "eta_seconds": 3535.599734401703, "data_wait_seconds": 2.0116567611694336e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 62820, "total_steps": 72479, "tokens": 32935772160, "loss": 3.226696401834488, "l_mix": 1.9996336698532104, "l_a": 2.0633195340633392, "l_c": 2.026889353990555, "z_loss": 53.14233684539795, "lr": 0.002230279755117525, "wd": 0.01, "grad_norm": 0.14812089502811432, "tokens_per_second": 1244145.9594379892, "eta_seconds": 4070.34058470726, "data_wait_seconds": 2.6941299438476562e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 62830, "total_steps": 72479, "tokens": 32941015040, "loss": 2.931978076696396, "l_mix": 1.8154667019844055, "l_a": 1.8800469040870667, "l_c": 1.8416576236486435, "z_loss": 51.88936519622803, "lr": 0.002229135574057912, "wd": 0.01, "grad_norm": 0.22985528409481049, "tokens_per_second": 1417786.106598722, "eta_seconds": 3568.136892056465, "data_wait_seconds": 3.6835670471191406e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 62840, "total_steps": 72479, "tokens": 32946257920, "loss": 4.157563090324402, "l_mix": 2.5750857293605804, "l_a": 2.6651682257652283, "l_c": 2.609756350517273, "z_loss": 44.6435604095459, "lr": 0.002227982379022986, "wd": 0.01, "grad_norm": 0.17317984998226166, "tokens_per_second": 1365013.3433776728, "eta_seconds": 3702.2436861276624, "data_wait_seconds": 3.17692756652832e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 62850, "total_steps": 72479, "tokens": 32951500800, "loss": 4.105723291635513, "l_mix": 2.542947769165039, "l_a": 2.628794312477112, "l_c": 2.5804575979709625, "z_loss": 44.590211391448975, "lr": 0.002226820179645337, "wd": 0.01, "grad_norm": 0.16392651200294495, "tokens_per_second": 1415969.387279372, "eta_seconds": 3565.309530949593, "data_wait_seconds": 1.895427703857422e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 62860, "total_steps": 72479, "tokens": 32956743680, "loss": 4.135158747434616, "l_mix": 2.561850845813751, "l_a": 2.64962574839592, "l_c": 2.5947335958480835, "z_loss": 43.282695293426514, "lr": 0.002225648985632769, "wd": 0.01, "grad_norm": 0.18221037089824677, "tokens_per_second": 1437980.821339279, "eta_seconds": 3507.0886879444124, "data_wait_seconds": 1.4841556549072266e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 62870, "total_steps": 72479, "tokens": 32961986560, "loss": 4.067112326622009, "l_mix": 2.518524706363678, "l_a": 2.608101963996887, "l_c": 2.553856611251831, "z_loss": 43.14011478424072, "lr": 0.0022244688067682185, "wd": 0.01, "grad_norm": 0.17103013396263123, "tokens_per_second": 1220789.9018659624, "eta_seconds": 4126.740714597702, "data_wait_seconds": 2.092123031616211e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 62880, "total_steps": 72479, "tokens": 32967229440, "loss": 4.172308027744293, "l_mix": 2.5848467350006104, "l_a": 2.6702440977096558, "l_c": 2.621293216943741, "z_loss": 43.2998161315918, "lr": 0.0022232796529096702, "wd": 0.01, "grad_norm": 0.13460755348205566, "tokens_per_second": 1406518.3701266076, "eta_seconds": 3578.083741307259, "data_wait_seconds": 2.327561378479004e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 62890, "total_steps": 72479, "tokens": 32972472320, "loss": 4.099963843822479, "l_mix": 2.5391522645950317, "l_a": 2.629150301218033, "l_c": 2.5735547244548798, "z_loss": 45.35960149765015, "lr": 0.0022220815339900777, "wd": 0.01, "grad_norm": 0.1606048345565796, "tokens_per_second": 1395279.5732862686, "eta_seconds": 3603.147160077095, "data_wait_seconds": 1.722574234008789e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 62900, "total_steps": 72479, "tokens": 32977715200, "loss": 4.094964921474457, "l_mix": 2.5354185104370117, "l_a": 2.624603509902954, "l_c": 2.573884129524231, "z_loss": 44.26096296310425, "lr": 0.0022208744600172806, "wd": 0.01, "grad_norm": 0.1382482945919037, "tokens_per_second": 1209602.7778830845, "eta_seconds": 4151.904115819931, "data_wait_seconds": 0.07081803679466248, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.1600648, "gate_a2b/L1_absmax": 0.68905944, "gate_a2b/L1_rms": 0.17009333, "gate_a2b/L3_absmean": 0.15210652, "gate_a2b/L3_absmax": 0.50016367, "gate_a2b/L3_rms": 0.15777823, "gate_a2b/L5_absmean": 0.2214262, "gate_a2b/L5_absmax": 0.7328431, "gate_a2b/L5_rms": 0.2274455, "gate_c2b/L1_absmean": 0.03760922, "gate_c2b/L1_absmax": 0.62853575, "gate_c2b/L1_rms": 0.07008501, "gate_c2b/L3_absmean": 0.16793585, "gate_c2b/L3_absmax": 0.38836241, "gate_c2b/L3_rms": 0.17154668, "gate_c2b/L5_absmean": 0.47443509, "gate_c2b/L5_absmax": 0.66769391, "gate_c2b/L5_rms": 0.48374817, "gate_b2a/L7_absmean": 0.05071297, "gate_b2a/L7_absmax": 0.33568564, "gate_b2a/L7_rms": 0.05668446, "gate_b2a/L11_absmean": 0.08296704, "gate_b2a/L11_absmax": 0.61612695, "gate_b2a/L11_rms": 0.09220178, "gate_b2a/L14_absmean": 0.21574911, "gate_b2a/L14_absmax": 0.82089251, "gate_b2a/L14_rms": 0.22513427, "gate_b2c/L3_absmean": 0.03511961, "gate_b2c/L3_absmax": 0.55707502, "gate_b2c/L3_rms": 0.04583526, "gate_b2c/L5_absmean": 0.03704853, "gate_b2c/L5_absmax": 0.52799523, "gate_b2c/L5_rms": 0.04616147, "gate_b2c/L6_absmean": 0.05350488, "gate_b2c/L6_absmax": 0.34021309, "gate_b2c/L6_rms": 0.05936974}, "streams": {"cos_a2b/round0": 0.355131, "cos_c2b/round0": 0.242396, "cos_b2a/round0": 0.161617, "cos_b2c/round0": 0.145669, "rms_a/round0": 7.803546, "rms_b/round0": 13.186296, "rms_c/round0": 5.015734, "cos_a2b/round1": 0.410459, "cos_c2b/round1": 0.304116, "cos_b2a/round1": 0.249762, "cos_b2c/round1": 0.110697, "rms_a/round1": 15.374773, "rms_b/round1": 30.926308, "rms_c/round1": 10.770776, "cos_a2b/round2": 0.532139, "cos_c2b/round2": 0.521213, "cos_b2a/round2": 0.095427, "cos_b2c/round2": 0.182016, "rms_a/round2": 43.949078, "rms_b/round2": 84.524414, "rms_c/round2": 20.997631}, "alpha_mean": 0.4266434434125159, "alpha_hist": [0.01045, 0.08499, 0.19522, 0.19664, 0.16437, 0.14476, 0.11859, 0.07155, 0.01323, 0.0002], "alpha_by_type": {"special": {"share": 0.00107, "mean": 0.56847, "std": 0.1059}, "whitespace": {"share": 0.02837, "mean": 0.53619, "std": 0.12805}, "digit": {"share": 0.03728, "mean": 0.46604, "std": 0.12072}, "word_start": {"share": 0.57626, "mean": 0.44039, "std": 0.17452}, "word_cont": {"share": 0.25253, "mean": 0.3099, "std": 0.12964}, "punct": {"share": 0.10449, "mean": 0.58772, "std": 0.16684}}, "alpha_p05": 0.16559, "alpha_p50": 0.40286, "alpha_p95": 0.75092}
{"step": 62910, "total_steps": 72479, "tokens": 32982958080, "loss": 4.092516005039215, "l_mix": 2.534060776233673, "l_a": 2.625209927558899, "l_c": 2.5696409046649933, "z_loss": 43.53233528137207, "lr": 0.0022196584410739168, "wd": 0.01, "grad_norm": 0.22398580610752106, "tokens_per_second": 1403600.7471187555, "eta_seconds": 3574.315475606918, "data_wait_seconds": 2.2083520889282227e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 62920, "total_steps": 72479, "tokens": 32988200960, "loss": 4.143882393836975, "l_mix": 2.566931366920471, "l_a": 2.655002325773239, "l_c": 2.6015006601810455, "z_loss": 43.64041566848755, "lr": 0.0022184334873173437, "wd": 0.01, "grad_norm": 0.1384962648153305, "tokens_per_second": 1372229.8165175284, "eta_seconds": 3652.2082027912143, "data_wait_seconds": 0.016961216926574707, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 62930, "total_steps": 72479, "tokens": 32993443840, "loss": 4.006622314453125, "l_mix": 2.480314075946808, "l_a": 2.571749269962311, "l_c": 2.515944719314575, "z_loss": 45.28548288345337, "lr": 0.00221719960897955, "wd": 0.01, "grad_norm": 0.14634054899215698, "tokens_per_second": 1380420.592287705, "eta_seconds": 3626.7396617889403, "data_wait_seconds": 3.656744956970215e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 62940, "total_steps": 72479, "tokens": 32998686720, "loss": 2.899510085582733, "l_mix": 1.7953526377677917, "l_a": 1.8585319221019745, "l_c": 1.8219930231571198, "z_loss": 52.332377910614014, "lr": 0.00221595681636707, "wd": 0.01, "grad_norm": 0.1802419126033783, "tokens_per_second": 1462794.4303656681, "eta_seconds": 3418.9241688251495, "data_wait_seconds": 2.3663043975830078e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 62950, "total_steps": 72479, "tokens": 33003929600, "loss": 3.932886838912964, "l_mix": 2.4352297484874725, "l_a": 2.523826688528061, "l_c": 2.468363404273987, "z_loss": 43.73449468612671, "lr": 0.0022147051198609016, "wd": 0.01, "grad_norm": 0.13158316910266876, "tokens_per_second": 1370615.7156738564, "eta_seconds": 3645.0336114406587, "data_wait_seconds": 1.958012580871582e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 62960, "total_steps": 72479, "tokens": 33009172480, "loss": 3.9610220789909363, "l_mix": 2.4528199434280396, "l_a": 2.539824217557907, "l_c": 2.4875159859657288, "z_loss": 45.51529121398926, "lr": 0.0022134445299164126, "wd": 0.01, "grad_norm": 0.17584413290023804, "tokens_per_second": 1430719.091428164, "eta_seconds": 3488.2441297531127, "data_wait_seconds": 2.8073787689208984e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 62970, "total_steps": 72479, "tokens": 33014415360, "loss": 3.9584430158138275, "l_mix": 2.450309306383133, "l_a": 2.5383159816265106, "l_c": 2.488796353340149, "z_loss": 44.49136734008789, "lr": 0.0022121750570632598, "wd": 0.01, "grad_norm": 0.20873220264911652, "tokens_per_second": 1356124.2364108416, "eta_seconds": 3676.252114772796, "data_wait_seconds": 1.58846378326416e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 62980, "total_steps": 72479, "tokens": 33019658240, "loss": 3.9884211719036102, "l_mix": 2.4694884717464447, "l_a": 2.5585386753082275, "l_c": 2.504570245742798, "z_loss": 44.50056982040405, "lr": 0.002210896711905298, "wd": 0.01, "grad_norm": 0.15081113576889038, "tokens_per_second": 1466394.3222345773, "eta_seconds": 3396.229538321495, "data_wait_seconds": 1.537799835205078e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 62990, "total_steps": 72479, "tokens": 33024901120, "loss": 4.03672069311142, "l_mix": 2.499361753463745, "l_a": 2.5876530706882477, "l_c": 2.53687646985054, "z_loss": 44.28659915924072, "lr": 0.0022096095051204923, "wd": 0.01, "grad_norm": 0.15757976472377777, "tokens_per_second": 1202612.8939148595, "eta_seconds": 4136.799844050407, "data_wait_seconds": 3.2067298889160156e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 63000, "total_steps": 72479, "tokens": 33030144000, "loss": 4.54837292432785, "l_mix": 2.821325719356537, "l_a": 2.899942994117737, "l_c": 2.856880843639374, "z_loss": 42.01004076004028, "lr": 0.002208313447460828, "wd": 0.01, "grad_norm": 0.12762269377708435, "tokens_per_second": 1415038.4601661556, "eta_seconds": 3512.078358221054, "data_wait_seconds": 2.1338462829589844e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.16027437, "gate_a2b/L1_absmax": 0.68745756, "gate_a2b/L1_rms": 0.1703496, "gate_a2b/L3_absmean": 0.15224718, "gate_a2b/L3_absmax": 0.49699512, "gate_a2b/L3_rms": 0.15787072, "gate_a2b/L5_absmean": 0.2213008, "gate_a2b/L5_absmax": 0.73122197, "gate_a2b/L5_rms": 0.22731982, "gate_c2b/L1_absmean": 0.03750597, "gate_c2b/L1_absmax": 0.62640339, "gate_c2b/L1_rms": 0.07009843, "gate_c2b/L3_absmean": 0.16780871, "gate_c2b/L3_absmax": 0.38722005, "gate_c2b/L3_rms": 0.17135455, "gate_c2b/L5_absmean": 0.47363272, "gate_c2b/L5_absmax": 0.66683984, "gate_c2b/L5_rms": 0.48293918, "gate_b2a/L7_absmean": 0.05131121, "gate_b2a/L7_absmax": 0.3342675, "gate_b2a/L7_rms": 0.05716716, "gate_b2a/L11_absmean": 0.08280304, "gate_b2a/L11_absmax": 0.61481917, "gate_b2a/L11_rms": 0.09203398, "gate_b2a/L14_absmean": 0.21603361, "gate_b2a/L14_absmax": 0.81796098, "gate_b2a/L14_rms": 0.22537683, "gate_b2c/L3_absmean": 0.03521167, "gate_b2c/L3_absmax": 0.55610454, "gate_b2c/L3_rms": 0.04586174, "gate_b2c/L5_absmean": 0.03701712, "gate_b2c/L5_absmax": 0.52765673, "gate_b2c/L5_rms": 0.04613092, "gate_b2c/L6_absmean": 0.05383141, "gate_b2c/L6_absmax": 0.33973551, "gate_b2c/L6_rms": 0.05965764}, "streams": {"cos_a2b/round0": 0.354738, "cos_c2b/round0": 0.239885, "cos_b2a/round0": 0.16612, "cos_b2c/round0": 0.147002, "rms_a/round0": 7.742523, "rms_b/round0": 13.171515, "rms_c/round0": 4.964416, "cos_a2b/round1": 0.413441, "cos_c2b/round1": 0.311359, "cos_b2a/round1": 0.252744, "cos_b2c/round1": 0.118983, "rms_a/round1": 15.267214, "rms_b/round1": 30.566843, "rms_c/round1": 10.730894, "cos_a2b/round2": 0.543385, "cos_c2b/round2": 0.516584, "cos_b2a/round2": 0.098019, "cos_b2c/round2": 0.189938, "rms_a/round2": 43.628819, "rms_b/round2": 82.83326, "rms_c/round2": 20.945698}, "alpha_mean": 0.4533346752968157, "alpha_hist": [0.00772, 0.05719, 0.1553, 0.18863, 0.18687, 0.17532, 0.13991, 0.07863, 0.01031, 0.00011], "alpha_by_type": {"special": {"share": 0.00084, "mean": 0.55459, "std": 0.1449}, "whitespace": {"share": 0.04865, "mean": 0.54741, "std": 0.12255}, "digit": {"share": 0.02686, "mean": 0.46908, "std": 0.11783}, "word_start": {"share": 0.59286, "mean": 0.46659, "std": 0.16652}, "word_cont": {"share": 0.23051, "mean": 0.33721, "std": 0.13196}, "punct": {"share": 0.10028, "mean": 0.59118, "std": 0.15974}}, "alpha_p05": 0.17953, "alpha_p50": 0.43614, "alpha_p95": 0.73335}
{"step": 63010, "total_steps": 72479, "tokens": 33035386880, "loss": 4.503899395465851, "l_mix": 2.7942302525043488, "l_a": 2.8736771047115326, "l_c": 2.825219750404358, "z_loss": 41.7855920791626, "lr": 0.0022070085497522224, "wd": 0.01, "grad_norm": 0.14446447789669037, "tokens_per_second": 1443567.1680971223, "eta_seconds": 3439.038502478599, "data_wait_seconds": 2.0891427993774414e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 63020, "total_steps": 72479, "tokens": 33040629760, "loss": 4.559223055839539, "l_mix": 2.8290646374225616, "l_a": 2.899543583393097, "l_c": 2.8676505386829376, "z_loss": 41.61913824081421, "lr": 0.002205694822894431, "wd": 0.01, "grad_norm": 0.24435529112815857, "tokens_per_second": 1235291.6771724331, "eta_seconds": 4014.630943965912, "data_wait_seconds": 2.5004148483276367e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 63030, "total_steps": 72479, "tokens": 33045872640, "loss": 4.72037136554718, "l_mix": 2.92907977104187, "l_a": 3.0033240914344788, "l_c": 2.9676477015018463, "z_loss": 41.98792505264282, "lr": 0.002204372277860963, "wd": 0.01, "grad_norm": 0.1721092164516449, "tokens_per_second": 1419792.1815228043, "eta_seconds": 3489.2411554813384, "data_wait_seconds": 2.2798776626586914e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 63040, "total_steps": 72479, "tokens": 33051115520, "loss": 4.500445604324341, "l_mix": 2.791273981332779, "l_a": 2.870315730571747, "l_c": 2.8269229233264923, "z_loss": 41.73917531967163, "lr": 0.002203040925698981, "wd": 0.01, "grad_norm": 0.15661102533340454, "tokens_per_second": 1438672.7605941035, "eta_seconds": 3439.805470395088, "data_wait_seconds": 2.2798776626586914e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 63050, "total_steps": 72479, "tokens": 33056358400, "loss": 2.946904331445694, "l_mix": 1.8254455626010895, "l_a": 1.8845673352479935, "l_c": 1.8536284267902374, "z_loss": 51.11490535736084, "lr": 0.002201700777529216, "wd": 0.01, "grad_norm": 0.21955035626888275, "tokens_per_second": 1430366.94511716, "eta_seconds": 3456.1142292022705, "data_wait_seconds": 2.1398067474365234e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 63060, "total_steps": 72479, "tokens": 33061601280, "loss": 4.0800133645534515, "l_mix": 2.5265796780586243, "l_a": 2.6145028173923492, "l_c": 2.5636093616485596, "z_loss": 44.23978137969971, "lr": 0.0022003518445458704, "wd": 0.01, "grad_norm": 0.12996351718902588, "tokens_per_second": 1424447.1908158006, "eta_seconds": 3466.796595788002, "data_wait_seconds": 2.142786979675293e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 63070, "total_steps": 72479, "tokens": 33066844160, "loss": 4.011565327644348, "l_mix": 2.4840284287929535, "l_a": 2.571570247411728, "l_c": 2.520219087600708, "z_loss": 43.90467548370361, "lr": 0.0021989941380165267, "wd": 0.01, "grad_norm": 0.13607463240623474, "tokens_per_second": 1396258.432417743, "eta_seconds": 3533.0320501327515, "data_wait_seconds": 1.6361474990844727e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 63080, "total_steps": 72479, "tokens": 33072087040, "loss": 3.9002837240695953, "l_mix": 2.414291948080063, "l_a": 2.5038110315799713, "l_c": 2.449494630098343, "z_loss": 43.50642395019531, "lr": 0.002197627669282052, "wd": 0.01, "grad_norm": 0.16539523005485535, "tokens_per_second": 1409512.2001970732, "eta_seconds": 3496.0909960985186, "data_wait_seconds": 2.7447938919067383e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 63090, "total_steps": 72479, "tokens": 33077329920, "loss": 3.9184335470199585, "l_mix": 2.424814820289612, "l_a": 2.5182247161865234, "l_c": 2.4605042040348053, "z_loss": 46.3199028968811, "lr": 0.0021962524497565034, "wd": 0.01, "grad_norm": 0.20833255350589752, "tokens_per_second": 1380671.7638293942, "eta_seconds": 3565.3224473476407, "data_wait_seconds": 2.4825334548950195e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 63100, "total_steps": 72479, "tokens": 33082572800, "loss": 3.856878101825714, "l_mix": 2.386415481567383, "l_a": 2.478628247976303, "l_c": 2.4229137897491455, "z_loss": 46.675639629364014, "lr": 0.0021948684909270334, "wd": 0.01, "grad_norm": 0.13792911171913147, "tokens_per_second": 1443959.3139722697, "eta_seconds": 3405.426388692856, "data_wait_seconds": 1.728534698486328e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.15955687, "gate_a2b/L1_absmax": 0.68750727, "gate_a2b/L1_rms": 0.16977739, "gate_a2b/L3_absmean": 0.15217452, "gate_a2b/L3_absmax": 0.50103247, "gate_a2b/L3_rms": 0.15788759, "gate_a2b/L5_absmean": 0.22107819, "gate_a2b/L5_absmax": 0.73185337, "gate_a2b/L5_rms": 0.22705819, "gate_c2b/L1_absmean": 0.03756827, "gate_c2b/L1_absmax": 0.6269061, "gate_c2b/L1_rms": 0.06988335, "gate_c2b/L3_absmean": 0.16715662, "gate_c2b/L3_absmax": 0.3885611, "gate_c2b/L3_rms": 0.17071745, "gate_c2b/L5_absmean": 0.4727881, "gate_c2b/L5_absmax": 0.66777432, "gate_c2b/L5_rms": 0.48206779, "gate_b2a/L7_absmean": 0.05127768, "gate_b2a/L7_absmax": 0.32736343, "gate_b2a/L7_rms": 0.0569782, "gate_b2a/L11_absmean": 0.08324672, "gate_b2a/L11_absmax": 0.60755795, "gate_b2a/L11_rms": 0.09227943, "gate_b2a/L14_absmean": 0.2162842, "gate_b2a/L14_absmax": 0.81155705, "gate_b2a/L14_rms": 0.22563852, "gate_b2c/L3_absmean": 0.03533975, "gate_b2c/L3_absmax": 0.54892302, "gate_b2c/L3_rms": 0.04571152, "gate_b2c/L5_absmean": 0.03719721, "gate_b2c/L5_absmax": 0.52067018, "gate_b2c/L5_rms": 0.04610035, "gate_b2c/L6_absmean": 0.0540956, "gate_b2c/L6_absmax": 0.34303802, "gate_b2c/L6_rms": 0.06006848}, "streams": {"cos_a2b/round0": 0.354847, "cos_c2b/round0": 0.245293, "cos_b2a/round0": 0.162458, "cos_b2c/round0": 0.146883, "rms_a/round0": 7.809617, "rms_b/round0": 12.953684, "rms_c/round0": 5.012526, "cos_a2b/round1": 0.418125, "cos_c2b/round1": 0.303113, "cos_b2a/round1": 0.251445, "cos_b2c/round1": 0.110262, "rms_a/round1": 15.263424, "rms_b/round1": 30.930382, "rms_c/round1": 10.54231, "cos_a2b/round2": 0.522258, "cos_c2b/round2": 0.530823, "cos_b2a/round2": 0.09628, "cos_b2c/round2": 0.179223, "rms_a/round2": 43.203648, "rms_b/round2": 85.035904, "rms_c/round2": 20.600817}, "alpha_mean": 0.4093691196666214, "alpha_hist": [0.02347, 0.11324, 0.19856, 0.18259, 0.16051, 0.13162, 0.10822, 0.06696, 0.0146, 0.00024], "alpha_by_type": {"special": {"share": 0.00076, "mean": 0.56361, "std": 0.11585}, "whitespace": {"share": 0.02515, "mean": 0.53474, "std": 0.12492}, "digit": {"share": 0.02849, "mean": 0.45782, "std": 0.11944}, "word_start": {"share": 0.60217, "mean": 0.41728, "std": 0.18362}, "word_cont": {"share": 0.23938, "mean": 0.29759, "std": 0.13893}, "punct": {"share": 0.10405, "mean": 0.57604, "std": 0.17369}}, "alpha_p05": 0.12592, "alpha_p50": 0.38815, "alpha_p95": 0.75851}
{"step": 63110, "total_steps": 72479, "tokens": 33087815680, "loss": 2.829987585544586, "l_mix": 1.7530884891748428, "l_a": 1.8130853027105331, "l_c": 1.7765781581401825, "z_loss": 51.325947761535645, "lr": 0.002193475804353794, "wd": 0.01, "grad_norm": 0.2453071027994156, "tokens_per_second": 1449428.628985524, "eta_seconds": 3388.959051704407, "data_wait_seconds": 2.7567148208618164e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 63120, "total_steps": 72479, "tokens": 33093058560, "loss": 4.514116883277893, "l_mix": 2.8000195622444153, "l_a": 2.8784346878528595, "l_c": 2.8352226614952087, "z_loss": 42.546491622924805, "lr": 0.002192074401669838, "wd": 0.01, "grad_norm": 0.11692412942647934, "tokens_per_second": 1396396.4814432047, "eta_seconds": 3513.9098796129224, "data_wait_seconds": 3.084540367126465e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 63130, "total_steps": 72479, "tokens": 33098301440, "loss": 4.551840841770172, "l_mix": 2.822605162858963, "l_a": 2.9010083973407745, "l_c": 2.8631102442741394, "z_loss": 41.602890968322754, "lr": 0.002190664294581026, "wd": 0.01, "grad_norm": 0.2794613838195801, "tokens_per_second": 1459891.1629466945, "eta_seconds": 3357.4889939785003, "data_wait_seconds": 2.7805566787719727e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 63140, "total_steps": 72479, "tokens": 33103544320, "loss": 4.5276495814323425, "l_mix": 2.808978319168091, "l_a": 2.886246472597122, "l_c": 2.8426574170589447, "z_loss": 41.68997812271118, "lr": 0.002189245494865924, "wd": 0.01, "grad_norm": 0.1997608244419098, "tokens_per_second": 1402019.8407312103, "eta_seconds": 3492.336905479431, "data_wait_seconds": 2.3096799850463867e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 63150, "total_steps": 72479, "tokens": 33108787200, "loss": 4.53951895236969, "l_mix": 2.8160833716392517, "l_a": 2.893275648355484, "l_c": 2.8515097200870514, "z_loss": 42.07703971862793, "lr": 0.0021878180143757088, "wd": 0.01, "grad_norm": 0.11757917702198029, "tokens_per_second": 1406480.4969793288, "eta_seconds": 3477.5332914352416, "data_wait_seconds": 2.4139881134033203e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 63160, "total_steps": 72479, "tokens": 33114030080, "loss": 4.569060921669006, "l_mix": 2.8347031474113464, "l_a": 2.909631460905075, "l_c": 2.8715609908103943, "z_loss": 41.028923988342285, "lr": 0.002186381865034066, "wd": 0.01, "grad_norm": 0.2154272496700287, "tokens_per_second": 1444152.2896349118, "eta_seconds": 3383.188813996315, "data_wait_seconds": 2.2590160369873047e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 63170, "total_steps": 72479, "tokens": 33119272960, "loss": 3.8605217039585114, "l_mix": 2.388477623462677, "l_a": 2.481692671775818, "l_c": 2.4251208007335663, "z_loss": 45.13748216629028, "lr": 0.0021849370588370924, "wd": 0.01, "grad_norm": 0.16640080511569977, "tokens_per_second": 1437778.6806365754, "eta_seconds": 3394.5398257255556, "data_wait_seconds": 1.5527009963989258e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 63180, "total_steps": 72479, "tokens": 33124515840, "loss": 3.8970096111297607, "l_mix": 2.4108942449092865, "l_a": 2.506676435470581, "l_c": 2.447041153907776, "z_loss": 45.53827953338623, "lr": 0.002183483607853195, "wd": 0.01, "grad_norm": 0.15022318065166473, "tokens_per_second": 1392946.9838531977, "eta_seconds": 3500.028478121758, "data_wait_seconds": 1.9997358322143555e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 63190, "total_steps": 72479, "tokens": 33129758720, "loss": 3.8474274575710297, "l_mix": 2.380331039428711, "l_a": 2.4741421341896057, "l_c": 2.4161790311336517, "z_loss": 45.1655330657959, "lr": 0.0021820215242229902, "wd": 0.01, "grad_norm": 0.11744540184736252, "tokens_per_second": 1356694.8422400837, "eta_seconds": 3589.688027381897, "data_wait_seconds": 2.0951032638549805e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 63200, "total_steps": 72479, "tokens": 33135001600, "loss": 3.886878550052643, "l_mix": 2.405689626932144, "l_a": 2.496795505285263, "l_c": 2.4405007660388947, "z_loss": 45.045278549194336, "lr": 0.0021805508201592034, "wd": 0.01, "grad_norm": 0.12727344036102295, "tokens_per_second": 1406459.6271380007, "eta_seconds": 3458.9463203430178, "data_wait_seconds": 1.5854835510253906e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16027862, "gate_a2b/L1_absmax": 0.6846745, "gate_a2b/L1_rms": 0.17025323, "gate_a2b/L3_absmean": 0.15239684, "gate_a2b/L3_absmax": 0.495671, "gate_a2b/L3_rms": 0.15800779, "gate_a2b/L5_absmean": 0.2211777, "gate_a2b/L5_absmax": 0.7302559, "gate_a2b/L5_rms": 0.22708754, "gate_c2b/L1_absmean": 0.03816151, "gate_c2b/L1_absmax": 0.62460643, "gate_c2b/L1_rms": 0.07035685, "gate_c2b/L3_absmean": 0.16731238, "gate_c2b/L3_absmax": 0.38473892, "gate_c2b/L3_rms": 0.17088825, "gate_c2b/L5_absmean": 0.47209913, "gate_c2b/L5_absmax": 0.66754061, "gate_c2b/L5_rms": 0.48130956, "gate_b2a/L7_absmean": 0.05117713, "gate_b2a/L7_absmax": 0.33179894, "gate_b2a/L7_rms": 0.0569273, "gate_b2a/L11_absmean": 0.08320048, "gate_b2a/L11_absmax": 0.61329675, "gate_b2a/L11_rms": 0.09235553, "gate_b2a/L14_absmean": 0.21646447, "gate_b2a/L14_absmax": 0.81297678, "gate_b2a/L14_rms": 0.22575831, "gate_b2c/L3_absmean": 0.03536013, "gate_b2c/L3_absmax": 0.54970431, "gate_b2c/L3_rms": 0.04585022, "gate_b2c/L5_absmean": 0.03723364, "gate_b2c/L5_absmax": 0.52304637, "gate_b2c/L5_rms": 0.04616138, "gate_b2c/L6_absmean": 0.05390219, "gate_b2c/L6_absmax": 0.34266457, "gate_b2c/L6_rms": 0.05985674}, "streams": {"cos_a2b/round0": 0.354198, "cos_c2b/round0": 0.24326, "cos_b2a/round0": 0.162306, "cos_b2c/round0": 0.146143, "rms_a/round0": 7.776454, "rms_b/round0": 13.082774, "rms_c/round0": 5.054926, "cos_a2b/round1": 0.41763, "cos_c2b/round1": 0.306616, "cos_b2a/round1": 0.253754, "cos_b2c/round1": 0.114081, "rms_a/round1": 15.233169, "rms_b/round1": 30.861807, "rms_c/round1": 10.632205, "cos_a2b/round2": 0.528441, "cos_c2b/round2": 0.528752, "cos_b2a/round2": 0.094972, "cos_b2c/round2": 0.183074, "rms_a/round2": 43.34565, "rms_b/round2": 84.352814, "rms_c/round2": 20.587202}, "alpha_mean": 0.4137992187773136, "alpha_hist": [0.01276, 0.10136, 0.21271, 0.19121, 0.15627, 0.13615, 0.1088, 0.06808, 0.01245, 0.00021], "alpha_by_type": {"special": {"share": 0.00067, "mean": 0.53886, "std": 0.12404}, "whitespace": {"share": 0.0251, "mean": 0.52134, "std": 0.11305}, "digit": {"share": 0.01781, "mean": 0.40016, "std": 0.11686}, "word_start": {"share": 0.58972, "mean": 0.42818, "std": 0.17923}, "word_cont": {"share": 0.26799, "mean": 0.30705, "std": 0.1295}, "punct": {"share": 0.09871, "mean": 0.59195, "std": 0.1643}}, "alpha_p05": 0.15254, "alpha_p50": 0.39001, "alpha_p95": 0.72409}
{"step": 63210, "total_steps": 72479, "tokens": 33140244480, "loss": 3.8890266120433807, "l_mix": 2.405900329351425, "l_a": 2.5015109479427338, "l_c": 2.4422431886196136, "z_loss": 45.89203977584839, "lr": 0.002179071507946565, "wd": 0.01, "grad_norm": 0.1492304801940918, "tokens_per_second": 1432645.3725861418, "eta_seconds": 3392.0644738674164, "data_wait_seconds": 1.8149614334106445e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 63220, "total_steps": 72479, "tokens": 33145487360, "loss": 2.673939883708954, "l_mix": 1.6552649587392807, "l_a": 1.7171755582094193, "l_c": 1.6784073412418365, "z_loss": 54.45070171356201, "lr": 0.00217758359994171, "wd": 0.01, "grad_norm": 0.17816731333732605, "tokens_per_second": 1389521.774221718, "eta_seconds": 3493.5635281562804, "data_wait_seconds": 2.5570392608642578e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 63230, "total_steps": 72479, "tokens": 33150730240, "loss": 2.9160094559192657, "l_mix": 1.8049670904874802, "l_a": 1.8700238913297653, "l_c": 1.8334505409002304, "z_loss": 52.46027660369873, "lr": 0.0021760871085730717, "wd": 0.01, "grad_norm": 0.18527522683143616, "tokens_per_second": 1381855.1800621834, "eta_seconds": 3509.151886510849, "data_wait_seconds": 1.6868114471435547e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 63240, "total_steps": 72479, "tokens": 33155973120, "loss": 4.524617254734039, "l_mix": 2.806933730840683, "l_a": 2.883009970188141, "l_c": 2.842601388692856, "z_loss": 41.63823080062866, "lr": 0.0021745820463407828, "wd": 0.01, "grad_norm": 0.1418958306312561, "tokens_per_second": 1351148.430565837, "eta_seconds": 3585.0219875335692, "data_wait_seconds": 2.6285648345947266e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 63250, "total_steps": 72479, "tokens": 33161216000, "loss": 4.438197195529938, "l_mix": 2.753018796443939, "l_a": 2.830355077981949, "l_c": 2.786905884742737, "z_loss": 41.79492473602295, "lr": 0.0021730684258165663, "wd": 0.01, "grad_norm": 0.1633932739496231, "tokens_per_second": 1413354.117722219, "eta_seconds": 3423.525563287735, "data_wait_seconds": 4.798173904418945e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 63260, "total_steps": 72479, "tokens": 33166458880, "loss": 4.660149037837982, "l_mix": 2.891720622777939, "l_a": 2.9655042588710785, "l_c": 2.929256945848465, "z_loss": 40.13440799713135, "lr": 0.0021715462596436316, "wd": 0.01, "grad_norm": 0.2454197257757187, "tokens_per_second": 1447791.381427897, "eta_seconds": 3338.4720575094225, "data_wait_seconds": 3.203749656677246e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 63270, "total_steps": 72479, "tokens": 33171701760, "loss": 4.563870012760162, "l_mix": 2.831361323595047, "l_a": 2.9083824157714844, "l_c": 2.866646498441696, "z_loss": 41.90651512145996, "lr": 0.0021700155605365716, "wd": 0.01, "grad_norm": 0.17039701342582703, "tokens_per_second": 1384229.7645732705, "eta_seconds": 3487.981775546074, "data_wait_seconds": 1.7911195755004883e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 63280, "total_steps": 72479, "tokens": 33176944640, "loss": 3.6704167425632477, "l_mix": 2.269087076187134, "l_a": 2.3641090095043182, "l_c": 2.3069896697998047, "z_loss": 47.28880071640015, "lr": 0.002168476341281253, "wd": 0.01, "grad_norm": 0.1835779845714569, "tokens_per_second": 1421616.248450639, "eta_seconds": 3392.5648481130597, "data_wait_seconds": 1.7315149307250977e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 63290, "total_steps": 72479, "tokens": 33182187520, "loss": 3.845831513404846, "l_mix": 2.3792333006858826, "l_a": 2.4750384986400604, "l_c": 2.4136220812797546, "z_loss": 45.93032455444336, "lr": 0.002166928614734711, "wd": 0.01, "grad_norm": 0.13778087496757507, "tokens_per_second": 1445163.338238118, "eta_seconds": 3333.659458780289, "data_wait_seconds": 1.7344951629638672e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 63300, "total_steps": 72479, "tokens": 33187430400, "loss": 3.7882737815380096, "l_mix": 2.3431389927864075, "l_a": 2.4375076591968536, "l_c": 2.3796081244945526, "z_loss": 46.13598680496216, "lr": 0.002165372393825042, "wd": 0.01, "grad_norm": 0.13830778002738953, "tokens_per_second": 1427587.1419786941, "eta_seconds": 3371.0303283691405, "data_wait_seconds": 1.8149614334106445e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16053347, "gate_a2b/L1_absmax": 0.68483037, "gate_a2b/L1_rms": 0.17049468, "gate_a2b/L3_absmean": 0.15288126, "gate_a2b/L3_absmax": 0.49743286, "gate_a2b/L3_rms": 0.15844616, "gate_a2b/L5_absmean": 0.2206486, "gate_a2b/L5_absmax": 0.72913498, "gate_a2b/L5_rms": 0.22658359, "gate_c2b/L1_absmean": 0.03824623, "gate_c2b/L1_absmax": 0.62494057, "gate_c2b/L1_rms": 0.07036809, "gate_c2b/L3_absmean": 0.16717149, "gate_c2b/L3_absmax": 0.38453484, "gate_c2b/L3_rms": 0.17076543, "gate_c2b/L5_absmean": 0.4720411, "gate_c2b/L5_absmax": 0.66763985, "gate_c2b/L5_rms": 0.48128632, "gate_b2a/L7_absmean": 0.05128258, "gate_b2a/L7_absmax": 0.33312312, "gate_b2a/L7_rms": 0.05706224, "gate_b2a/L11_absmean": 0.0830835, "gate_b2a/L11_absmax": 0.61455911, "gate_b2a/L11_rms": 0.09224801, "gate_b2a/L14_absmean": 0.21660599, "gate_b2a/L14_absmax": 0.81442004, "gate_b2a/L14_rms": 0.22591697, "gate_b2c/L3_absmean": 0.03541762, "gate_b2c/L3_absmax": 0.55099344, "gate_b2c/L3_rms": 0.04592099, "gate_b2c/L5_absmean": 0.03720331, "gate_b2c/L5_absmax": 0.52645046, "gate_b2c/L5_rms": 0.04627312, "gate_b2c/L6_absmean": 0.05382591, "gate_b2c/L6_absmax": 0.34207267, "gate_b2c/L6_rms": 0.05973297}, "streams": {"cos_a2b/round0": 0.35596, "cos_c2b/round0": 0.24417, "cos_b2a/round0": 0.16578, "cos_b2c/round0": 0.146536, "rms_a/round0": 7.808493, "rms_b/round0": 13.157717, "rms_c/round0": 5.019863, "cos_a2b/round1": 0.422204, "cos_c2b/round1": 0.309394, "cos_b2a/round1": 0.25345, "cos_b2c/round1": 0.116102, "rms_a/round1": 15.525015, "rms_b/round1": 31.318653, "rms_c/round1": 10.819332, "cos_a2b/round2": 0.527171, "cos_c2b/round2": 0.529552, "cos_b2a/round2": 0.098053, "cos_b2c/round2": 0.182306, "rms_a/round2": 43.966011, "rms_b/round2": 85.118149, "rms_c/round2": 21.277128}, "alpha_mean": 0.41384964854965745, "alpha_hist": [0.01773, 0.11604, 0.20941, 0.18193, 0.14215, 0.12386, 0.11005, 0.08096, 0.01756, 0.00031], "alpha_by_type": {"special": {"share": 0.00105, "mean": 0.56982, "std": 0.11962}, "whitespace": {"share": 0.02332, "mean": 0.5267, "std": 0.11343}, "digit": {"share": 0.01138, "mean": 0.42946, "std": 0.12072}, "word_start": {"share": 0.61215, "mean": 0.42557, "std": 0.18918}, "word_cont": {"share": 0.25269, "mean": 0.30073, "std": 0.13536}, "punct": {"share": 0.09941, "mean": 0.59928, "std": 0.17368}}, "alpha_p05": 0.1401, "alpha_p50": 0.39094, "alpha_p95": 0.75092}
{"step": 63310, "total_steps": 72479, "tokens": 33192673280, "loss": 3.8685237169265747, "l_mix": 2.394346743822098, "l_a": 2.4836227893829346, "l_c": 2.4303003549575806, "z_loss": 44.36363887786865, "lr": 0.0021638076915512965, "wd": 0.01, "grad_norm": 0.18417517840862274, "tokens_per_second": 1387070.304830549, "eta_seconds": 3465.719549512863, "data_wait_seconds": 1.7464160919189453e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 63320, "total_steps": 72479, "tokens": 33197916160, "loss": 3.8238061368465424, "l_mix": 2.364631235599518, "l_a": 2.4641694128513336, "l_c": 2.3997468054294586, "z_loss": 46.9182448387146, "lr": 0.002162234520983368, "wd": 0.01, "grad_norm": 0.15885451436042786, "tokens_per_second": 1395558.8542018158, "eta_seconds": 3440.882322907448, "data_wait_seconds": 1.8209218978881836e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 63330, "total_steps": 72479, "tokens": 33203159040, "loss": 3.1066021025180817, "l_mix": 1.9235150068998337, "l_a": 1.9907644838094711, "l_c": 1.9528590887784958, "z_loss": 53.74243116378784, "lr": 0.0021606528952618853, "wd": 0.01, "grad_norm": 0.15874485671520233, "tokens_per_second": 1463055.0605548027, "eta_seconds": 3278.558026504517, "data_wait_seconds": 2.1249055862426758e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 63340, "total_steps": 72479, "tokens": 33208401920, "loss": 2.778738886117935, "l_mix": 1.7187601029872894, "l_a": 1.7915741950273514, "l_c": 1.7416882812976837, "z_loss": 53.70052528381348, "lr": 0.0021590628275981044, "wd": 0.01, "grad_norm": 0.15706075727939606, "tokens_per_second": 1375403.9811188239, "eta_seconds": 3483.6805024385453, "data_wait_seconds": 3.6150217056274414e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 63350, "total_steps": 72479, "tokens": 33213644800, "loss": 4.517970979213715, "l_mix": 2.802575409412384, "l_a": 2.881176143884659, "l_c": 2.836808830499649, "z_loss": 42.26833915710449, "lr": 0.002157464331273795, "wd": 0.01, "grad_norm": 0.10857494175434113, "tokens_per_second": 1332252.4707157465, "eta_seconds": 3592.581179022789, "data_wait_seconds": 1.8656253814697266e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 63360, "total_steps": 72479, "tokens": 33218887680, "loss": 3.9150587022304535, "l_mix": 2.4227525293827057, "l_a": 2.5128336548805237, "l_c": 2.4615199267864227, "z_loss": 46.43488121032715, "lr": 0.0021558574196411315, "wd": 0.01, "grad_norm": 0.1557193100452423, "tokens_per_second": 1396081.6773118621, "eta_seconds": 3424.5720359325405, "data_wait_seconds": 2.2798776626586914e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 63370, "total_steps": 72479, "tokens": 33224130560, "loss": 3.8131855130195618, "l_mix": 2.3594906628131866, "l_a": 2.4526573717594147, "l_c": 2.3929921090602875, "z_loss": 46.41834497451782, "lr": 0.002154242106122581, "wd": 0.01, "grad_norm": 0.16528579592704773, "tokens_per_second": 1432705.9499876895, "eta_seconds": 3333.370250916481, "data_wait_seconds": 2.7179718017578125e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 63380, "total_steps": 72479, "tokens": 33229373440, "loss": 3.7903770804405212, "l_mix": 2.3450019657611847, "l_a": 2.4358644485473633, "l_c": 2.3820524513721466, "z_loss": 45.35863971710205, "lr": 0.002152618404210792, "wd": 0.01, "grad_norm": 0.20958487689495087, "tokens_per_second": 1376342.9955378543, "eta_seconds": 3466.0666181802753, "data_wait_seconds": 2.8908252716064453e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 63390, "total_steps": 72479, "tokens": 33234616320, "loss": 3.903089165687561, "l_mix": 2.4160087406635284, "l_a": 2.507410615682602, "l_c": 2.4495240449905396, "z_loss": 44.630964279174805, "lr": 0.0021509863274684787, "wd": 0.01, "grad_norm": 0.17946180701255798, "tokens_per_second": 1240691.2498576802, "eta_seconds": 3840.805383729935, "data_wait_seconds": 0.04894557595252991, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 63400, "total_steps": 72479, "tokens": 33239859200, "loss": 3.9032068252563477, "l_mix": 2.4160618484020233, "l_a": 2.50622296333313, "l_c": 2.450926959514618, "z_loss": 46.04965114593506, "lr": 0.0021493458895283125, "wd": 0.01, "grad_norm": 0.16148823499679565, "tokens_per_second": 1444873.2513634327, "eta_seconds": 3294.414058470726, "data_wait_seconds": 1.537799835205078e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.16021939, "gate_a2b/L1_absmax": 0.68383896, "gate_a2b/L1_rms": 0.17015938, "gate_a2b/L3_absmean": 0.15245219, "gate_a2b/L3_absmax": 0.49559009, "gate_a2b/L3_rms": 0.158051, "gate_a2b/L5_absmean": 0.22064982, "gate_a2b/L5_absmax": 0.73013377, "gate_a2b/L5_rms": 0.2266551, "gate_c2b/L1_absmean": 0.03856563, "gate_c2b/L1_absmax": 0.62479359, "gate_c2b/L1_rms": 0.07040247, "gate_c2b/L3_absmean": 0.1670662, "gate_c2b/L3_absmax": 0.38491857, "gate_c2b/L3_rms": 0.17063157, "gate_c2b/L5_absmean": 0.47183806, "gate_c2b/L5_absmax": 0.66734129, "gate_c2b/L5_rms": 0.48107198, "gate_b2a/L7_absmean": 0.05139321, "gate_b2a/L7_absmax": 0.33172673, "gate_b2a/L7_rms": 0.05719925, "gate_b2a/L11_absmean": 0.08318423, "gate_b2a/L11_absmax": 0.61343336, "gate_b2a/L11_rms": 0.09231994, "gate_b2a/L14_absmean": 0.21655662, "gate_b2a/L14_absmax": 0.8127777, "gate_b2a/L14_rms": 0.22584191, "gate_b2c/L3_absmean": 0.03551461, "gate_b2c/L3_absmax": 0.55162311, "gate_b2c/L3_rms": 0.04592889, "gate_b2c/L5_absmean": 0.0373769, "gate_b2c/L5_absmax": 0.5267604, "gate_b2c/L5_rms": 0.04641918, "gate_b2c/L6_absmean": 0.05374515, "gate_b2c/L6_absmax": 0.34197763, "gate_b2c/L6_rms": 0.05964445}, "streams": {"cos_a2b/round0": 0.356128, "cos_c2b/round0": 0.244545, "cos_b2a/round0": 0.161385, "cos_b2c/round0": 0.146826, "rms_a/round0": 7.79417, "rms_b/round0": 13.122932, "rms_c/round0": 5.046437, "cos_a2b/round1": 0.413644, "cos_c2b/round1": 0.303187, "cos_b2a/round1": 0.251683, "cos_b2c/round1": 0.112315, "rms_a/round1": 15.312374, "rms_b/round1": 31.031824, "rms_c/round1": 10.81366, "cos_a2b/round2": 0.527558, "cos_c2b/round2": 0.52295, "cos_b2a/round2": 0.094368, "cos_b2c/round2": 0.182321, "rms_a/round2": 43.736187, "rms_b/round2": 84.659088, "rms_c/round2": 21.014748}, "alpha_mean": 0.4119858783977577, "alpha_hist": [0.01349, 0.10254, 0.20293, 0.20168, 0.16203, 0.13416, 0.10689, 0.06398, 0.01212, 0.0002], "alpha_by_type": {"special": {"share": 0.00089, "mean": 0.55034, "std": 0.1134}, "whitespace": {"share": 0.02443, "mean": 0.52478, "std": 0.12223}, "digit": {"share": 0.0332, "mean": 0.4716, "std": 0.11436}, "word_start": {"share": 0.57445, "mean": 0.41874, "std": 0.18042}, "word_cont": {"share": 0.26111, "mean": 0.31115, "std": 0.12713}, "punct": {"share": 0.10593, "mean": 0.57803, "std": 0.16384}}, "alpha_p05": 0.16345, "alpha_p50": 0.39397, "alpha_p95": 0.73564}
{"step": 63410, "total_steps": 72479, "tokens": 33245102080, "loss": 4.012968361377716, "l_mix": 2.484744071960449, "l_a": 2.5733512341976166, "l_c": 2.5207299292087555, "z_loss": 44.825663566589355, "lr": 0.0021476971040928043, "wd": 0.01, "grad_norm": 0.11497750878334045, "tokens_per_second": 1424204.9296865351, "eta_seconds": 3338.5419281244276, "data_wait_seconds": 1.913309097290039e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 63420, "total_steps": 72479, "tokens": 33250344960, "loss": 3.802344560623169, "l_mix": 2.3527328968048096, "l_a": 2.4432753920555115, "l_c": 2.3887633085250854, "z_loss": 45.04840564727783, "lr": 0.002146039984934191, "wd": 0.01, "grad_norm": 0.11716213822364807, "tokens_per_second": 1408238.2124388448, "eta_seconds": 3372.6715764760975, "data_wait_seconds": 3.1620264053344727e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 63430, "total_steps": 72479, "tokens": 33255587840, "loss": 3.922911912202835, "l_mix": 2.428543210029602, "l_a": 2.517896831035614, "l_c": 2.4633319973945618, "z_loss": 44.9668869972229, "lr": 0.0021443745458943216, "wd": 0.01, "grad_norm": 0.11995765566825867, "tokens_per_second": 1341157.8232415374, "eta_seconds": 3537.45251288414, "data_wait_seconds": 2.8580427169799805e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 63440, "total_steps": 72479, "tokens": 33260830720, "loss": 2.8175223767757416, "l_mix": 1.744610145688057, "l_a": 1.80666384100914, "l_c": 1.7697101086378098, "z_loss": 53.16240835189819, "lr": 0.00214270080088454, "wd": 0.01, "grad_norm": 0.20193755626678467, "tokens_per_second": 1412020.2267684354, "eta_seconds": 3356.2120018959045, "data_wait_seconds": 3.2454729080200195e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 63450, "total_steps": 72479, "tokens": 33266073600, "loss": 2.9194331765174866, "l_mix": 1.8089193999767303, "l_a": 1.8664443492889404, "l_c": 1.8352679461240768, "z_loss": 51.60768795013428, "lr": 0.0021410187638855696, "wd": 0.01, "grad_norm": 0.22487662732601166, "tokens_per_second": 1342837.6341759372, "eta_seconds": 3525.2187096357347, "data_wait_seconds": 1.8209218978881836e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 63460, "total_steps": 72479, "tokens": 33271316480, "loss": 3.719782739877701, "l_mix": 2.3009888529777527, "l_a": 2.3946148455142975, "l_c": 2.3346977531909943, "z_loss": 46.49206590652466, "lr": 0.0021393284489473977, "wd": 0.01, "grad_norm": 0.18133684992790222, "tokens_per_second": 1385945.1119556387, "eta_seconds": 3411.789854598045, "data_wait_seconds": 2.6166439056396484e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 63470, "total_steps": 72479, "tokens": 33276559360, "loss": 3.7427059412002563, "l_mix": 2.3149904906749725, "l_a": 2.4075966477394104, "l_c": 2.3514545559883118, "z_loss": 46.014774799346924, "lr": 0.0021376298701891545, "wd": 0.01, "grad_norm": 0.11736410856246948, "tokens_per_second": 1416149.299294934, "eta_seconds": 3335.3196547508237, "data_wait_seconds": 2.3365020751953125e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 63480, "total_steps": 72479, "tokens": 33281802240, "loss": 3.730056971311569, "l_mix": 2.306679129600525, "l_a": 2.4034196734428406, "l_c": 2.3411730527877808, "z_loss": 46.0787296295166, "lr": 0.0021359230417989992, "wd": 0.01, "grad_norm": 0.14024266600608826, "tokens_per_second": 1404382.3133693675, "eta_seconds": 3359.532277703285, "data_wait_seconds": 1.609325408935547e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 63490, "total_steps": 72479, "tokens": 33287045120, "loss": 3.8129683136940002, "l_mix": 2.3590491712093353, "l_a": 2.453193575143814, "l_c": 2.393203228712082, "z_loss": 44.55720853805542, "lr": 0.0021342079780340005, "wd": 0.01, "grad_norm": 0.13862459361553192, "tokens_per_second": 1416281.0935635373, "eta_seconds": 3327.60555331707, "data_wait_seconds": 2.637505531311035e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 63500, "total_steps": 72479, "tokens": 33292288000, "loss": 3.8572769463062286, "l_mix": 2.386486440896988, "l_a": 2.4791336059570312, "l_c": 2.4235014617443085, "z_loss": 46.11889457702637, "lr": 0.002132484693220015, "wd": 0.01, "grad_norm": 0.13371694087982178, "tokens_per_second": 1315957.6432195895, "eta_seconds": 3577.3050722837447, "data_wait_seconds": 1.800060272216797e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.1607426, "gate_a2b/L1_absmax": 0.68360841, "gate_a2b/L1_rms": 0.1706209, "gate_a2b/L3_absmean": 0.15277119, "gate_a2b/L3_absmax": 0.49496749, "gate_a2b/L3_rms": 0.15832938, "gate_a2b/L5_absmean": 0.22059408, "gate_a2b/L5_absmax": 0.72980332, "gate_a2b/L5_rms": 0.22656845, "gate_c2b/L1_absmean": 0.03845543, "gate_c2b/L1_absmax": 0.62453395, "gate_c2b/L1_rms": 0.07036266, "gate_c2b/L3_absmean": 0.1667773, "gate_c2b/L3_absmax": 0.38441983, "gate_c2b/L3_rms": 0.1703044, "gate_c2b/L5_absmean": 0.47183838, "gate_c2b/L5_absmax": 0.66706628, "gate_c2b/L5_rms": 0.48106232, "gate_b2a/L7_absmean": 0.05109758, "gate_b2a/L7_absmax": 0.33196512, "gate_b2a/L7_rms": 0.05688454, "gate_b2a/L11_absmean": 0.08308516, "gate_b2a/L11_absmax": 0.61348969, "gate_b2a/L11_rms": 0.09223595, "gate_b2a/L14_absmean": 0.21666355, "gate_b2a/L14_absmax": 0.8143996, "gate_b2a/L14_rms": 0.22595313, "gate_b2c/L3_absmean": 0.03546144, "gate_b2c/L3_absmax": 0.55114472, "gate_b2c/L3_rms": 0.04592146, "gate_b2c/L5_absmean": 0.03749444, "gate_b2c/L5_absmax": 0.52799886, "gate_b2c/L5_rms": 0.04652652, "gate_b2c/L6_absmean": 0.05379572, "gate_b2c/L6_absmax": 0.34056491, "gate_b2c/L6_rms": 0.05966919}, "streams": {"cos_a2b/round0": 0.357056, "cos_c2b/round0": 0.244477, "cos_b2a/round0": 0.169096, "cos_b2c/round0": 0.14935, "rms_a/round0": 7.718905, "rms_b/round0": 13.078619, "rms_c/round0": 4.996753, "cos_a2b/round1": 0.420256, "cos_c2b/round1": 0.307892, "cos_b2a/round1": 0.254479, "cos_b2c/round1": 0.11593, "rms_a/round1": 15.318589, "rms_b/round1": 31.097628, "rms_c/round1": 10.761226, "cos_a2b/round2": 0.53108, "cos_c2b/round2": 0.527854, "cos_b2a/round2": 0.093838, "cos_b2c/round2": 0.185402, "rms_a/round2": 43.866337, "rms_b/round2": 84.676781, "rms_c/round2": 21.040705}, "alpha_mean": 0.41066077073154617, "alpha_hist": [0.01442, 0.11079, 0.19978, 0.1926, 0.16356, 0.13483, 0.10953, 0.06503, 0.00941, 5e-05], "alpha_by_type": {"special": {"share": 0.00095, "mean": 0.51913, "std": 0.11191}, "whitespace": {"share": 0.02356, "mean": 0.51812, "std": 0.10889}, "digit": {"share": 0.01892, "mean": 0.44413, "std": 0.11308}, "word_start": {"share": 0.62207, "mean": 0.41692, "std": 0.17898}, "word_cont": {"share": 0.23833, "mean": 0.3103, "std": 0.12952}, "punct": {"share": 0.09618, "mean": 0.5849, "std": 0.16984}}, "alpha_p05": 0.1401, "alpha_p50": 0.39724, "alpha_p95": 0.72797}
{"step": 63510, "total_steps": 72479, "tokens": 33297530880, "loss": 3.8888419568538666, "l_mix": 2.4055675268173218, "l_a": 2.499604970216751, "l_c": 2.4446429014205933, "z_loss": 47.20127868652344, "lr": 0.0021307532017515703, "wd": 0.01, "grad_norm": 0.2037528157234192, "tokens_per_second": 1377511.147168377, "eta_seconds": 3413.6486529827116, "data_wait_seconds": 2.3663043975830078e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 63520, "total_steps": 72479, "tokens": 33302773760, "loss": 4.562902748584747, "l_mix": 2.8308947384357452, "l_a": 2.90890634059906, "l_c": 2.8644535541534424, "z_loss": 41.951913833618164, "lr": 0.002129013518091744, "wd": 0.01, "grad_norm": 0.10624513030052185, "tokens_per_second": 1374094.7055247459, "eta_seconds": 3418.3205663442614, "data_wait_seconds": 1.8209218978881836e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 63530, "total_steps": 72479, "tokens": 33308016640, "loss": 4.577963829040527, "l_mix": 2.8400606513023376, "l_a": 2.9182943403720856, "l_c": 2.8747163116931915, "z_loss": 41.30250406265259, "lr": 0.0021272656567720427, "wd": 0.01, "grad_norm": 0.1136651411652565, "tokens_per_second": 1449811.8268002826, "eta_seconds": 3236.1808789730076, "data_wait_seconds": 2.1457672119140625e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 63540, "total_steps": 72479, "tokens": 33313259520, "loss": 4.672865390777588, "l_mix": 2.8995524644851685, "l_a": 2.974547564983368, "l_c": 2.936494827270508, "z_loss": 41.53209114074707, "lr": 0.0021255096323922803, "wd": 0.01, "grad_norm": 0.13772600889205933, "tokens_per_second": 1381613.0394442896, "eta_seconds": 3392.129560303688, "data_wait_seconds": 1.8298625946044922e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 63550, "total_steps": 72479, "tokens": 33318502400, "loss": 4.553980350494385, "l_mix": 2.8243328332901, "l_a": 2.9053222835063934, "l_c": 2.86016908288002, "z_loss": 42.222206592559814, "lr": 0.0021237454596204565, "wd": 0.01, "grad_norm": 0.15133164823055267, "tokens_per_second": 1435954.5608131427, "eta_seconds": 3260.1084182977675, "data_wait_seconds": 2.512335777282715e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 63560, "total_steps": 72479, "tokens": 33323745280, "loss": 2.787917733192444, "l_mix": 1.7263197898864746, "l_a": 1.7872665226459503, "l_c": 1.751393049955368, "z_loss": 54.07106876373291, "lr": 0.0021219731531926344, "wd": 0.01, "grad_norm": 0.24980391561985016, "tokens_per_second": 1435436.776827713, "eta_seconds": 3257.631926035881, "data_wait_seconds": 2.390146255493164e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 63570, "total_steps": 72479, "tokens": 33328988160, "loss": 4.04021555185318, "l_mix": 2.5024326145648956, "l_a": 2.5878545939922333, "l_c": 2.538088381290436, "z_loss": 45.166969776153564, "lr": 0.002120192727912818, "wd": 0.01, "grad_norm": 0.11387435346841812, "tokens_per_second": 1298064.785646849, "eta_seconds": 3598.342581701279, "data_wait_seconds": 2.580881118774414e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 63580, "total_steps": 72479, "tokens": 33334231040, "loss": 3.903247684240341, "l_mix": 2.415508061647415, "l_a": 2.5076173543930054, "l_c": 2.451514571905136, "z_loss": 45.38086986541748, "lr": 0.002118404198652825, "wd": 0.01, "grad_norm": 0.13012807071208954, "tokens_per_second": 1407431.5411173992, "eta_seconds": 3315.002382493019, "data_wait_seconds": 1.8775463104248047e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 63590, "total_steps": 72479, "tokens": 33339473920, "loss": 3.7214417159557343, "l_mix": 2.3019003570079803, "l_a": 2.3957319855690002, "l_c": 2.3360724449157715, "z_loss": 46.97980785369873, "lr": 0.002116607580352169, "wd": 0.01, "grad_norm": 0.16477003693580627, "tokens_per_second": 1388341.499072239, "eta_seconds": 3356.808130502701, "data_wait_seconds": 1.537799835205078e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 63600, "total_steps": 72479, "tokens": 33344716800, "loss": 3.8311562538146973, "l_mix": 2.3703438341617584, "l_a": 2.4627625644207, "l_c": 2.4066118597984314, "z_loss": 45.817994594573975, "lr": 0.002114802888017928, "wd": 0.01, "grad_norm": 0.1477261483669281, "tokens_per_second": 1452529.4456455887, "eta_seconds": 3204.859747219086, "data_wait_seconds": 1.4513731002807617e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16016255, "gate_a2b/L1_absmax": 0.68353432, "gate_a2b/L1_rms": 0.17010532, "gate_a2b/L3_absmean": 0.15298562, "gate_a2b/L3_absmax": 0.49574101, "gate_a2b/L3_rms": 0.15854186, "gate_a2b/L5_absmean": 0.22076538, "gate_a2b/L5_absmax": 0.72886819, "gate_a2b/L5_rms": 0.22672959, "gate_c2b/L1_absmean": 0.0386981, "gate_c2b/L1_absmax": 0.62499255, "gate_c2b/L1_rms": 0.07014359, "gate_c2b/L3_absmean": 0.16644385, "gate_c2b/L3_absmax": 0.38451552, "gate_c2b/L3_rms": 0.16994227, "gate_c2b/L5_absmean": 0.47099254, "gate_c2b/L5_absmax": 0.66733807, "gate_c2b/L5_rms": 0.48015687, "gate_b2a/L7_absmean": 0.05119212, "gate_b2a/L7_absmax": 0.33112782, "gate_b2a/L7_rms": 0.05695859, "gate_b2a/L11_absmean": 0.08329225, "gate_b2a/L11_absmax": 0.61424857, "gate_b2a/L11_rms": 0.09245532, "gate_b2a/L14_absmean": 0.21650356, "gate_b2a/L14_absmax": 0.81326729, "gate_b2a/L14_rms": 0.22577475, "gate_b2c/L3_absmean": 0.0351868, "gate_b2c/L3_absmax": 0.55121869, "gate_b2c/L3_rms": 0.04573287, "gate_b2c/L5_absmean": 0.03755318, "gate_b2c/L5_absmax": 0.52761871, "gate_b2c/L5_rms": 0.04657644, "gate_b2c/L6_absmean": 0.05397107, "gate_b2c/L6_absmax": 0.34150967, "gate_b2c/L6_rms": 0.05982975}, "streams": {"cos_a2b/round0": 0.354673, "cos_c2b/round0": 0.242304, "cos_b2a/round0": 0.164175, "cos_b2c/round0": 0.144479, "rms_a/round0": 7.723588, "rms_b/round0": 13.069562, "rms_c/round0": 5.055307, "cos_a2b/round1": 0.41784, "cos_c2b/round1": 0.300463, "cos_b2a/round1": 0.251131, "cos_b2c/round1": 0.11311, "rms_a/round1": 15.30645, "rms_b/round1": 31.339727, "rms_c/round1": 10.752471, "cos_a2b/round2": 0.519962, "cos_c2b/round2": 0.528276, "cos_b2a/round2": 0.09464, "cos_b2c/round2": 0.181878, "rms_a/round2": 43.761066, "rms_b/round2": 85.985916, "rms_c/round2": 21.313328}, "alpha_mean": 0.40804305491437276, "alpha_hist": [0.01695, 0.12454, 0.20531, 0.18594, 0.15295, 0.12279, 0.10008, 0.07271, 0.0184, 0.00032], "alpha_by_type": {"special": {"share": 0.00101, "mean": 0.56191, "std": 0.09917}, "whitespace": {"share": 0.02197, "mean": 0.53557, "std": 0.11879}, "digit": {"share": 0.0228, "mean": 0.43379, "std": 0.11794}, "word_start": {"share": 0.59607, "mean": 0.41288, "std": 0.1906}, "word_cont": {"share": 0.25716, "mean": 0.31103, "std": 0.13181}, "punct": {"share": 0.101, "mean": 0.59141, "std": 0.17766}}, "alpha_p05": 0.13523, "alpha_p50": 0.3863, "alpha_p95": 0.75528}
{"step": 63610, "total_steps": 72479, "tokens": 33349959680, "loss": 3.89957195520401, "l_mix": 2.4131194949150085, "l_a": 2.5059314966201782, "l_c": 2.448910176753998, "z_loss": 46.38828134536743, "lr": 0.0021129901367246237, "wd": 0.01, "grad_norm": 0.14284886419773102, "tokens_per_second": 1323580.6646784476, "eta_seconds": 3513.1294949293138, "data_wait_seconds": 1.710653305053711e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 63620, "total_steps": 72479, "tokens": 33355202560, "loss": 3.963084250688553, "l_mix": 2.453617751598358, "l_a": 2.543609470129013, "l_c": 2.4879454374313354, "z_loss": 44.329795837402344, "lr": 0.0021111693416140933, "wd": 0.01, "grad_norm": 0.14023032784461975, "tokens_per_second": 1378571.7768480126, "eta_seconds": 3369.187930583954, "data_wait_seconds": 1.58846378326416e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 63630, "total_steps": 72479, "tokens": 33360445440, "loss": 4.653918623924255, "l_mix": 2.887539863586426, "l_a": 2.9647784531116486, "l_c": 2.923150449991226, "z_loss": 40.981104373931885, "lr": 0.0021093405178953645, "wd": 0.01, "grad_norm": 0.09904347360134125, "tokens_per_second": 1406256.2685669109, "eta_seconds": 3299.131613278389, "data_wait_seconds": 4.443526268005371e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 63640, "total_steps": 72479, "tokens": 33365688320, "loss": 4.467888534069061, "l_mix": 2.7709064185619354, "l_a": 2.850494295358658, "l_c": 2.8061124980449677, "z_loss": 42.889596939086914, "lr": 0.0021075036808445263, "wd": 0.01, "grad_norm": 0.1248100996017456, "tokens_per_second": 1439026.4663891078, "eta_seconds": 3220.3588608264927, "data_wait_seconds": 2.2619962692260742e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 63650, "total_steps": 72479, "tokens": 33370931200, "loss": 4.5624518394470215, "l_mix": 2.8305894434452057, "l_a": 2.9075525403022766, "l_c": 2.865321695804596, "z_loss": 41.18280553817749, "lr": 0.0021056588458046033, "wd": 0.01, "grad_norm": 0.13475190103054047, "tokens_per_second": 1450203.452062347, "eta_seconds": 3191.923688650131, "data_wait_seconds": 2.2470951080322266e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 63660, "total_steps": 72479, "tokens": 33376174080, "loss": 4.6555017828941345, "l_mix": 2.888470232486725, "l_a": 2.9641358256340027, "l_c": 2.9259690940380096, "z_loss": 41.678218841552734, "lr": 0.002103806028185428, "wd": 0.01, "grad_norm": 0.1229444071650505, "tokens_per_second": 1431556.6002146727, "eta_seconds": 3229.8379758834835, "data_wait_seconds": 1.728534698486328e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 63670, "total_steps": 72479, "tokens": 33381416960, "loss": 4.522540628910065, "l_mix": 2.8059417009353638, "l_a": 2.879715323448181, "l_c": 2.8422812819480896, "z_loss": 41.76369524002075, "lr": 0.00210194524346351, "wd": 0.01, "grad_norm": 0.12341897934675217, "tokens_per_second": 1161259.0214729784, "eta_seconds": 3977.1083854675294, "data_wait_seconds": 2.4557113647460938e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 63680, "total_steps": 72479, "tokens": 33386659840, "loss": 3.821607679128647, "l_mix": 2.3648224472999573, "l_a": 2.4579752683639526, "l_c": 2.3979752957820892, "z_loss": 46.17714786529541, "lr": 0.0021000765071819073, "wd": 0.01, "grad_norm": 0.21148018538951874, "tokens_per_second": 1436309.0888482002, "eta_seconds": 3211.8505326032637, "data_wait_seconds": 2.2232532501220703e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 63690, "total_steps": 72479, "tokens": 33391902720, "loss": 3.959558814764023, "l_mix": 2.45053231716156, "l_a": 2.542434811592102, "l_c": 2.4876533448696136, "z_loss": 44.89960336685181, "lr": 0.002098199834950098, "wd": 0.01, "grad_norm": 0.15201778709888458, "tokens_per_second": 1343429.2012063, "eta_seconds": 3430.003775310516, "data_wait_seconds": 1.823902130126953e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 63700, "total_steps": 72479, "tokens": 33397145600, "loss": 3.7979632914066315, "l_mix": 2.349813222885132, "l_a": 2.4423374831676483, "l_c": 2.3848291635513306, "z_loss": 45.16742277145386, "lr": 0.002096315242443849, "wd": 0.01, "grad_norm": 0.16193918883800507, "tokens_per_second": 1435350.7656343717, "eta_seconds": 3206.689585709572, "data_wait_seconds": 1.519918441772461e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16072477, "gate_a2b/L1_absmax": 0.68225217, "gate_a2b/L1_rms": 0.17060055, "gate_a2b/L3_absmean": 0.1525687, "gate_a2b/L3_absmax": 0.49601689, "gate_a2b/L3_rms": 0.1581624, "gate_a2b/L5_absmean": 0.22093563, "gate_a2b/L5_absmax": 0.72939658, "gate_a2b/L5_rms": 0.2269147, "gate_c2b/L1_absmean": 0.03894285, "gate_c2b/L1_absmax": 0.62315142, "gate_c2b/L1_rms": 0.0703504, "gate_c2b/L3_absmean": 0.1666491, "gate_c2b/L3_absmax": 0.3851307, "gate_c2b/L3_rms": 0.17016535, "gate_c2b/L5_absmean": 0.4708201, "gate_c2b/L5_absmax": 0.66726542, "gate_c2b/L5_rms": 0.47998324, "gate_b2a/L7_absmean": 0.05104976, "gate_b2a/L7_absmax": 0.33176339, "gate_b2a/L7_rms": 0.05684112, "gate_b2a/L11_absmean": 0.08345744, "gate_b2a/L11_absmax": 0.61319453, "gate_b2a/L11_rms": 0.09254181, "gate_b2a/L14_absmean": 0.21650499, "gate_b2a/L14_absmax": 0.81236243, "gate_b2a/L14_rms": 0.22572513, "gate_b2c/L3_absmean": 0.03554313, "gate_b2c/L3_absmax": 0.55020857, "gate_b2c/L3_rms": 0.04596727, "gate_b2c/L5_absmean": 0.03754734, "gate_b2c/L5_absmax": 0.52634662, "gate_b2c/L5_rms": 0.04652701, "gate_b2c/L6_absmean": 0.0541564, "gate_b2c/L6_absmax": 0.34051716, "gate_b2c/L6_rms": 0.05993253}, "streams": {"cos_a2b/round0": 0.357282, "cos_c2b/round0": 0.244729, "cos_b2a/round0": 0.16082, "cos_b2c/round0": 0.148066, "rms_a/round0": 7.744561, "rms_b/round0": 13.065945, "rms_c/round0": 5.035139, "cos_a2b/round1": 0.414612, "cos_c2b/round1": 0.303695, "cos_b2a/round1": 0.25368, "cos_b2c/round1": 0.109491, "rms_a/round1": 15.082549, "rms_b/round1": 30.827509, "rms_c/round1": 10.65169, "cos_a2b/round2": 0.523933, "cos_c2b/round2": 0.527776, "cos_b2a/round2": 0.090183, "cos_b2c/round2": 0.183864, "rms_a/round2": 43.579147, "rms_b/round2": 85.029739, "rms_c/round2": 20.69939}, "alpha_mean": 0.4132966065129722, "alpha_hist": [0.01213, 0.10095, 0.21069, 0.19727, 0.15894, 0.13271, 0.10832, 0.06496, 0.01379, 0.00024], "alpha_by_type": {"special": {"share": 0.0006, "mean": 0.57953, "std": 0.09822}, "whitespace": {"share": 0.02461, "mean": 0.52923, "std": 0.1075}, "digit": {"share": 0.03635, "mean": 0.45401, "std": 0.09645}, "word_start": {"share": 0.5838, "mean": 0.43034, "std": 0.17907}, "word_cont": {"share": 0.25595, "mean": 0.29378, "std": 0.12286}, "punct": {"share": 0.09869, "mean": 0.57754, "std": 0.17012}}, "alpha_p05": 0.1587, "alpha_p50": 0.40028, "alpha_p95": 0.71103}
{"step": 63710, "total_steps": 72479, "tokens": 33402388480, "loss": 3.815234959125519, "l_mix": 2.3605760633945465, "l_a": 2.453603506088257, "l_c": 2.3952592611312866, "z_loss": 46.5381965637207, "lr": 0.002094422745405084, "wd": 0.01, "grad_norm": 0.1780240386724472, "tokens_per_second": 1398188.8054723477, "eta_seconds": 3288.169275856018, "data_wait_seconds": 1.6063451766967773e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 63720, "total_steps": 72479, "tokens": 33407631360, "loss": 3.896501749753952, "l_mix": 2.411078602075577, "l_a": 2.5044862627983093, "l_c": 2.4469240605831146, "z_loss": 45.886436462402344, "lr": 0.002092522359641753, "wd": 0.01, "grad_norm": 0.14044392108917236, "tokens_per_second": 1394795.1258876503, "eta_seconds": 3292.410839962959, "data_wait_seconds": 1.8358230590820312e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 63730, "total_steps": 72479, "tokens": 33412874240, "loss": 3.8224488496780396, "l_mix": 2.365169882774353, "l_a": 2.455588221549988, "l_c": 2.4020082652568817, "z_loss": 44.98109149932861, "lr": 0.0020906141010277007, "wd": 0.01, "grad_norm": 0.1835658848285675, "tokens_per_second": 1301526.1938118322, "eta_seconds": 3524.320704269409, "data_wait_seconds": 1.5556812286376953e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 63740, "total_steps": 72479, "tokens": 33418117120, "loss": 3.0436868965625763, "l_mix": 1.873133048415184, "l_a": 1.9836358726024628, "l_c": 1.9182099997997284, "z_loss": 46.710012912750244, "lr": 0.0020886979855025338, "wd": 0.01, "grad_norm": 0.12545567750930786, "tokens_per_second": 1456292.325276239, "eta_seconds": 3146.176596879959, "data_wait_seconds": 3.2395124435424805e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 63750, "total_steps": 72479, "tokens": 33423360000, "loss": 3.027862787246704, "l_mix": 1.8638248592615128, "l_a": 1.9712912291288376, "l_c": 1.9088349044322968, "z_loss": 47.277854919433594, "lr": 0.0020867740290714865, "wd": 0.01, "grad_norm": 0.1440029889345169, "tokens_per_second": 1452219.8028330335, "eta_seconds": 3151.389302825928, "data_wait_seconds": 1.6123056411743164e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 63760, "total_steps": 72479, "tokens": 33428602880, "loss": 3.0282594561576843, "l_mix": 1.864399865269661, "l_a": 1.9705132842063904, "l_c": 1.9090184271335602, "z_loss": 45.37662887573242, "lr": 0.002084842247805288, "wd": 0.01, "grad_norm": 0.16391657292842865, "tokens_per_second": 1210886.8843461073, "eta_seconds": 3775.1396361589436, "data_wait_seconds": 2.7865171432495117e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 63770, "total_steps": 72479, "tokens": 33433845760, "loss": 4.4387542605400085, "l_mix": 2.7531475722789764, "l_a": 2.829113245010376, "l_c": 2.789575546979904, "z_loss": 42.63018846511841, "lr": 0.0020829026578400287, "wd": 0.01, "grad_norm": 0.12140484154224396, "tokens_per_second": 1431688.2952301456, "eta_seconds": 3189.2585887670516, "data_wait_seconds": 1.7374753952026367e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 63780, "total_steps": 72479, "tokens": 33439088640, "loss": 4.538026332855225, "l_mix": 2.815673679113388, "l_a": 2.8921016454696655, "l_c": 2.8490735292434692, "z_loss": 41.087157249450684, "lr": 0.002080955275377024, "wd": 0.01, "grad_norm": 0.16978973150253296, "tokens_per_second": 1413068.9409394837, "eta_seconds": 3227.571691560745, "data_wait_seconds": 1.6242265701293945e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 63790, "total_steps": 72479, "tokens": 33444331520, "loss": 3.9321380853652954, "l_mix": 2.4342217445373535, "l_a": 2.521934896707535, "l_c": 2.4711192548274994, "z_loss": 45.546631813049316, "lr": 0.00207900011668268, "wd": 0.01, "grad_norm": 0.19560426473617554, "tokens_per_second": 1401723.2252194814, "eta_seconds": 3249.9557330846787, "data_wait_seconds": 1.683831214904785e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 63800, "total_steps": 72479, "tokens": 33449574400, "loss": 3.836357533931732, "l_mix": 2.374337822198868, "l_a": 2.4672330021858215, "l_c": 2.4061659574508667, "z_loss": 45.491318225860596, "lr": 0.0020770371980883572, "wd": 0.01, "grad_norm": 0.20030207931995392, "tokens_per_second": 1437635.9941229487, "eta_seconds": 3165.123557424545, "data_wait_seconds": 2.5212764739990234e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16097955, "gate_a2b/L1_absmax": 0.68252009, "gate_a2b/L1_rms": 0.17076735, "gate_a2b/L3_absmean": 0.15295212, "gate_a2b/L3_absmax": 0.49439916, "gate_a2b/L3_rms": 0.15851352, "gate_a2b/L5_absmean": 0.22086015, "gate_a2b/L5_absmax": 0.72854495, "gate_a2b/L5_rms": 0.2268156, "gate_c2b/L1_absmean": 0.03858229, "gate_c2b/L1_absmax": 0.62215352, "gate_c2b/L1_rms": 0.07027178, "gate_c2b/L3_absmean": 0.1666023, "gate_c2b/L3_absmax": 0.38491565, "gate_c2b/L3_rms": 0.17010811, "gate_c2b/L5_absmean": 0.47047338, "gate_c2b/L5_absmax": 0.66738778, "gate_c2b/L5_rms": 0.47966379, "gate_b2a/L7_absmean": 0.05126555, "gate_b2a/L7_absmax": 0.33231273, "gate_b2a/L7_rms": 0.05701261, "gate_b2a/L11_absmean": 0.08329763, "gate_b2a/L11_absmax": 0.61459392, "gate_b2a/L11_rms": 0.09243333, "gate_b2a/L14_absmean": 0.21662062, "gate_b2a/L14_absmax": 0.81170458, "gate_b2a/L14_rms": 0.22581072, "gate_b2c/L3_absmean": 0.03557014, "gate_b2c/L3_absmax": 0.5514279, "gate_b2c/L3_rms": 0.04601935, "gate_b2c/L5_absmean": 0.03750731, "gate_b2c/L5_absmax": 0.5271281, "gate_b2c/L5_rms": 0.04651494, "gate_b2c/L6_absmean": 0.05422264, "gate_b2c/L6_absmax": 0.34045234, "gate_b2c/L6_rms": 0.06002796}, "streams": {"cos_a2b/round0": 0.355322, "cos_c2b/round0": 0.244409, "cos_b2a/round0": 0.164975, "cos_b2c/round0": 0.146301, "rms_a/round0": 7.748673, "rms_b/round0": 13.142617, "rms_c/round0": 5.038694, "cos_a2b/round1": 0.418134, "cos_c2b/round1": 0.304303, "cos_b2a/round1": 0.252647, "cos_b2c/round1": 0.11523, "rms_a/round1": 15.338376, "rms_b/round1": 31.254364, "rms_c/round1": 10.858859, "cos_a2b/round2": 0.530779, "cos_c2b/round2": 0.519822, "cos_b2a/round2": 0.094969, "cos_b2c/round2": 0.181663, "rms_a/round2": 44.06559, "rms_b/round2": 84.838646, "rms_c/round2": 21.347689}, "alpha_mean": 0.4200780606492316, "alpha_hist": [0.01392, 0.10374, 0.2052, 0.1915, 0.14995, 0.12448, 0.10866, 0.0817, 0.02042, 0.00044], "alpha_by_type": {"special": {"share": 0.00107, "mean": 0.56557, "std": 0.11123}, "whitespace": {"share": 0.02263, "mean": 0.52326, "std": 0.11221}, "digit": {"share": 0.02075, "mean": 0.43229, "std": 0.11032}, "word_start": {"share": 0.60139, "mean": 0.43575, "std": 0.18553}, "word_cont": {"share": 0.25935, "mean": 0.30186, "std": 0.13204}, "punct": {"share": 0.0948, "mean": 0.61511, "std": 0.17256}}, "alpha_p05": 0.1394, "alpha_p50": 0.38168, "alpha_p95": 0.74055}
{"step": 63810, "total_steps": 72479, "tokens": 33454817280, "loss": 3.734885275363922, "l_mix": 2.309803932905197, "l_a": 2.4034635722637177, "l_c": 2.346807152032852, "z_loss": 46.39628839492798, "lr": 0.002075066535990235, "wd": 0.01, "grad_norm": 0.12094168365001678, "tokens_per_second": 1445675.5237310156, "eta_seconds": 3143.8954297780992, "data_wait_seconds": 1.5974044799804688e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 63820, "total_steps": 72479, "tokens": 33460060160, "loss": 3.961240917444229, "l_mix": 2.4518233835697174, "l_a": 2.541450798511505, "l_c": 2.489940881729126, "z_loss": 45.840081214904785, "lr": 0.0020730881468491716, "wd": 0.01, "grad_norm": 0.15399375557899475, "tokens_per_second": 1398235.0350070095, "eta_seconds": 3246.8145042419433, "data_wait_seconds": 1.6391277313232422e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 63830, "total_steps": 72479, "tokens": 33465303040, "loss": 3.7984141409397125, "l_mix": 2.350751966238022, "l_a": 2.440624386072159, "l_c": 2.384916305541992, "z_loss": 45.19134759902954, "lr": 0.0020711020471905726, "wd": 0.01, "grad_norm": 0.1319703608751297, "tokens_per_second": 1421832.531722242, "eta_seconds": 3189.241215705871, "data_wait_seconds": 1.671910285949707e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 63840, "total_steps": 72479, "tokens": 33470545920, "loss": 3.8753721714019775, "l_mix": 2.3976848125457764, "l_a": 2.48946750164032, "l_c": 2.436156779527664, "z_loss": 45.99437093734741, "lr": 0.0020691082536042457, "wd": 0.01, "grad_norm": 0.20615746080875397, "tokens_per_second": 1379779.211026873, "eta_seconds": 3282.6440605878834, "data_wait_seconds": 1.5914440155029297e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 63850, "total_steps": 72479, "tokens": 33475788800, "loss": 3.0003641843795776, "l_mix": 1.846335530281067, "l_a": 1.9552185833454132, "l_c": 1.891543596982956, "z_loss": 47.444766998291016, "lr": 0.0020671067827442683, "wd": 0.01, "grad_norm": 0.14530634880065918, "tokens_per_second": 1367286.8215039922, "eta_seconds": 3308.801840877533, "data_wait_seconds": 4.106760025024414e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 63860, "total_steps": 72479, "tokens": 33481031680, "loss": 3.075896829366684, "l_mix": 1.8930605500936508, "l_a": 2.004473313689232, "l_c": 1.9383142441511154, "z_loss": 47.21238374710083, "lr": 0.002065097651328844, "wd": 0.01, "grad_norm": 0.1441492885351181, "tokens_per_second": 1402230.291317755, "eta_seconds": 3222.6077984333037, "data_wait_seconds": 2.35140323638916e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 63870, "total_steps": 72479, "tokens": 33486274560, "loss": 3.079216718673706, "l_mix": 1.8955432176589966, "l_a": 2.0043942034244537, "l_c": 1.9411839544773102, "z_loss": 45.8037748336792, "lr": 0.0020630808761401653, "wd": 0.01, "grad_norm": 0.1696937382221222, "tokens_per_second": 952678.195533074, "eta_seconds": 4737.796470165253, "data_wait_seconds": 3.3020973205566406e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 63880, "total_steps": 72479, "tokens": 33491517440, "loss": 3.0819635093212128, "l_mix": 1.8973293602466583, "l_a": 2.0070558041334152, "l_c": 1.9417245984077454, "z_loss": 46.334067821502686, "lr": 0.002061056474024273, "wd": 0.01, "grad_norm": 0.16205763816833496, "tokens_per_second": 1320639.110233764, "eta_seconds": 3413.7657116651535, "data_wait_seconds": 1.7642974853515625e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 63890, "total_steps": 72479, "tokens": 33496760320, "loss": 3.055128961801529, "l_mix": 1.8797425329685211, "l_a": 1.98994679749012, "l_c": 1.9280077368021011, "z_loss": 47.93324899673462, "lr": 0.0020590244618909153, "wd": 0.01, "grad_norm": 0.173757866024971, "tokens_per_second": 1366685.3585183623, "eta_seconds": 3294.9132029056545, "data_wait_seconds": 1.8149614334106445e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 63900, "total_steps": 72479, "tokens": 33502003200, "loss": 3.9260148406028748, "l_mix": 2.431313842535019, "l_a": 2.5164975225925446, "l_c": 2.465839207172394, "z_loss": 44.27238988876343, "lr": 0.0020569848567134064, "wd": 0.01, "grad_norm": 0.15057788789272308, "tokens_per_second": 1263592.9650691205, "eta_seconds": 3559.585148334503, "data_wait_seconds": 2.2858381271362305e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.16003191, "gate_a2b/L1_absmax": 0.68301719, "gate_a2b/L1_rms": 0.17001364, "gate_a2b/L3_absmean": 0.15273423, "gate_a2b/L3_absmax": 0.4959726, "gate_a2b/L3_rms": 0.15832409, "gate_a2b/L5_absmean": 0.22054043, "gate_a2b/L5_absmax": 0.72830558, "gate_a2b/L5_rms": 0.22649053, "gate_c2b/L1_absmean": 0.03935563, "gate_c2b/L1_absmax": 0.62198502, "gate_c2b/L1_rms": 0.07061695, "gate_c2b/L3_absmean": 0.16664751, "gate_c2b/L3_absmax": 0.38328716, "gate_c2b/L3_rms": 0.1701335, "gate_c2b/L5_absmean": 0.47024202, "gate_c2b/L5_absmax": 0.66681546, "gate_c2b/L5_rms": 0.47941905, "gate_b2a/L7_absmean": 0.05108706, "gate_b2a/L7_absmax": 0.33280519, "gate_b2a/L7_rms": 0.05690717, "gate_b2a/L11_absmean": 0.08320248, "gate_b2a/L11_absmax": 0.61519462, "gate_b2a/L11_rms": 0.09238437, "gate_b2a/L14_absmean": 0.21701106, "gate_b2a/L14_absmax": 0.81173444, "gate_b2a/L14_rms": 0.22619955, "gate_b2c/L3_absmean": 0.03576806, "gate_b2c/L3_absmax": 0.55021971, "gate_b2c/L3_rms": 0.04610667, "gate_b2c/L5_absmean": 0.03760842, "gate_b2c/L5_absmax": 0.52676725, "gate_b2c/L5_rms": 0.04662737, "gate_b2c/L6_absmean": 0.05400202, "gate_b2c/L6_absmax": 0.33964953, "gate_b2c/L6_rms": 0.05980561}, "streams": {"cos_a2b/round0": 0.356522, "cos_c2b/round0": 0.245903, "cos_b2a/round0": 0.166915, "cos_b2c/round0": 0.148139, "rms_a/round0": 7.756925, "rms_b/round0": 13.060795, "rms_c/round0": 5.00528, "cos_a2b/round1": 0.419978, "cos_c2b/round1": 0.306538, "cos_b2a/round1": 0.252543, "cos_b2c/round1": 0.111146, "rms_a/round1": 15.417027, "rms_b/round1": 30.907324, "rms_c/round1": 10.76991, "cos_a2b/round2": 0.535177, "cos_c2b/round2": 0.52331, "cos_b2a/round2": 0.091715, "cos_b2c/round2": 0.183976, "rms_a/round2": 44.059246, "rms_b/round2": 84.818199, "rms_c/round2": 21.100845}, "alpha_mean": 0.4288308496487758, "alpha_hist": [0.00771, 0.07874, 0.18594, 0.20723, 0.17331, 0.15065, 0.11986, 0.06583, 0.01056, 0.00018], "alpha_by_type": {"special": {"share": 0.00084, "mean": 0.57558, "std": 0.11516}, "whitespace": {"share": 0.0224, "mean": 0.51785, "std": 0.13518}, "digit": {"share": 0.02414, "mean": 0.465, "std": 0.12227}, "word_start": {"share": 0.59854, "mean": 0.43502, "std": 0.17192}, "word_cont": {"share": 0.25128, "mean": 0.33237, "std": 0.12476}, "punct": {"share": 0.1028, "mean": 0.5995, "std": 0.15568}}, "alpha_p05": 0.17781, "alpha_p50": 0.40521, "alpha_p95": 0.72563}
{"step": 63910, "total_steps": 72479, "tokens": 33507246080, "loss": 3.9445391297340393, "l_mix": 2.441687285900116, "l_a": 2.532940775156021, "l_c": 2.47656512260437, "z_loss": 45.74306678771973, "lr": 0.002054937675528485, "wd": 0.01, "grad_norm": 0.16761818528175354, "tokens_per_second": 1421802.5624969045, "eta_seconds": 3159.8085349559788, "data_wait_seconds": 3.11732292175293e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 63920, "total_steps": 72479, "tokens": 33512488960, "loss": 3.8685843646526337, "l_mix": 2.3943809270858765, "l_a": 2.4835635125637054, "l_c": 2.4304477274417877, "z_loss": 45.17158651351929, "lr": 0.0020528829354361715, "wd": 0.01, "grad_norm": 0.13991659879684448, "tokens_per_second": 1430494.606039837, "eta_seconds": 3136.9436648368833, "data_wait_seconds": 1.6063451766967773e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 63930, "total_steps": 72479, "tokens": 33517731840, "loss": 3.8213318586349487, "l_mix": 2.363545745611191, "l_a": 2.459208518266678, "l_c": 2.4000784158706665, "z_loss": 47.34740591049194, "lr": 0.002050820653599626, "wd": 0.01, "grad_norm": 0.2063327133655548, "tokens_per_second": 1358443.4463253103, "eta_seconds": 3299.4661088943485, "data_wait_seconds": 2.4139881134033203e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 63940, "total_steps": 72479, "tokens": 33522974720, "loss": 3.914548695087433, "l_mix": 2.42301207780838, "l_a": 2.5137211084365845, "l_c": 2.458067446947098, "z_loss": 45.06916427612305, "lr": 0.0020487508472450044, "wd": 0.01, "grad_norm": 0.13399018347263336, "tokens_per_second": 1385866.0648361663, "eta_seconds": 3230.395307016373, "data_wait_seconds": 1.9788742065429688e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 63950, "total_steps": 72479, "tokens": 33528217600, "loss": 3.922861635684967, "l_mix": 2.4280532598495483, "l_a": 2.519291251897812, "l_c": 2.463403105735779, "z_loss": 45.28832197189331, "lr": 0.0020466735336613145, "wd": 0.01, "grad_norm": 0.13152839243412018, "tokens_per_second": 1442338.1763541815, "eta_seconds": 3100.2801044225694, "data_wait_seconds": 3.1113624572753906e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 63960, "total_steps": 72479, "tokens": 33533460480, "loss": 3.0492136776447296, "l_mix": 1.8769125491380692, "l_a": 1.985692635178566, "l_c": 1.9219776093959808, "z_loss": 48.29178524017334, "lr": 0.0020445887302002717, "wd": 0.01, "grad_norm": 0.18849153816699982, "tokens_per_second": 1416342.3930961026, "eta_seconds": 3153.481456017494, "data_wait_seconds": 3.987550735473633e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 63970, "total_steps": 72479, "tokens": 33538703360, "loss": 3.066956788301468, "l_mix": 1.8876546770334244, "l_a": 1.9945359379053116, "l_c": 1.9364709258079529, "z_loss": 47.13297748565674, "lr": 0.002042496454276154, "wd": 0.01, "grad_norm": 0.1401849389076233, "tokens_per_second": 1406740.2523827811, "eta_seconds": 3171.279548192024, "data_wait_seconds": 3.0100345611572266e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 63980, "total_steps": 72479, "tokens": 33543946240, "loss": 3.0733297765254974, "l_mix": 1.8919074088335037, "l_a": 2.0002989023923874, "l_c": 1.937775582075119, "z_loss": 46.62643051147461, "lr": 0.0020403967233656567, "wd": 0.01, "grad_norm": 0.1305733621120453, "tokens_per_second": 1310935.816779491, "eta_seconds": 3399.040330553055, "data_wait_seconds": 1.7195940017700195e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 63990, "total_steps": 72479, "tokens": 33549189120, "loss": 3.0740262269973755, "l_mix": 1.8925288319587708, "l_a": 2.001294508576393, "l_c": 1.9370300769805908, "z_loss": 47.57431888580322, "lr": 0.0020382895550077468, "wd": 0.01, "grad_norm": 0.15521448850631714, "tokens_per_second": 1414774.0841039878, "eta_seconds": 3145.8597397327426, "data_wait_seconds": 4.744529724121094e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 64000, "total_steps": 72479, "tokens": 33554432000, "loss": 3.0142405033111572, "l_mix": 1.8557083159685135, "l_a": 1.9627512991428375, "l_c": 1.8990224450826645, "z_loss": 46.50926733016968, "lr": 0.002036174966803514, "wd": 0.01, "grad_norm": 0.16866916418075562, "tokens_per_second": 1366965.119976806, "eta_seconds": 3252.0492930173873, "data_wait_seconds": 1.677870750427246e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16056794, "gate_a2b/L1_absmax": 0.68246245, "gate_a2b/L1_rms": 0.17041487, "gate_a2b/L3_absmean": 0.15265644, "gate_a2b/L3_absmax": 0.49570489, "gate_a2b/L3_rms": 0.15825036, "gate_a2b/L5_absmean": 0.22064638, "gate_a2b/L5_absmax": 0.7278049, "gate_a2b/L5_rms": 0.22659779, "gate_c2b/L1_absmean": 0.03917306, "gate_c2b/L1_absmax": 0.62187862, "gate_c2b/L1_rms": 0.07054109, "gate_c2b/L3_absmean": 0.16654316, "gate_c2b/L3_absmax": 0.3826884, "gate_c2b/L3_rms": 0.17003185, "gate_c2b/L5_absmean": 0.47018, "gate_c2b/L5_absmax": 0.66549271, "gate_c2b/L5_rms": 0.47931296, "gate_b2a/L7_absmean": 0.05136262, "gate_b2a/L7_absmax": 0.33254042, "gate_b2a/L7_rms": 0.0571311, "gate_b2a/L11_absmean": 0.08362547, "gate_b2a/L11_absmax": 0.61489576, "gate_b2a/L11_rms": 0.09273691, "gate_b2a/L14_absmean": 0.21706389, "gate_b2a/L14_absmax": 0.81439954, "gate_b2a/L14_rms": 0.22629796, "gate_b2c/L3_absmean": 0.03599149, "gate_b2c/L3_absmax": 0.54974467, "gate_b2c/L3_rms": 0.0462551, "gate_b2c/L5_absmean": 0.03745354, "gate_b2c/L5_absmax": 0.52590001, "gate_b2c/L5_rms": 0.04646366, "gate_b2c/L6_absmean": 0.05418138, "gate_b2c/L6_absmax": 0.3404659, "gate_b2c/L6_rms": 0.05997689}, "streams": {"cos_a2b/round0": 0.354451, "cos_c2b/round0": 0.20972, "cos_b2a/round0": 0.1517, "cos_b2c/round0": 0.129419, "rms_a/round0": 8.047616, "rms_b/round0": 13.376309, "rms_c/round0": 5.108971, "cos_a2b/round1": 0.410182, "cos_c2b/round1": 0.307485, "cos_b2a/round1": 0.242115, "cos_b2c/round1": 0.119062, "rms_a/round1": 16.127508, "rms_b/round1": 32.097187, "rms_c/round1": 11.553405, "cos_a2b/round2": 0.525814, "cos_c2b/round2": 0.506273, "cos_b2a/round2": 0.085533, "cos_b2c/round2": 0.175349, "rms_a/round2": 45.003586, "rms_b/round2": 84.664536, "rms_c/round2": 22.503149}, "alpha_mean": 0.42435546599046603, "alpha_hist": [0.02568, 0.1429, 0.19083, 0.15404, 0.1254, 0.11211, 0.10422, 0.09366, 0.04817, 0.00301], "alpha_by_type": {"special": {"share": 0.00117, "mean": 0.68266, "std": 0.09783}, "whitespace": {"share": 0.03687, "mean": 0.62633, "std": 0.16498}, "digit": {"share": 0.00777, "mean": 0.38325, "std": 0.17535}, "word_start": {"share": 0.57919, "mean": 0.42963, "std": 0.20894}, "word_cont": {"share": 0.25359, "mean": 0.28692, "std": 0.14031}, "punct": {"share": 0.12141, "mean": 0.62504, "std": 0.18263}}, "alpha_p05": 0.11596, "alpha_p50": 0.37251, "alpha_p95": 0.79692}
{"step": 64010, "total_steps": 72479, "tokens": 33559674880, "loss": 3.916865289211273, "l_mix": 2.4233610332012177, "l_a": 2.5186009407043457, "l_c": 2.4597465693950653, "z_loss": 46.1513237953186, "lr": 0.002034052976416028, "wd": 0.01, "grad_norm": 0.12800192832946777, "tokens_per_second": 1469773.4203825674, "eta_seconds": 3021.0065105438234, "data_wait_seconds": 2.944469451904297e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 64020, "total_steps": 72479, "tokens": 33564917760, "loss": 3.9083767533302307, "l_mix": 2.419129818677902, "l_a": 2.5083323419094086, "l_c": 2.4558235704898834, "z_loss": 44.80172061920166, "lr": 0.002031923601570188, "wd": 0.01, "grad_norm": 0.15393075346946716, "tokens_per_second": 1402456.189206495, "eta_seconds": 3162.2750329971313, "data_wait_seconds": 1.7136335372924805e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 64030, "total_steps": 72479, "tokens": 33570160640, "loss": 3.9083759784698486, "l_mix": 2.4193306863307953, "l_a": 2.507474333047867, "l_c": 2.4560096859931946, "z_loss": 44.88692617416382, "lr": 0.002029786860052574, "wd": 0.01, "grad_norm": 0.1327582746744156, "tokens_per_second": 1148782.387420386, "eta_seconds": 3856.00385286808, "data_wait_seconds": 1.6361474990844727e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 64040, "total_steps": 72479, "tokens": 33575403520, "loss": 4.59221088886261, "l_mix": 2.8496269285678864, "l_a": 2.9259199500083923, "l_c": 2.882693290710449, "z_loss": 42.00890636444092, "lr": 0.0020276427697113017, "wd": 0.01, "grad_norm": 0.16406144201755524, "tokens_per_second": 1233720.423591941, "eta_seconds": 3586.2796362876893, "data_wait_seconds": 3.254413604736328e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 64050, "total_steps": 72479, "tokens": 33580646400, "loss": 4.378347337245941, "l_mix": 2.7167321741580963, "l_a": 2.7867147624492645, "l_c": 2.7520024329423904, "z_loss": 41.979957580566406, "lr": 0.002025491348455869, "wd": 0.01, "grad_norm": 0.18200483918190002, "tokens_per_second": 1250438.20631853, "eta_seconds": 3534.1398956537246, "data_wait_seconds": 4.208087921142578e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 64060, "total_steps": 72479, "tokens": 33585889280, "loss": 4.611353397369385, "l_mix": 2.8604647517204285, "l_a": 2.9398162364959717, "l_c": 2.89647901058197, "z_loss": 42.512335777282715, "lr": 0.0020233326142570104, "wd": 0.01, "grad_norm": 0.12499363720417023, "tokens_per_second": 1277922.1470352723, "eta_seconds": 3454.029404091835, "data_wait_seconds": 2.193450927734375e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 64070, "total_steps": 72479, "tokens": 33591132160, "loss": 3.030093401670456, "l_mix": 1.8645917922258377, "l_a": 1.9740918725728989, "l_c": 1.910913497209549, "z_loss": 46.26350212097168, "lr": 0.0020211665851465437, "wd": 0.01, "grad_norm": 0.15667015314102173, "tokens_per_second": 1460653.8329881227, "eta_seconds": 3018.3317172288894, "data_wait_seconds": 1.946091651916504e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 64080, "total_steps": 72479, "tokens": 33596375040, "loss": 3.035636246204376, "l_mix": 1.8683513849973679, "l_a": 1.977062240242958, "l_c": 1.9138871133327484, "z_loss": 47.583613872528076, "lr": 0.002018993279217222, "wd": 0.01, "grad_norm": 0.14114157855510712, "tokens_per_second": 1406237.2039056274, "eta_seconds": 3131.4026536703113, "data_wait_seconds": 3.75211238861084e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 64090, "total_steps": 72479, "tokens": 33601617920, "loss": 3.03560408949852, "l_mix": 1.868167668581009, "l_a": 1.9780386835336685, "l_c": 1.9134156852960587, "z_loss": 47.841485023498535, "lr": 0.002016812714622581, "wd": 0.01, "grad_norm": 0.15175791084766388, "tokens_per_second": 1231507.1503113904, "eta_seconds": 3571.43848567009, "data_wait_seconds": 3.254413604736328e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 64100, "total_steps": 72479, "tokens": 33606860800, "loss": 3.005198359489441, "l_mix": 1.8490772992372513, "l_a": 1.9582918137311935, "l_c": 1.895444929599762, "z_loss": 47.923505783081055, "lr": 0.0020146249095767867, "wd": 0.01, "grad_norm": 0.14185325801372528, "tokens_per_second": 1387210.1307483765, "eta_seconds": 3166.794312286377, "data_wait_seconds": 3.293156623840332e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.15962231, "gate_a2b/L1_absmax": 0.68704212, "gate_a2b/L1_rms": 0.16975224, "gate_a2b/L3_absmean": 0.15263389, "gate_a2b/L3_absmax": 0.50095356, "gate_a2b/L3_rms": 0.15830797, "gate_a2b/L5_absmean": 0.22035663, "gate_a2b/L5_absmax": 0.73469675, "gate_a2b/L5_rms": 0.22642104, "gate_c2b/L1_absmean": 0.03997518, "gate_c2b/L1_absmax": 0.62463695, "gate_c2b/L1_rms": 0.07111552, "gate_c2b/L3_absmean": 0.16609123, "gate_c2b/L3_absmax": 0.38798791, "gate_c2b/L3_rms": 0.16962004, "gate_c2b/L5_absmean": 0.46981058, "gate_c2b/L5_absmax": 0.67559952, "gate_c2b/L5_rms": 0.47898644, "gate_b2a/L7_absmean": 0.05166837, "gate_b2a/L7_absmax": 0.33191976, "gate_b2a/L7_rms": 0.05745338, "gate_b2a/L11_absmean": 0.08421001, "gate_b2a/L11_absmax": 0.6152311, "gate_b2a/L11_rms": 0.09328289, "gate_b2a/L14_absmean": 0.2156828, "gate_b2a/L14_absmax": 0.81235838, "gate_b2a/L14_rms": 0.22515318, "gate_b2c/L3_absmean": 0.03549193, "gate_b2c/L3_absmax": 0.55410743, "gate_b2c/L3_rms": 0.04613654, "gate_b2c/L5_absmean": 0.03742861, "gate_b2c/L5_absmax": 0.52497488, "gate_b2c/L5_rms": 0.04647837, "gate_b2c/L6_absmean": 0.05402664, "gate_b2c/L6_absmax": 0.3437497, "gate_b2c/L6_rms": 0.05989293}, "streams": {"cos_a2b/round0": 0.353728, "cos_c2b/round0": 0.212787, "cos_b2a/round0": 0.148465, "cos_b2c/round0": 0.127532, "rms_a/round0": 8.023575, "rms_b/round0": 13.357992, "rms_c/round0": 5.124769, "cos_a2b/round1": 0.410926, "cos_c2b/round1": 0.302458, "cos_b2a/round1": 0.239801, "cos_b2c/round1": 0.115816, "rms_a/round1": 16.127657, "rms_b/round1": 32.306843, "rms_c/round1": 11.513831, "cos_a2b/round2": 0.514157, "cos_c2b/round2": 0.511283, "cos_b2a/round2": 0.085435, "cos_b2c/round2": 0.17196, "rms_a/round2": 44.967735, "rms_b/round2": 85.798416, "rms_c/round2": 22.540674}, "alpha_mean": 0.41033297926216505, "alpha_hist": [0.03294, 0.17192, 0.19191, 0.14481, 0.11253, 0.10562, 0.10468, 0.08507, 0.04509, 0.00543], "alpha_by_type": {"special": {"share": 0.00104, "mean": 0.65849, "std": 0.10066}, "whitespace": {"share": 0.04324, "mean": 0.62315, "std": 0.1858}, "digit": {"share": 0.0099, "mean": 0.41387, "std": 0.18584}, "word_start": {"share": 0.55313, "mean": 0.40512, "std": 0.21431}, "word_cont": {"share": 0.26866, "mean": 0.28906, "std": 0.14571}, "punct": {"share": 0.12402, "mean": 0.61974, "std": 0.1916}}, "alpha_p05": 0.12169, "alpha_p50": 0.36207, "alpha_p95": 0.81197}
{"step": 64110, "total_steps": 72479, "tokens": 33612103680, "loss": 3.047305405139923, "l_mix": 1.8762006908655167, "l_a": 1.982196107506752, "l_c": 1.9214860945940018, "z_loss": 45.520508766174316, "lr": 0.0020124298823544855, "wd": 0.01, "grad_norm": 0.1537570357322693, "tokens_per_second": 1410290.2355281638, "eta_seconds": 3111.2505507469177, "data_wait_seconds": 2.3424625396728516e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 64120, "total_steps": 72479, "tokens": 33617346560, "loss": 4.088205128908157, "l_mix": 2.5319111049175262, "l_a": 2.6206184029579163, "l_c": 2.567028194665909, "z_loss": 45.81114959716797, "lr": 0.0020102276512906506, "wd": 0.01, "grad_norm": 0.20599795877933502, "tokens_per_second": 1434293.5207613586, "eta_seconds": 3055.5275671005247, "data_wait_seconds": 1.856684684753418e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 64130, "total_steps": 72479, "tokens": 33622589440, "loss": 4.097122609615326, "l_mix": 2.5377222299575806, "l_a": 2.6234854459762573, "l_c": 2.574515640735626, "z_loss": 44.31173038482666, "lr": 0.0020080182347804276, "wd": 0.01, "grad_norm": 0.1653050035238266, "tokens_per_second": 1295148.0084391634, "eta_seconds": 3379.7531119823457, "data_wait_seconds": 2.0116567611694336e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 64140, "total_steps": 72479, "tokens": 33627832320, "loss": 4.224485754966736, "l_mix": 2.6169886887073517, "l_a": 2.7051589488983154, "l_c": 2.6531645953655243, "z_loss": 44.94548463821411, "lr": 0.0020058016512789823, "wd": 0.01, "grad_norm": 0.15884654223918915, "tokens_per_second": 1473442.143250698, "eta_seconds": 2967.227218270302, "data_wait_seconds": 1.6182661056518555e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 64150, "total_steps": 72479, "tokens": 33633075200, "loss": 3.0733809173107147, "l_mix": 1.8922199308872223, "l_a": 2.0017120093107224, "l_c": 1.9354910850524902, "z_loss": 46.11836910247803, "lr": 0.0020035779193013463, "wd": 0.01, "grad_norm": 0.1908397227525711, "tokens_per_second": 1353424.3153628714, "eta_seconds": 3226.478719520569, "data_wait_seconds": 2.3663043975830078e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 64160, "total_steps": 72479, "tokens": 33638318080, "loss": 3.042746424674988, "l_mix": 1.8722945749759674, "l_a": 1.9818061143159866, "l_c": 1.9196999669075012, "z_loss": 46.5198540687561, "lr": 0.002001347057422261, "wd": 0.01, "grad_norm": 0.15926912426948547, "tokens_per_second": 1430739.5703844856, "eta_seconds": 3048.4596653938293, "data_wait_seconds": 1.6987323760986328e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 64170, "total_steps": 72479, "tokens": 33643560960, "loss": 3.079346716403961, "l_mix": 1.8952624946832657, "l_a": 2.0047939121723175, "l_c": 1.9421533197164536, "z_loss": 46.935728549957275, "lr": 0.001999109084276025, "wd": 0.01, "grad_norm": 0.21748130023479462, "tokens_per_second": 1406846.0894999276, "eta_seconds": 3096.5071620225904, "data_wait_seconds": 2.6851892471313477e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 64180, "total_steps": 72479, "tokens": 33648803840, "loss": 4.672227382659912, "l_mix": 2.8983516693115234, "l_a": 2.974691241979599, "l_c": 2.9382272362709045, "z_loss": 42.34416580200195, "lr": 0.001996864018556337, "wd": 0.01, "grad_norm": 0.15538376569747925, "tokens_per_second": 1460593.682578689, "eta_seconds": 2978.9709238767628, "data_wait_seconds": 3.55839729309082e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 64190, "total_steps": 72479, "tokens": 33654046720, "loss": 3.0494672656059265, "l_mix": 1.8765757977962494, "l_a": 1.9856074303388596, "l_c": 1.9240305423736572, "z_loss": 47.61371326446533, "lr": 0.001994611879016138, "wd": 0.01, "grad_norm": 0.1429259032011032, "tokens_per_second": 1337336.5363909507, "eta_seconds": 3249.610785126686, "data_wait_seconds": 2.256035804748535e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 64200, "total_steps": 72479, "tokens": 33659289600, "loss": 3.023402214050293, "l_mix": 1.8618107587099075, "l_a": 1.967115119099617, "l_c": 1.9048562347888947, "z_loss": 46.32409954071045, "lr": 0.001992352684467458, "wd": 0.01, "grad_norm": 0.16052287817001343, "tokens_per_second": 1437477.267509468, "eta_seconds": 3019.581909298897, "data_wait_seconds": 1.6868114471435547e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.15995407, "gate_a2b/L1_absmax": 0.68543702, "gate_a2b/L1_rms": 0.16993231, "gate_a2b/L3_absmean": 0.1519758, "gate_a2b/L3_absmax": 0.50187904, "gate_a2b/L3_rms": 0.1575959, "gate_a2b/L5_absmean": 0.21995042, "gate_a2b/L5_absmax": 0.73536158, "gate_a2b/L5_rms": 0.22600737, "gate_c2b/L1_absmean": 0.03970388, "gate_c2b/L1_absmax": 0.62275553, "gate_c2b/L1_rms": 0.07079148, "gate_c2b/L3_absmean": 0.16643523, "gate_c2b/L3_absmax": 0.3897025, "gate_c2b/L3_rms": 0.16991374, "gate_c2b/L5_absmean": 0.46978092, "gate_c2b/L5_absmax": 0.67451668, "gate_c2b/L5_rms": 0.47895509, "gate_b2a/L7_absmean": 0.05112758, "gate_b2a/L7_absmax": 0.33176446, "gate_b2a/L7_rms": 0.05694789, "gate_b2a/L11_absmean": 0.08341324, "gate_b2a/L11_absmax": 0.61562574, "gate_b2a/L11_rms": 0.09260701, "gate_b2a/L14_absmean": 0.21649241, "gate_b2a/L14_absmax": 0.81294537, "gate_b2a/L14_rms": 0.22580332, "gate_b2c/L3_absmean": 0.0355275, "gate_b2c/L3_absmax": 0.55200273, "gate_b2c/L3_rms": 0.04596981, "gate_b2c/L5_absmean": 0.03736784, "gate_b2c/L5_absmax": 0.52596366, "gate_b2c/L5_rms": 0.04638585, "gate_b2c/L6_absmean": 0.05423541, "gate_b2c/L6_absmax": 0.34250671, "gate_b2c/L6_rms": 0.06007893}, "streams": {"cos_a2b/round0": 0.353527, "cos_c2b/round0": 0.20896, "cos_b2a/round0": 0.153252, "cos_b2c/round0": 0.128405, "rms_a/round0": 8.00503, "rms_b/round0": 13.383575, "rms_c/round0": 5.145071, "cos_a2b/round1": 0.41331, "cos_c2b/round1": 0.308569, "cos_b2a/round1": 0.246459, "cos_b2c/round1": 0.11844, "rms_a/round1": 16.08396, "rms_b/round1": 32.347809, "rms_c/round1": 11.562187, "cos_a2b/round2": 0.51425, "cos_c2b/round2": 0.513782, "cos_b2a/round2": 0.085965, "cos_b2c/round2": 0.174434, "rms_a/round2": 44.715328, "rms_b/round2": 85.313736, "rms_c/round2": 22.552715}, "alpha_mean": 0.40022710240771175, "alpha_hist": [0.02985, 0.17296, 0.20274, 0.15186, 0.12085, 0.10646, 0.09666, 0.07877, 0.03784, 0.00201], "alpha_by_type": {"special": {"share": 0.00102, "mean": 0.60386, "std": 0.11688}, "whitespace": {"share": 0.02983, "mean": 0.61531, "std": 0.16839}, "digit": {"share": 0.01213, "mean": 0.39928, "std": 0.16243}, "word_start": {"share": 0.57906, "mean": 0.40364, "std": 0.2071}, "word_cont": {"share": 0.26564, "mean": 0.28007, "std": 0.13847}, "punct": {"share": 0.11232, "mean": 0.60793, "std": 0.18874}}, "alpha_p05": 0.1373, "alpha_p50": 0.3675, "alpha_p95": 0.77764}
{"step": 64210, "total_steps": 72479, "tokens": 33664532480, "loss": 3.0212668776512146, "l_mix": 1.8590645492076874, "l_a": 1.9687384516000748, "l_c": 1.9052691012620926, "z_loss": 46.712303161621094, "lr": 0.001990086453781257, "wd": 0.01, "grad_norm": 0.16424962878227234, "tokens_per_second": 1298842.7519416725, "eta_seconds": 3337.8462985754013, "data_wait_seconds": 1.868605613708496e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 64220, "total_steps": 72479, "tokens": 33669775360, "loss": 3.0636028349399567, "l_mix": 1.885627031326294, "l_a": 1.9953102320432663, "l_c": 1.9312757849693298, "z_loss": 46.14185428619385, "lr": 0.001987813205887267, "wd": 0.01, "grad_norm": 0.17164625227451324, "tokens_per_second": 1402084.2029385436, "eta_seconds": 3088.3270654678345, "data_wait_seconds": 2.2977590560913086e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 64230, "total_steps": 72479, "tokens": 33675018240, "loss": 4.4554067850112915, "l_mix": 2.7620929479599, "l_a": 2.8425689935684204, "l_c": 2.8018099069595337, "z_loss": 43.822725772857666, "lr": 0.0019855329597738345, "wd": 0.01, "grad_norm": 0.1160205528140068, "tokens_per_second": 1340686.6825506992, "eta_seconds": 3225.8481927871703, "data_wait_seconds": 1.823902130126953e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 64240, "total_steps": 72479, "tokens": 33680261120, "loss": 4.417162775993347, "l_mix": 2.7378152906894684, "l_a": 2.82678946852684, "l_c": 2.771035522222519, "z_loss": 43.026488304138184, "lr": 0.0019832457344877625, "wd": 0.01, "grad_norm": 0.21607457101345062, "tokens_per_second": 1389398.073219051, "eta_seconds": 3108.978567957878, "data_wait_seconds": 1.8984079360961914e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 64250, "total_steps": 72479, "tokens": 33685504000, "loss": 4.476986110210419, "l_mix": 2.7752647697925568, "l_a": 2.860794812440872, "l_c": 2.8116092681884766, "z_loss": 42.893250942230225, "lr": 0.0019809515491341495, "wd": 0.01, "grad_norm": 0.11857002973556519, "tokens_per_second": 1361468.517752272, "eta_seconds": 3168.9061449050905, "data_wait_seconds": 1.9103288650512695e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 64260, "total_steps": 72479, "tokens": 33690746880, "loss": 3.014266610145569, "l_mix": 1.8545276373624802, "l_a": 1.9637176990509033, "l_c": 1.9020785689353943, "z_loss": 48.27695846557617, "lr": 0.0019786504228762325, "wd": 0.01, "grad_norm": 0.13376854360103607, "tokens_per_second": 1434555.2293368988, "eta_seconds": 3003.804234147072, "data_wait_seconds": 2.1576881408691406e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 64270, "total_steps": 72479, "tokens": 33695989760, "loss": 3.051529288291931, "l_mix": 1.8781251460313797, "l_a": 1.9874599277973175, "l_c": 1.923886999487877, "z_loss": 48.39129018783569, "lr": 0.0019763423749352247, "wd": 0.01, "grad_norm": 0.1684562712907791, "tokens_per_second": 1350496.6327003057, "eta_seconds": 3186.887021994591, "data_wait_seconds": 2.872943878173828e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 64280, "total_steps": 72479, "tokens": 33701232640, "loss": 3.042065143585205, "l_mix": 1.8713657557964325, "l_a": 1.9847560971975327, "l_c": 1.9175751209259033, "z_loss": 47.715415477752686, "lr": 0.001974027424590156, "wd": 0.01, "grad_norm": 0.14562514424324036, "tokens_per_second": 1426175.2967856268, "eta_seconds": 3014.101647734642, "data_wait_seconds": 3.8951635360717773e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 64290, "total_steps": 72479, "tokens": 33706475520, "loss": 3.0327998399734497, "l_mix": 1.8675275295972824, "l_a": 1.9737564027309418, "l_c": 1.9104847013950348, "z_loss": 45.981263160705566, "lr": 0.001971705591177713, "wd": 0.01, "grad_norm": 0.19807922840118408, "tokens_per_second": 1397098.2313279223, "eta_seconds": 3073.079856324196, "data_wait_seconds": 2.187490463256836e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 64300, "total_steps": 72479, "tokens": 33711718400, "loss": 4.597809910774231, "l_mix": 2.852337032556534, "l_a": 2.929952770471573, "l_c": 2.8882898688316345, "z_loss": 42.68703651428223, "lr": 0.001969376894092074, "wd": 0.01, "grad_norm": 0.1460915058851242, "tokens_per_second": 1406117.3421939316, "eta_seconds": 3049.639900827408, "data_wait_seconds": 4.57763671875e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.15977962, "gate_a2b/L1_absmax": 0.68395603, "gate_a2b/L1_rms": 0.16972351, "gate_a2b/L3_absmean": 0.15166458, "gate_a2b/L3_absmax": 0.5007118, "gate_a2b/L3_rms": 0.15731621, "gate_a2b/L5_absmean": 0.21990967, "gate_a2b/L5_absmax": 0.73410654, "gate_a2b/L5_rms": 0.22597396, "gate_c2b/L1_absmean": 0.03980517, "gate_c2b/L1_absmax": 0.62244666, "gate_c2b/L1_rms": 0.0707231, "gate_c2b/L3_absmean": 0.16589268, "gate_c2b/L3_absmax": 0.38773215, "gate_c2b/L3_rms": 0.16941114, "gate_c2b/L5_absmean": 0.46951362, "gate_c2b/L5_absmax": 0.67360163, "gate_c2b/L5_rms": 0.47867408, "gate_b2a/L7_absmean": 0.05133492, "gate_b2a/L7_absmax": 0.3341113, "gate_b2a/L7_rms": 0.05720157, "gate_b2a/L11_absmean": 0.08352314, "gate_b2a/L11_absmax": 0.61668485, "gate_b2a/L11_rms": 0.09270126, "gate_b2a/L14_absmean": 0.21700934, "gate_b2a/L14_absmax": 0.81233978, "gate_b2a/L14_rms": 0.22624519, "gate_b2c/L3_absmean": 0.03585452, "gate_b2c/L3_absmax": 0.55301678, "gate_b2c/L3_rms": 0.04624958, "gate_b2c/L5_absmean": 0.03730362, "gate_b2c/L5_absmax": 0.52899057, "gate_b2c/L5_rms": 0.04643011, "gate_b2c/L6_absmean": 0.05396364, "gate_b2c/L6_absmax": 0.3429395, "gate_b2c/L6_rms": 0.0597869}, "streams": {"cos_a2b/round0": 0.354493, "cos_c2b/round0": 0.23917, "cos_b2a/round0": 0.172216, "cos_b2c/round0": 0.148725, "rms_a/round0": 7.719051, "rms_b/round0": 13.172028, "rms_c/round0": 4.981264, "cos_a2b/round1": 0.415574, "cos_c2b/round1": 0.313821, "cos_b2a/round1": 0.257824, "cos_b2c/round1": 0.11644, "rms_a/round1": 15.243796, "rms_b/round1": 30.339247, "rms_c/round1": 10.964674, "cos_a2b/round2": 0.546513, "cos_c2b/round2": 0.517111, "cos_b2a/round2": 0.101192, "cos_b2c/round2": 0.189757, "rms_a/round2": 43.869629, "rms_b/round2": 83.135239, "rms_c/round2": 21.570927}, "alpha_mean": 0.4498809826231991, "alpha_hist": [0.00587, 0.05328, 0.16176, 0.19058, 0.18726, 0.18901, 0.13741, 0.06688, 0.00787, 8e-05], "alpha_by_type": {"special": {"share": 0.00096, "mean": 0.57613, "std": 0.12344}, "whitespace": {"share": 0.05917, "mean": 0.54411, "std": 0.11858}, "digit": {"share": 0.01898, "mean": 0.4934, "std": 0.13374}, "word_start": {"share": 0.57419, "mean": 0.46542, "std": 0.15841}, "word_cont": {"share": 0.2462, "mean": 0.33292, "std": 0.12924}, "punct": {"share": 0.10049, "mean": 0.58276, "std": 0.15452}}, "alpha_p05": 0.18922, "alpha_p50": 0.44589, "alpha_p95": 0.71083}
{"step": 64310, "total_steps": 72479, "tokens": 33716961280, "loss": 4.606815934181213, "l_mix": 2.8585213720798492, "l_a": 2.931218385696411, "l_c": 2.8964299261569977, "z_loss": 40.49333333969116, "lr": 0.0019670413527847516, "wd": 0.01, "grad_norm": 0.17900539934635162, "tokens_per_second": 1415358.3197229095, "eta_seconds": 3026.0243023395537, "data_wait_seconds": 1.901388168334961e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 64320, "total_steps": 72479, "tokens": 33722204160, "loss": 3.0565950870513916, "l_mix": 1.8810797184705734, "l_a": 1.9902812391519547, "l_c": 1.928103268146515, "z_loss": 46.627952098846436, "lr": 0.0019646989867644253, "wd": 0.01, "grad_norm": 0.16742849349975586, "tokens_per_second": 1421209.9646200663, "eta_seconds": 3009.8760200738907, "data_wait_seconds": 2.950429916381836e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 64330, "total_steps": 72479, "tokens": 33727447040, "loss": 3.0916740596294403, "l_mix": 1.9027747809886932, "l_a": 2.012851744890213, "l_c": 1.950145810842514, "z_loss": 44.93363904953003, "lr": 0.001962349815596784, "wd": 0.01, "grad_norm": 0.2233974039554596, "tokens_per_second": 1345767.8286507453, "eta_seconds": 3174.7102442502974, "data_wait_seconds": 1.7583370208740234e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 64340, "total_steps": 72479, "tokens": 33732689920, "loss": 4.505091190338135, "l_mix": 2.7931941747665405, "l_a": 2.8767835199832916, "l_c": 2.8295395374298096, "z_loss": 42.47469758987427, "lr": 0.0019599938589043584, "wd": 0.01, "grad_norm": 0.13825681805610657, "tokens_per_second": 1353976.1441359168, "eta_seconds": 3151.591739988327, "data_wait_seconds": 1.65402889251709e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 64350, "total_steps": 72479, "tokens": 33737932800, "loss": 4.491642117500305, "l_mix": 2.78509321808815, "l_a": 2.8668556213378906, "l_c": 2.821640580892563, "z_loss": 42.20031023025513, "lr": 0.0019576311363663575, "wd": 0.01, "grad_norm": 0.14920692145824432, "tokens_per_second": 1455721.996416539, "eta_seconds": 2927.713644838333, "data_wait_seconds": 1.582503318786621e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 64360, "total_steps": 72479, "tokens": 33743175680, "loss": 4.357909679412842, "l_mix": 2.7016525268554688, "l_a": 2.7856187224388123, "l_c": 2.735238194465637, "z_loss": 44.62964344024658, "lr": 0.0019552616677185066, "wd": 0.01, "grad_norm": 0.20744888484477997, "tokens_per_second": 1410201.3331629224, "eta_seconds": 3018.501097607613, "data_wait_seconds": 2.0742416381835938e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 64370, "total_steps": 72479, "tokens": 33748418560, "loss": 3.0576557517051697, "l_mix": 1.8814433217048645, "l_a": 1.992355078458786, "l_c": 1.9283527880907059, "z_loss": 47.74236345291138, "lr": 0.0019528854727528802, "wd": 0.01, "grad_norm": 0.15013274550437927, "tokens_per_second": 1412078.9818035737, "eta_seconds": 3010.7745011329653, "data_wait_seconds": 2.6285648345947266e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 64380, "total_steps": 72479, "tokens": 33753661440, "loss": 3.0695437490940094, "l_mix": 1.888851523399353, "l_a": 2.0004834830760956, "l_c": 1.9351570159196854, "z_loss": 48.04952430725098, "lr": 0.0019505025713177381, "wd": 0.01, "grad_norm": 0.17305852472782135, "tokens_per_second": 1462189.344104969, "eta_seconds": 2904.007288193703, "data_wait_seconds": 1.570582389831543e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 64390, "total_steps": 72479, "tokens": 33758904320, "loss": 3.0414777994155884, "l_mix": 1.8729065358638763, "l_a": 1.97990483045578, "l_c": 1.915332779288292, "z_loss": 45.08221960067749, "lr": 0.0019481129833173578, "wd": 0.01, "grad_norm": 0.18060502409934998, "tokens_per_second": 1343750.6755664102, "eta_seconds": 3156.0658603668217, "data_wait_seconds": 2.8640031814575195e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 64400, "total_steps": 72479, "tokens": 33764147200, "loss": 3.106312721967697, "l_mix": 1.9122761934995651, "l_a": 2.0206352919340134, "l_c": 1.9594862312078476, "z_loss": 45.13143301010132, "lr": 0.0019457167287118707, "wd": 0.01, "grad_norm": 0.2652427554130554, "tokens_per_second": 1429096.788895717, "eta_seconds": 2963.9159397125245, "data_wait_seconds": 2.2590160369873047e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16083126, "gate_a2b/L1_absmax": 0.6868664, "gate_a2b/L1_rms": 0.1708013, "gate_a2b/L3_absmean": 0.15180232, "gate_a2b/L3_absmax": 0.49974376, "gate_a2b/L3_rms": 0.15740523, "gate_a2b/L5_absmean": 0.21964133, "gate_a2b/L5_absmax": 0.73479903, "gate_a2b/L5_rms": 0.22569501, "gate_c2b/L1_absmean": 0.04006816, "gate_c2b/L1_absmax": 0.62654757, "gate_c2b/L1_rms": 0.0711396, "gate_c2b/L3_absmean": 0.16594754, "gate_c2b/L3_absmax": 0.38738653, "gate_c2b/L3_rms": 0.16947994, "gate_c2b/L5_absmean": 0.47011462, "gate_c2b/L5_absmax": 0.67450219, "gate_c2b/L5_rms": 0.47928715, "gate_b2a/L7_absmean": 0.05145983, "gate_b2a/L7_absmax": 0.33340535, "gate_b2a/L7_rms": 0.05721081, "gate_b2a/L11_absmean": 0.08358666, "gate_b2a/L11_absmax": 0.61557001, "gate_b2a/L11_rms": 0.09273791, "gate_b2a/L14_absmean": 0.21713115, "gate_b2a/L14_absmax": 0.81497431, "gate_b2a/L14_rms": 0.22640255, "gate_b2c/L3_absmean": 0.03582885, "gate_b2c/L3_absmax": 0.55305189, "gate_b2c/L3_rms": 0.0462385, "gate_b2c/L5_absmean": 0.03742163, "gate_b2c/L5_absmax": 0.52776158, "gate_b2c/L5_rms": 0.04646668, "gate_b2c/L6_absmean": 0.05419196, "gate_b2c/L6_absmax": 0.34233007, "gate_b2c/L6_rms": 0.06008022}, "streams": {"cos_a2b/round0": 0.352963, "cos_c2b/round0": 0.212368, "cos_b2a/round0": 0.152201, "cos_b2c/round0": 0.128511, "rms_a/round0": 8.063267, "rms_b/round0": 13.421353, "rms_c/round0": 5.087956, "cos_a2b/round1": 0.409995, "cos_c2b/round1": 0.307133, "cos_b2a/round1": 0.240849, "cos_b2c/round1": 0.117812, "rms_a/round1": 16.222956, "rms_b/round1": 32.188457, "rms_c/round1": 11.629503, "cos_a2b/round2": 0.526738, "cos_c2b/round2": 0.50709, "cos_b2a/round2": 0.089956, "cos_b2c/round2": 0.174715, "rms_a/round2": 45.680538, "rms_b/round2": 85.640831, "rms_c/round2": 22.873812}, "alpha_mean": 0.42276747772100975, "alpha_hist": [0.02802, 0.1387, 0.18555, 0.15521, 0.12964, 0.11969, 0.11334, 0.08717, 0.03958, 0.0031], "alpha_by_type": {"special": {"share": 0.00119, "mean": 0.62023, "std": 0.11391}, "whitespace": {"share": 0.03871, "mean": 0.61358, "std": 0.16624}, "digit": {"share": 0.00667, "mean": 0.38805, "std": 0.1907}, "word_start": {"share": 0.58801, "mean": 0.42544, "std": 0.20089}, "word_cont": {"share": 0.24084, "mean": 0.28252, "std": 0.14385}, "punct": {"share": 0.12457, "mean": 0.622, "std": 0.17508}}, "alpha_p05": 0.12443, "alpha_p50": 0.36841, "alpha_p95": 0.78133}
{"step": 64410, "total_steps": 72479, "tokens": 33769390080, "loss": 2.715240180492401, "l_mix": 1.6809544265270233, "l_a": 1.7421645820140839, "l_c": 1.7054544538259506, "z_loss": 53.7135214805603, "lr": 0.0019433138275170936, "wd": 0.01, "grad_norm": 0.21579982340335846, "tokens_per_second": 1443358.7170245412, "eta_seconds": 2930.9968631505967, "data_wait_seconds": 2.7477741241455078e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 64420, "total_steps": 72479, "tokens": 33774632960, "loss": 4.40345686674118, "l_mix": 2.7296365797519684, "l_a": 2.808816075325012, "l_c": 2.7705847322940826, "z_loss": 44.43295192718506, "lr": 0.0019409042998043608, "wd": 0.01, "grad_norm": 0.1661863923072815, "tokens_per_second": 1389532.22265366, "eta_seconds": 3040.762152266502, "data_wait_seconds": 1.8537044525146484e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 64430, "total_steps": 72479, "tokens": 33779875840, "loss": 4.42130047082901, "l_mix": 2.741092622280121, "l_a": 2.8232553899288177, "l_c": 2.7774369418621063, "z_loss": 43.153520584106445, "lr": 0.0019384881657003603, "wd": 0.01, "grad_norm": 0.1457148641347885, "tokens_per_second": 1430134.6644210517, "eta_seconds": 2950.766957116127, "data_wait_seconds": 1.946091651916504e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 64440, "total_steps": 72479, "tokens": 33785118720, "loss": 4.390449166297913, "l_mix": 2.721875488758087, "l_a": 2.8043502271175385, "l_c": 2.7575618028640747, "z_loss": 43.626450061798096, "lr": 0.0019360654453869604, "wd": 0.01, "grad_norm": 0.14595578610897064, "tokens_per_second": 1451971.6467416093, "eta_seconds": 2902.777916812897, "data_wait_seconds": 2.568960189819336e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 64450, "total_steps": 72479, "tokens": 33790361600, "loss": 4.437798738479614, "l_mix": 2.7510440945625305, "l_a": 2.83519583940506, "l_c": 2.787319451570511, "z_loss": 42.56272792816162, "lr": 0.0019336361591010454, "wd": 0.01, "grad_norm": 0.13101984560489655, "tokens_per_second": 1445202.563503297, "eta_seconds": 2912.7462532281875, "data_wait_seconds": 2.0742416381835938e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 64460, "total_steps": 72479, "tokens": 33795604480, "loss": 4.3584442138671875, "l_mix": 2.7014219164848328, "l_a": 2.7839296758174896, "l_c": 2.73947736620903, "z_loss": 43.68441152572632, "lr": 0.001931200327134344, "wd": 0.01, "grad_norm": 0.17013034224510193, "tokens_per_second": 1384465.4144188755, "eta_seconds": 3036.7428671121597, "data_wait_seconds": 2.0265579223632812e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 64470, "total_steps": 72479, "tokens": 33800847360, "loss": 4.3785558342933655, "l_mix": 2.7141820788383484, "l_a": 2.7960971295833588, "l_c": 2.7518153488636017, "z_loss": 43.53837585449219, "lr": 0.0019287579698332607, "wd": 0.01, "grad_norm": 0.2155819535255432, "tokens_per_second": 1396567.6398389856, "eta_seconds": 3006.673269677162, "data_wait_seconds": 2.631545066833496e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 64480, "total_steps": 72479, "tokens": 33806090240, "loss": 3.097460061311722, "l_mix": 1.906297743320465, "l_a": 2.0189583599567413, "l_c": 1.9515826404094696, "z_loss": 46.523075580596924, "lr": 0.0019263091075987066, "wd": 0.01, "grad_norm": 0.14944587647914886, "tokens_per_second": 1422363.36177685, "eta_seconds": 2948.4587586402895, "data_wait_seconds": 2.637505531311035e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 64490, "total_steps": 72479, "tokens": 33811333120, "loss": 3.0692930817604065, "l_mix": 1.8885633796453476, "l_a": 2.0015444606542587, "l_c": 1.9342211335897446, "z_loss": 48.657015323638916, "lr": 0.0019238537608859273, "wd": 0.01, "grad_norm": 0.2235131561756134, "tokens_per_second": 1395031.6435709642, "eta_seconds": 3002.4672567844386, "data_wait_seconds": 2.54213809967041e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 64500, "total_steps": 72479, "tokens": 33816576000, "loss": 3.0453215539455414, "l_mix": 1.8748111575841904, "l_a": 1.9804807603359222, "l_c": 1.9212202429771423, "z_loss": 47.4241738319397, "lr": 0.0019213919502043344, "wd": 0.01, "grad_norm": 0.1813281625509262, "tokens_per_second": 1333864.8045277894, "eta_seconds": 3136.220355916023, "data_wait_seconds": 1.9669532775878906e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16039674, "gate_a2b/L1_absmax": 0.68639076, "gate_a2b/L1_rms": 0.17039925, "gate_a2b/L3_absmean": 0.15157211, "gate_a2b/L3_absmax": 0.50188172, "gate_a2b/L3_rms": 0.15724173, "gate_a2b/L5_absmean": 0.21920624, "gate_a2b/L5_absmax": 0.734882, "gate_a2b/L5_rms": 0.22525845, "gate_c2b/L1_absmean": 0.03989865, "gate_c2b/L1_absmax": 0.62712252, "gate_c2b/L1_rms": 0.07093826, "gate_c2b/L3_absmean": 0.1657892, "gate_c2b/L3_absmax": 0.39032775, "gate_c2b/L3_rms": 0.16928114, "gate_c2b/L5_absmean": 0.4697884, "gate_c2b/L5_absmax": 0.67407769, "gate_c2b/L5_rms": 0.47898212, "gate_b2a/L7_absmean": 0.05121044, "gate_b2a/L7_absmax": 0.33318821, "gate_b2a/L7_rms": 0.05704538, "gate_b2a/L11_absmean": 0.0834793, "gate_b2a/L11_absmax": 0.61515194, "gate_b2a/L11_rms": 0.0925971, "gate_b2a/L14_absmean": 0.21653581, "gate_b2a/L14_absmax": 0.81641561, "gate_b2a/L14_rms": 0.22586089, "gate_b2c/L3_absmean": 0.03572122, "gate_b2c/L3_absmax": 0.55199897, "gate_b2c/L3_rms": 0.04616667, "gate_b2c/L5_absmean": 0.03751313, "gate_b2c/L5_absmax": 0.52812022, "gate_b2c/L5_rms": 0.04651496, "gate_b2c/L6_absmean": 0.05390298, "gate_b2c/L6_absmax": 0.3387773, "gate_b2c/L6_rms": 0.0597217}, "streams": {"cos_a2b/round0": 0.353712, "cos_c2b/round0": 0.207575, "cos_b2a/round0": 0.152962, "cos_b2c/round0": 0.12782, "rms_a/round0": 8.031233, "rms_b/round0": 13.481626, "rms_c/round0": 5.141536, "cos_a2b/round1": 0.408854, "cos_c2b/round1": 0.306874, "cos_b2a/round1": 0.245615, "cos_b2c/round1": 0.116782, "rms_a/round1": 16.075956, "rms_b/round1": 32.303585, "rms_c/round1": 11.735874, "cos_a2b/round2": 0.517116, "cos_c2b/round2": 0.510159, "cos_b2a/round2": 0.08877, "cos_b2c/round2": 0.175246, "rms_a/round2": 45.001602, "rms_b/round2": 85.373055, "rms_c/round2": 23.04215}, "alpha_mean": 0.4098795680629479, "alpha_hist": [0.03345, 0.16862, 0.19478, 0.14485, 0.11378, 0.10594, 0.1031, 0.08676, 0.04298, 0.00572], "alpha_by_type": {"special": {"share": 0.0011, "mean": 0.66761, "std": 0.10158}, "whitespace": {"share": 0.03119, "mean": 0.65912, "std": 0.17838}, "digit": {"share": 0.00806, "mean": 0.43019, "std": 0.18471}, "word_start": {"share": 0.57935, "mean": 0.41064, "std": 0.21054}, "word_cont": {"share": 0.26875, "mean": 0.28266, "std": 0.14596}, "punct": {"share": 0.11156, "mean": 0.63871, "std": 0.18179}}, "alpha_p05": 0.10971, "alpha_p50": 0.3675, "alpha_p95": 0.79974}
{"step": 64510, "total_steps": 72479, "tokens": 33821818880, "loss": 3.035132974386215, "l_mix": 1.8679896742105484, "l_a": 1.977572113275528, "l_c": 1.912905365228653, "z_loss": 46.67034387588501, "lr": 0.001918923696117331, "wd": 0.01, "grad_norm": 0.16850648820400238, "tokens_per_second": 1464489.798229538, "eta_seconds": 2852.905549120903, "data_wait_seconds": 2.8699636459350586e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 64520, "total_steps": 72479, "tokens": 33827061760, "loss": 3.195304125547409, "l_mix": 1.9803397357463837, "l_a": 2.0444424003362656, "l_c": 2.005438670516014, "z_loss": 53.11656427383423, "lr": 0.0019164490192421424, "wd": 0.01, "grad_norm": 0.33252301812171936, "tokens_per_second": 1401518.4648595746, "eta_seconds": 2977.347995495796, "data_wait_seconds": 2.378225326538086e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 64530, "total_steps": 72479, "tokens": 33832304640, "loss": 4.451884984970093, "l_mix": 2.7601478695869446, "l_a": 2.8416718542575836, "l_c": 2.797451823949814, "z_loss": 44.02574110031128, "lr": 0.001913967940249644, "wd": 0.01, "grad_norm": 0.13722127676010132, "tokens_per_second": 1342474.6335427933, "eta_seconds": 3104.3903608083724, "data_wait_seconds": 1.8358230590820312e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 64540, "total_steps": 72479, "tokens": 33837547520, "loss": 4.5324419140815735, "l_mix": 2.811032921075821, "l_a": 2.889357477426529, "l_c": 2.848672389984131, "z_loss": 42.71455430984497, "lr": 0.0019114804798641867, "wd": 0.01, "grad_norm": 0.1878262311220169, "tokens_per_second": 1452360.3151938596, "eta_seconds": 2865.9020688295363, "data_wait_seconds": 1.5795230865478516e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 64550, "total_steps": 72479, "tokens": 33842790400, "loss": 4.4122307896614075, "l_mix": 2.7350330650806427, "l_a": 2.8179719746112823, "l_c": 2.772686779499054, "z_loss": 43.443203926086426, "lr": 0.0019089866588634257, "wd": 0.01, "grad_norm": 0.17953041195869446, "tokens_per_second": 1411613.9716029153, "eta_seconds": 2944.912444639206, "data_wait_seconds": 1.767277717590332e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 64560, "total_steps": 72479, "tokens": 33848033280, "loss": 4.464357554912567, "l_mix": 2.768642395734787, "l_a": 2.8486209511756897, "l_c": 2.8037624955177307, "z_loss": 43.41624927520752, "lr": 0.0019064864980781458, "wd": 0.01, "grad_norm": 0.12306838482618332, "tokens_per_second": 1471319.8077843704, "eta_seconds": 2821.845155644417, "data_wait_seconds": 1.9431114196777344e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 64570, "total_steps": 72479, "tokens": 33853276160, "loss": 4.358148455619812, "l_mix": 2.7014095187187195, "l_a": 2.783380836248398, "l_c": 2.7390820682048798, "z_loss": 45.41735315322876, "lr": 0.0019039800183920871, "wd": 0.01, "grad_norm": 0.16085994243621826, "tokens_per_second": 1437805.284746185, "eta_seconds": 2883.9745103120804, "data_wait_seconds": 1.7970800399780273e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 64580, "total_steps": 72479, "tokens": 33858519040, "loss": 4.687835991382599, "l_mix": 2.9099434912204742, "l_a": 2.977540910243988, "l_c": 2.9487671554088593, "z_loss": 42.1278862953186, "lr": 0.0019014672407417727, "wd": 0.01, "grad_norm": 0.13023316860198975, "tokens_per_second": 1440945.4543710346, "eta_seconds": 2874.051130414009, "data_wait_seconds": 2.0205974578857422e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 64590, "total_steps": 72479, "tokens": 33863761920, "loss": 4.826225161552429, "l_mix": 2.995789259672165, "l_a": 3.069034159183502, "l_c": 3.032418519258499, "z_loss": 40.74537706375122, "lr": 0.001898948186116331, "wd": 0.01, "grad_norm": 0.1979255974292755, "tokens_per_second": 1456358.873464349, "eta_seconds": 2840.0335297584534, "data_wait_seconds": 2.536177635192871e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 64600, "total_steps": 72479, "tokens": 33869004800, "loss": 3.0720706582069397, "l_mix": 1.8921002894639969, "l_a": 1.9982595890760422, "l_c": 1.934974804520607, "z_loss": 45.69022464752197, "lr": 0.0018964228755573226, "wd": 0.01, "grad_norm": 0.2709755003452301, "tokens_per_second": 1362317.9502182442, "eta_seconds": 3032.2327848196032, "data_wait_seconds": 1.558661460876465e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16081293, "gate_a2b/L1_absmax": 0.68527126, "gate_a2b/L1_rms": 0.17058188, "gate_a2b/L3_absmean": 0.15196617, "gate_a2b/L3_absmax": 0.50095177, "gate_a2b/L3_rms": 0.15757222, "gate_a2b/L5_absmean": 0.21959071, "gate_a2b/L5_absmax": 0.73472363, "gate_a2b/L5_rms": 0.22563031, "gate_c2b/L1_absmean": 0.0399454, "gate_c2b/L1_absmax": 0.62706769, "gate_c2b/L1_rms": 0.07099397, "gate_c2b/L3_absmean": 0.1658376, "gate_c2b/L3_absmax": 0.38901702, "gate_c2b/L3_rms": 0.16934119, "gate_c2b/L5_absmean": 0.47001886, "gate_c2b/L5_absmax": 0.67508924, "gate_c2b/L5_rms": 0.47916594, "gate_b2a/L7_absmean": 0.05122539, "gate_b2a/L7_absmax": 0.33296558, "gate_b2a/L7_rms": 0.05706821, "gate_b2a/L11_absmean": 0.08360215, "gate_b2a/L11_absmax": 0.6146996, "gate_b2a/L11_rms": 0.09269916, "gate_b2a/L14_absmean": 0.21681395, "gate_b2a/L14_absmax": 0.81514841, "gate_b2a/L14_rms": 0.22608167, "gate_b2c/L3_absmean": 0.03573713, "gate_b2c/L3_absmax": 0.55269575, "gate_b2c/L3_rms": 0.04619997, "gate_b2c/L5_absmean": 0.03750606, "gate_b2c/L5_absmax": 0.52845049, "gate_b2c/L5_rms": 0.04654281, "gate_b2c/L6_absmean": 0.05422575, "gate_b2c/L6_absmax": 0.33989483, "gate_b2c/L6_rms": 0.06004354}, "streams": {"cos_a2b/round0": 0.353854, "cos_c2b/round0": 0.209263, "cos_b2a/round0": 0.155571, "cos_b2c/round0": 0.128598, "rms_a/round0": 8.016655, "rms_b/round0": 13.469022, "rms_c/round0": 5.125047, "cos_a2b/round1": 0.411205, "cos_c2b/round1": 0.308066, "cos_b2a/round1": 0.245648, "cos_b2c/round1": 0.118026, "rms_a/round1": 16.140966, "rms_b/round1": 32.23209, "rms_c/round1": 11.67723, "cos_a2b/round2": 0.523847, "cos_c2b/round2": 0.508941, "cos_b2a/round2": 0.089873, "cos_b2c/round2": 0.176563, "rms_a/round2": 45.37973, "rms_b/round2": 85.406693, "rms_c/round2": 22.999035}, "alpha_mean": 0.43486377737269777, "alpha_hist": [0.01851, 0.12831, 0.19337, 0.15594, 0.12437, 0.12128, 0.10921, 0.09387, 0.04988, 0.00525], "alpha_by_type": {"special": {"share": 0.00117, "mean": 0.67203, "std": 0.11361}, "whitespace": {"share": 0.03595, "mean": 0.63981, "std": 0.15944}, "digit": {"share": 0.01183, "mean": 0.46538, "std": 0.16216}, "word_start": {"share": 0.58318, "mean": 0.43733, "std": 0.20352}, "word_cont": {"share": 0.25165, "mean": 0.2965, "std": 0.14297}, "punct": {"share": 0.11623, "mean": 0.65318, "std": 0.17349}}, "alpha_p05": 0.13707, "alpha_p50": 0.42441, "alpha_p95": 0.81108}
{"step": 64610, "total_steps": 72479, "tokens": 33874247680, "loss": 3.0552320182323456, "l_mix": 1.879989042878151, "l_a": 1.990760937333107, "l_c": 1.9267155826091766, "z_loss": 47.62053918838501, "lr": 0.0018938913301585632, "wd": 0.01, "grad_norm": 0.17788954079151154, "tokens_per_second": 1424148.0204241, "eta_seconds": 2896.9055272579194, "data_wait_seconds": 1.6361474990844727e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 64620, "total_steps": 72479, "tokens": 33879490560, "loss": 3.059031456708908, "l_mix": 1.8824537545442581, "l_a": 1.9939614832401276, "l_c": 1.9279641509056091, "z_loss": 48.08769226074219, "lr": 0.0018913535710659481, "wd": 0.01, "grad_norm": 0.18240736424922943, "tokens_per_second": 1411150.8070150716, "eta_seconds": 2919.871761059761, "data_wait_seconds": 2.5451183319091797e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 64630, "total_steps": 72479, "tokens": 33884733440, "loss": 2.957888513803482, "l_mix": 1.8301741182804108, "l_a": 1.898028388619423, "l_c": 1.8610195368528366, "z_loss": 55.31377172470093, "lr": 0.0018888096194772754, "wd": 0.01, "grad_norm": 0.21661531925201416, "tokens_per_second": 1426169.3771609983, "eta_seconds": 2885.447253251076, "data_wait_seconds": 1.96993350982666e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 64640, "total_steps": 72479, "tokens": 33889976320, "loss": 4.401305258274078, "l_mix": 2.7287726998329163, "l_a": 2.809143602848053, "l_c": 2.7659647166728973, "z_loss": 44.33253717422485, "lr": 0.0018862594966420683, "wd": 0.01, "grad_norm": 0.13505928218364716, "tokens_per_second": 1386774.9955521508, "eta_seconds": 2963.634075593948, "data_wait_seconds": 1.8358230590820312e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 64650, "total_steps": 72479, "tokens": 33895219200, "loss": 4.417405843734741, "l_mix": 2.7388822734355927, "l_a": 2.8198746740818024, "l_c": 2.775203585624695, "z_loss": 44.76894664764404, "lr": 0.0018837032238613982, "wd": 0.01, "grad_norm": 0.19341064989566803, "tokens_per_second": 1392017.9632049042, "eta_seconds": 2948.705304455757, "data_wait_seconds": 3.281235694885254e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 64660, "total_steps": 72479, "tokens": 33900462080, "loss": 4.392976105213165, "l_mix": 2.7231721580028534, "l_a": 2.80574694275856, "l_c": 2.7602660953998566, "z_loss": 43.64283037185669, "lr": 0.0018811408224877076, "wd": 0.01, "grad_norm": 0.13698913156986237, "tokens_per_second": 1429773.6024270135, "eta_seconds": 2867.17272233963, "data_wait_seconds": 1.7762184143066406e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 64670, "total_steps": 72479, "tokens": 33905704960, "loss": 4.4044628739356995, "l_mix": 2.7312135994434357, "l_a": 2.8111522495746613, "l_c": 2.7663451731204987, "z_loss": 43.36239671707153, "lr": 0.0018785723139246304, "wd": 0.01, "grad_norm": 0.14766192436218262, "tokens_per_second": 1461006.7828479218, "eta_seconds": 2802.2902015686036, "data_wait_seconds": 2.3066997528076172e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 64680, "total_steps": 72479, "tokens": 33910947840, "loss": 4.393026947975159, "l_mix": 2.7235421240329742, "l_a": 2.8058885633945465, "l_c": 2.759060651063919, "z_loss": 44.52141857147217, "lr": 0.0018759977196268138, "wd": 0.01, "grad_norm": 0.1831279695034027, "tokens_per_second": 1452733.8361796, "eta_seconds": 2814.6395507335665, "data_wait_seconds": 2.7835369110107422e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 64690, "total_steps": 72479, "tokens": 33916190720, "loss": 2.957666039466858, "l_mix": 1.83144810795784, "l_a": 1.8981806188821793, "l_c": 1.8558789789676666, "z_loss": 54.61098003387451, "lr": 0.001873417061099738, "wd": 0.01, "grad_norm": 0.1405215561389923, "tokens_per_second": 1441180.9770264812, "eta_seconds": 2833.564484333992, "data_wait_seconds": 2.6464462280273438e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 64700, "total_steps": 72479, "tokens": 33921433600, "loss": 4.782308757305145, "l_mix": 2.9681206047534943, "l_a": 3.042199581861496, "l_c": 3.0050941109657288, "z_loss": 42.98922538757324, "lr": 0.0018708303598995383, "wd": 0.01, "grad_norm": 0.14464090764522552, "tokens_per_second": 1429076.8212962558, "eta_seconds": 2853.895809674263, "data_wait_seconds": 1.615285873413086e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.16054225, "gate_a2b/L1_absmax": 0.68421835, "gate_a2b/L1_rms": 0.17042617, "gate_a2b/L3_absmean": 0.15169027, "gate_a2b/L3_absmax": 0.50143486, "gate_a2b/L3_rms": 0.15729254, "gate_a2b/L5_absmean": 0.21999617, "gate_a2b/L5_absmax": 0.73450845, "gate_a2b/L5_rms": 0.22604537, "gate_c2b/L1_absmean": 0.03964325, "gate_c2b/L1_absmax": 0.6265707, "gate_c2b/L1_rms": 0.07091923, "gate_c2b/L3_absmean": 0.16564091, "gate_c2b/L3_absmax": 0.38910681, "gate_c2b/L3_rms": 0.16918415, "gate_c2b/L5_absmean": 0.46930408, "gate_c2b/L5_absmax": 0.67442316, "gate_c2b/L5_rms": 0.47847328, "gate_b2a/L7_absmean": 0.05152982, "gate_b2a/L7_absmax": 0.33333221, "gate_b2a/L7_rms": 0.0573141, "gate_b2a/L11_absmean": 0.08353107, "gate_b2a/L11_absmax": 0.61526138, "gate_b2a/L11_rms": 0.09268326, "gate_b2a/L14_absmean": 0.21725094, "gate_b2a/L14_absmax": 0.81415749, "gate_b2a/L14_rms": 0.2264846, "gate_b2c/L3_absmean": 0.0359442, "gate_b2c/L3_absmax": 0.55221808, "gate_b2c/L3_rms": 0.04633867, "gate_b2c/L5_absmean": 0.03736963, "gate_b2c/L5_absmax": 0.52742505, "gate_b2c/L5_rms": 0.04640502, "gate_b2c/L6_absmean": 0.05438748, "gate_b2c/L6_absmax": 0.34062377, "gate_b2c/L6_rms": 0.06018334}, "streams": {"cos_a2b/round0": 0.35338, "cos_c2b/round0": 0.235411, "cos_b2a/round0": 0.164208, "cos_b2c/round0": 0.14672, "rms_a/round0": 7.620794, "rms_b/round0": 13.126427, "rms_c/round0": 4.987387, "cos_a2b/round1": 0.409277, "cos_c2b/round1": 0.312113, "cos_b2a/round1": 0.253026, "cos_b2c/round1": 0.113198, "rms_a/round1": 14.934613, "rms_b/round1": 30.069616, "rms_c/round1": 10.824794, "cos_a2b/round2": 0.545896, "cos_c2b/round2": 0.515424, "cos_b2a/round2": 0.090845, "cos_b2c/round2": 0.19047, "rms_a/round2": 43.92905, "rms_b/round2": 83.110558, "rms_c/round2": 20.974367}, "alpha_mean": 0.443594899925408, "alpha_hist": [0.00603, 0.05467, 0.14552, 0.19894, 0.23016, 0.18266, 0.12004, 0.05612, 0.00578, 6e-05], "alpha_by_type": {"special": {"share": 0.0005, "mean": 0.50678, "std": 0.15047}, "whitespace": {"share": 0.05382, "mean": 0.54029, "std": 0.11375}, "digit": {"share": 0.02597, "mean": 0.46796, "std": 0.11527}, "word_start": {"share": 0.57011, "mean": 0.45907, "std": 0.15084}, "word_cont": {"share": 0.25621, "mean": 0.33583, "std": 0.12333}, "punct": {"share": 0.09338, "mean": 0.58193, "std": 0.1492}}, "alpha_p05": 0.18097, "alpha_p50": 0.42656, "alpha_p95": 0.70821}
{"step": 64710, "total_steps": 72479, "tokens": 33926676480, "loss": 4.738033175468445, "l_mix": 2.93987774848938, "l_a": 3.016019284725189, "l_c": 2.9778319895267487, "z_loss": 43.53900194168091, "lr": 0.0018682376376328244, "wd": 0.01, "grad_norm": 0.19524359703063965, "tokens_per_second": 1464006.1065199312, "eta_seconds": 2782.2243731498716, "data_wait_seconds": 1.9669532775878906e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 64720, "total_steps": 72479, "tokens": 33931919360, "loss": 4.674976170063019, "l_mix": 2.9017366766929626, "l_a": 2.973435193300247, "l_c": 2.937363177537918, "z_loss": 42.247920989990234, "lr": 0.0018656389159564988, "wd": 0.01, "grad_norm": 0.20895203948020935, "tokens_per_second": 1423378.4858338768, "eta_seconds": 2857.954249334335, "data_wait_seconds": 2.390146255493164e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 64730, "total_steps": 72479, "tokens": 33937162240, "loss": 4.7689074873924255, "l_mix": 2.959535449743271, "l_a": 3.0332387685775757, "l_c": 2.9980013370513916, "z_loss": 41.42129182815552, "lr": 0.0018630342165775783, "wd": 0.01, "grad_norm": 0.20845408737659454, "tokens_per_second": 1418040.2707261096, "eta_seconds": 2865.0157515764236, "data_wait_seconds": 1.913309097290039e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 64740, "total_steps": 72479, "tokens": 33942405120, "loss": 3.0085633993148804, "l_mix": 1.8514300137758255, "l_a": 1.9588456600904465, "l_c": 1.8982655107975006, "z_loss": 48.144282817840576, "lr": 0.001860423561253009, "wd": 0.01, "grad_norm": 0.1678914874792099, "tokens_per_second": 1367433.9962335334, "eta_seconds": 2967.2107342481613, "data_wait_seconds": 1.7434358596801758e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 64750, "total_steps": 72479, "tokens": 33947648000, "loss": 4.417391717433929, "l_mix": 2.7387570440769196, "l_a": 2.8211061358451843, "l_c": 2.7743423879146576, "z_loss": 43.04282236099243, "lr": 0.0018578069717894894, "wd": 0.01, "grad_norm": 0.11222914606332779, "tokens_per_second": 1455021.9399319831, "eta_seconds": 2784.9902745723725, "data_wait_seconds": 3.173947334289551e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 64760, "total_steps": 72479, "tokens": 33952890880, "loss": 4.37808895111084, "l_mix": 2.713857054710388, "l_a": 2.797312617301941, "l_c": 2.750127136707306, "z_loss": 44.276411056518555, "lr": 0.0018551844700432833, "wd": 0.01, "grad_norm": 0.14410404860973358, "tokens_per_second": 1385722.14362584, "eta_seconds": 2920.483800172806, "data_wait_seconds": 1.749396324157715e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 64770, "total_steps": 72479, "tokens": 33958133760, "loss": 4.520868599414825, "l_mix": 2.8037708401679993, "l_a": 2.8836087584495544, "l_c": 2.840050369501114, "z_loss": 43.8422064781189, "lr": 0.001852556077920041, "wd": 0.01, "grad_norm": 0.12996520102024078, "tokens_per_second": 1301703.3935042669, "eta_seconds": 3104.9594033241274, "data_wait_seconds": 2.2739171981811523e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 64780, "total_steps": 72479, "tokens": 33963376640, "loss": 4.439754366874695, "l_mix": 2.752064138650894, "l_a": 2.8367514610290527, "l_c": 2.7888823449611664, "z_loss": 43.518327713012695, "lr": 0.0018499218173746138, "wd": 0.01, "grad_norm": 0.1477067768573761, "tokens_per_second": 1378566.5050699555, "eta_seconds": 2928.0366940259933, "data_wait_seconds": 2.2172927856445312e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 64790, "total_steps": 72479, "tokens": 33968619520, "loss": 4.435981810092926, "l_mix": 2.7505146265029907, "l_a": 2.833340674638748, "l_c": 2.784883141517639, "z_loss": 43.98820495605469, "lr": 0.001847281710410873, "wd": 0.01, "grad_norm": 0.15025843679904938, "tokens_per_second": 1464485.2142847534, "eta_seconds": 2752.6740404605866, "data_wait_seconds": 1.9758939743041992e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 64800, "total_steps": 72479, "tokens": 33973862400, "loss": 4.354587376117706, "l_mix": 2.6992488503456116, "l_a": 2.78039887547493, "l_c": 2.737395703792572, "z_loss": 44.39787483215332, "lr": 0.0018446357790815242, "wd": 0.01, "grad_norm": 0.1514979600906372, "tokens_per_second": 1330931.944856544, "eta_seconds": 3024.953730773926, "data_wait_seconds": 2.2351741790771484e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.16080487, "gate_a2b/L1_absmax": 0.68437058, "gate_a2b/L1_rms": 0.17063953, "gate_a2b/L3_absmean": 0.15169553, "gate_a2b/L3_absmax": 0.49924102, "gate_a2b/L3_rms": 0.15731803, "gate_a2b/L5_absmean": 0.21992789, "gate_a2b/L5_absmax": 0.73529607, "gate_a2b/L5_rms": 0.22595778, "gate_c2b/L1_absmean": 0.04005494, "gate_c2b/L1_absmax": 0.62632996, "gate_c2b/L1_rms": 0.07092664, "gate_c2b/L3_absmean": 0.16545372, "gate_c2b/L3_absmax": 0.38808668, "gate_c2b/L3_rms": 0.16897348, "gate_c2b/L5_absmean": 0.46937785, "gate_c2b/L5_absmax": 0.67565256, "gate_c2b/L5_rms": 0.47848508, "gate_b2a/L7_absmean": 0.05115033, "gate_b2a/L7_absmax": 0.33349937, "gate_b2a/L7_rms": 0.05698868, "gate_b2a/L11_absmean": 0.08379738, "gate_b2a/L11_absmax": 0.61490965, "gate_b2a/L11_rms": 0.09287149, "gate_b2a/L14_absmean": 0.2175235, "gate_b2a/L14_absmax": 0.8127411, "gate_b2a/L14_rms": 0.22667785, "gate_b2c/L3_absmean": 0.03616247, "gate_b2c/L3_absmax": 0.55249518, "gate_b2c/L3_rms": 0.04648596, "gate_b2c/L5_absmean": 0.03762194, "gate_b2c/L5_absmax": 0.52669156, "gate_b2c/L5_rms": 0.04656975, "gate_b2c/L6_absmean": 0.05434006, "gate_b2c/L6_absmax": 0.34050465, "gate_b2c/L6_rms": 0.06013189}, "streams": {"cos_a2b/round0": 0.357683, "cos_c2b/round0": 0.243971, "cos_b2a/round0": 0.158354, "cos_b2c/round0": 0.151462, "rms_a/round0": 7.710791, "rms_b/round0": 13.159683, "rms_c/round0": 4.984558, "cos_a2b/round1": 0.408473, "cos_c2b/round1": 0.300061, "cos_b2a/round1": 0.246936, "cos_b2c/round1": 0.107601, "rms_a/round1": 15.19663, "rms_b/round1": 30.773766, "rms_c/round1": 10.784139, "cos_a2b/round2": 0.530228, "cos_c2b/round2": 0.519461, "cos_b2a/round2": 0.098329, "cos_b2c/round2": 0.18086, "rms_a/round2": 43.875156, "rms_b/round2": 84.651855, "rms_c/round2": 21.207197}, "alpha_mean": 0.424805078594229, "alpha_hist": [0.01083, 0.07874, 0.17932, 0.20566, 0.18864, 0.15767, 0.11311, 0.05797, 0.008, 6e-05], "alpha_by_type": {"special": {"share": 0.00093, "mean": 0.51099, "std": 0.15096}, "whitespace": {"share": 0.0303, "mean": 0.53389, "std": 0.13755}, "digit": {"share": 0.05966, "mean": 0.45104, "std": 0.12737}, "word_start": {"share": 0.55727, "mean": 0.43639, "std": 0.16369}, "word_cont": {"share": 0.24561, "mean": 0.31376, "std": 0.12783}, "punct": {"share": 0.10623, "mean": 0.57416, "std": 0.15857}}, "alpha_p05": 0.15003, "alpha_p50": 0.40781, "alpha_p95": 0.71283}
{"step": 64810, "total_steps": 72479, "tokens": 33979105280, "loss": 3.341124325990677, "l_mix": 2.0701486468315125, "l_a": 2.138268545269966, "l_c": 2.098316863179207, "z_loss": 52.66870450973511, "lr": 0.0018419840454879225, "wd": 0.01, "grad_norm": 0.19169805943965912, "tokens_per_second": 1461723.2994049606, "eta_seconds": 2750.701636648178, "data_wait_seconds": 2.2143125534057617e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 64820, "total_steps": 72479, "tokens": 33984348160, "loss": 4.77405309677124, "l_mix": 2.963316857814789, "l_a": 3.038430154323578, "l_c": 2.9973570704460144, "z_loss": 42.37601613998413, "lr": 0.0018393265317798915, "wd": 0.01, "grad_norm": 0.1698349416255951, "tokens_per_second": 1367853.9286489154, "eta_seconds": 2935.636406707764, "data_wait_seconds": 3.7044286727905273e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 64830, "total_steps": 72479, "tokens": 33989591040, "loss": 4.68751448392868, "l_mix": 2.908776193857193, "l_a": 2.983846068382263, "l_c": 2.945281147956848, "z_loss": 41.66374206542969, "lr": 0.0018366632601555333, "wd": 0.01, "grad_norm": 0.15075792372226715, "tokens_per_second": 1419676.6887111266, "eta_seconds": 2824.7832368373874, "data_wait_seconds": 2.1457672119140625e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 64840, "total_steps": 72479, "tokens": 33994833920, "loss": 4.598430693149567, "l_mix": 2.8536936938762665, "l_a": 2.927191376686096, "l_c": 2.888598471879959, "z_loss": 43.27506351470947, "lr": 0.0018339942528610479, "wd": 0.01, "grad_norm": 0.12700021266937256, "tokens_per_second": 1297778.3529861954, "eta_seconds": 3086.0709171056747, "data_wait_seconds": 2.092123031616211e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 64850, "total_steps": 72479, "tokens": 34000076800, "loss": 4.760395407676697, "l_mix": 2.9549241065979004, "l_a": 3.027994990348816, "l_c": 2.9902427196502686, "z_loss": 41.42509460449219, "lr": 0.0018313195321905435, "wd": 0.01, "grad_norm": 0.14456889033317566, "tokens_per_second": 1467433.4357158176, "eta_seconds": 2725.706703042984, "data_wait_seconds": 2.5957822799682617e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 64860, "total_steps": 72479, "tokens": 34005319680, "loss": 4.724015414714813, "l_mix": 2.9314657747745514, "l_a": 3.0072967410087585, "l_c": 2.9678685665130615, "z_loss": 41.6274790763855, "lr": 0.0018286391204858524, "wd": 0.01, "grad_norm": 0.19235610961914062, "tokens_per_second": 1443628.4831202948, "eta_seconds": 2767.0209605216983, "data_wait_seconds": 1.6897916793823242e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 64870, "total_steps": 72479, "tokens": 34010562560, "loss": 4.41186797618866, "l_mix": 2.7352956533432007, "l_a": 2.819235324859619, "l_c": 2.7693389654159546, "z_loss": 44.11874198913574, "lr": 0.001825953040136344, "wd": 0.01, "grad_norm": 0.11875459551811218, "tokens_per_second": 1444484.973673735, "eta_seconds": 2761.7507033348083, "data_wait_seconds": 1.7404556274414062e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 64880, "total_steps": 72479, "tokens": 34015805440, "loss": 4.482162058353424, "l_mix": 2.779434621334076, "l_a": 2.8612833619117737, "l_c": 2.8144745230674744, "z_loss": 43.486284255981445, "lr": 0.001823261313578738, "wd": 0.01, "grad_norm": 0.14735105633735657, "tokens_per_second": 1363096.1130364786, "eta_seconds": 2922.8052768230436, "data_wait_seconds": 2.199411392211914e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 64890, "total_steps": 72479, "tokens": 34021048320, "loss": 4.49344265460968, "l_mix": 2.785897195339203, "l_a": 2.8695096373558044, "l_c": 2.8223081529140472, "z_loss": 44.139432430267334, "lr": 0.0018205639632969156, "wd": 0.01, "grad_norm": 0.18879133462905884, "tokens_per_second": 1459830.97842228, "eta_seconds": 2725.5358262777327, "data_wait_seconds": 1.9341707229614258e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 64900, "total_steps": 72479, "tokens": 34026291200, "loss": 4.484281957149506, "l_mix": 2.781047970056534, "l_a": 2.8598416447639465, "l_c": 2.8176050186157227, "z_loss": 43.23307514190674, "lr": 0.0018178610118217342, "wd": 0.01, "grad_norm": 0.1975427120923996, "tokens_per_second": 1411300.9647715664, "eta_seconds": 2815.5431415319445, "data_wait_seconds": 1.6570091247558594e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16075616, "gate_a2b/L1_absmax": 0.6821335, "gate_a2b/L1_rms": 0.17056154, "gate_a2b/L3_absmean": 0.15165591, "gate_a2b/L3_absmax": 0.49875885, "gate_a2b/L3_rms": 0.15727614, "gate_a2b/L5_absmean": 0.21978207, "gate_a2b/L5_absmax": 0.73366266, "gate_a2b/L5_rms": 0.22578916, "gate_c2b/L1_absmean": 0.04004298, "gate_c2b/L1_absmax": 0.62586081, "gate_c2b/L1_rms": 0.07090868, "gate_c2b/L3_absmean": 0.16534427, "gate_c2b/L3_absmax": 0.38914582, "gate_c2b/L3_rms": 0.1688799, "gate_c2b/L5_absmean": 0.46918008, "gate_c2b/L5_absmax": 0.67482013, "gate_c2b/L5_rms": 0.47830257, "gate_b2a/L7_absmean": 0.05122866, "gate_b2a/L7_absmax": 0.33368343, "gate_b2a/L7_rms": 0.05704473, "gate_b2a/L11_absmean": 0.08338422, "gate_b2a/L11_absmax": 0.6146546, "gate_b2a/L11_rms": 0.09253319, "gate_b2a/L14_absmean": 0.21724397, "gate_b2a/L14_absmax": 0.81216633, "gate_b2a/L14_rms": 0.22643955, "gate_b2c/L3_absmean": 0.03584415, "gate_b2c/L3_absmax": 0.55215645, "gate_b2c/L3_rms": 0.04624633, "gate_b2c/L5_absmean": 0.03758238, "gate_b2c/L5_absmax": 0.52896625, "gate_b2c/L5_rms": 0.04663187, "gate_b2c/L6_absmean": 0.0543814, "gate_b2c/L6_absmax": 0.33993125, "gate_b2c/L6_rms": 0.06014733}, "streams": {"cos_a2b/round0": 0.3604, "cos_c2b/round0": 0.239631, "cos_b2a/round0": 0.158637, "cos_b2c/round0": 0.151628, "rms_a/round0": 7.647493, "rms_b/round0": 12.991848, "rms_c/round0": 4.945225, "cos_a2b/round1": 0.405019, "cos_c2b/round1": 0.299505, "cos_b2a/round1": 0.250603, "cos_b2c/round1": 0.103749, "rms_a/round1": 14.936315, "rms_b/round1": 30.200014, "rms_c/round1": 10.709309, "cos_a2b/round2": 0.52965, "cos_c2b/round2": 0.522183, "cos_b2a/round2": 0.096703, "cos_b2c/round2": 0.177946, "rms_a/round2": 43.272011, "rms_b/round2": 83.380882, "rms_c/round2": 20.837326}, "alpha_mean": 0.4250073810442814, "alpha_hist": [0.00752, 0.07065, 0.18591, 0.21667, 0.18166, 0.1613, 0.11815, 0.05202, 0.006, 0.00012], "alpha_by_type": {"special": {"share": 0.00087, "mean": 0.55875, "std": 0.10937}, "whitespace": {"share": 0.03181, "mean": 0.4991, "std": 0.13819}, "digit": {"share": 0.08974, "mean": 0.46337, "std": 0.10604}, "word_start": {"share": 0.51093, "mean": 0.43741, "std": 0.16619}, "word_cont": {"share": 0.25101, "mean": 0.31562, "std": 0.1211}, "punct": {"share": 0.11565, "mean": 0.55647, "std": 0.15547}}, "alpha_p05": 0.18447, "alpha_p50": 0.40969, "alpha_p95": 0.71879}
{"step": 64910, "total_steps": 72479, "tokens": 34031534080, "loss": 4.586459755897522, "l_mix": 2.8451855778694153, "l_a": 2.922177106142044, "l_c": 2.8820698261260986, "z_loss": 43.373974323272705, "lr": 0.001815152481730836, "wd": 0.01, "grad_norm": 0.13523730635643005, "tokens_per_second": 1396187.4236093876, "eta_seconds": 2842.265876984596, "data_wait_seconds": 2.8401613235473633e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 64920, "total_steps": 72479, "tokens": 34036776960, "loss": 3.027343302965164, "l_mix": 1.8743480145931244, "l_a": 1.9435670524835587, "l_c": 1.8997504860162735, "z_loss": 52.42510223388672, "lr": 0.0018124383956484625, "wd": 0.01, "grad_norm": 0.1757802665233612, "tokens_per_second": 1439650.7947894752, "eta_seconds": 2752.815478825569, "data_wait_seconds": 2.3484230041503906e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 64930, "total_steps": 72479, "tokens": 34042019840, "loss": 4.460016965866089, "l_mix": 2.7652796804904938, "l_a": 2.846453458070755, "l_c": 2.8026709258556366, "z_loss": 44.45615005493164, "lr": 0.001809718776245264, "wd": 0.01, "grad_norm": 0.11885891854763031, "tokens_per_second": 1469536.0221651, "eta_seconds": 2693.265120625496, "data_wait_seconds": 2.0623207092285156e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 64940, "total_steps": 72479, "tokens": 34047262720, "loss": 4.460912525653839, "l_mix": 2.7654780447483063, "l_a": 2.8491512537002563, "l_c": 2.8022965490818024, "z_loss": 44.008739948272705, "lr": 0.0018069936462381097, "wd": 0.01, "grad_norm": 0.11619716882705688, "tokens_per_second": 1476874.0726329598, "eta_seconds": 2676.3332807064057, "data_wait_seconds": 1.5169382095336914e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 64950, "total_steps": 72479, "tokens": 34052505600, "loss": 4.423277735710144, "l_mix": 2.7424280047416687, "l_a": 2.8246911764144897, "l_c": 2.778141111135483, "z_loss": 44.30199909210205, "lr": 0.0018042630283898993, "wd": 0.01, "grad_norm": 0.18547140061855316, "tokens_per_second": 1435649.6941798176, "eta_seconds": 2749.531705403328, "data_wait_seconds": 2.3096799850463867e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 64960, "total_steps": 72479, "tokens": 34057748480, "loss": 4.403547048568726, "l_mix": 2.7303155958652496, "l_a": 2.812193900346756, "l_c": 2.7652440071105957, "z_loss": 43.27933979034424, "lr": 0.0018015269455093723, "wd": 0.01, "grad_norm": 0.13754448294639587, "tokens_per_second": 1460059.6284648993, "eta_seconds": 2699.972929286957, "data_wait_seconds": 2.709031105041504e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 64970, "total_steps": 72479, "tokens": 34062991360, "loss": 4.421631813049316, "l_mix": 2.7414172887802124, "l_a": 2.8225077092647552, "l_c": 2.7782072722911835, "z_loss": 43.38548469543457, "lr": 0.0017987854204509163, "wd": 0.01, "grad_norm": 0.16385766863822937, "tokens_per_second": 1430227.958333783, "eta_seconds": 2752.6231528759004, "data_wait_seconds": 1.564621925354004e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 64980, "total_steps": 72479, "tokens": 34068234240, "loss": 4.356258988380432, "l_mix": 2.700508773326874, "l_a": 2.7846652567386627, "l_c": 2.734501749277115, "z_loss": 43.44512224197388, "lr": 0.0017960384761143788, "wd": 0.01, "grad_norm": 0.10352959483861923, "tokens_per_second": 1400580.3879751351, "eta_seconds": 2807.1474838256836, "data_wait_seconds": 2.6792287826538086e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 64990, "total_steps": 72479, "tokens": 34073477120, "loss": 4.4184101819992065, "l_mix": 2.739359349012375, "l_a": 2.82247731089592, "l_c": 2.774358481168747, "z_loss": 45.00399398803711, "lr": 0.001793286135444873, "wd": 0.01, "grad_norm": 0.16916395723819733, "tokens_per_second": 1405055.9342775585, "eta_seconds": 2794.474395084381, "data_wait_seconds": 2.7358531951904297e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 65000, "total_steps": 72479, "tokens": 34078720000, "loss": 4.431203484535217, "l_mix": 2.7474417984485626, "l_a": 2.8297028243541718, "l_c": 2.782835841178894, "z_loss": 44.00836229324341, "lr": 0.0017905284214325883, "wd": 0.01, "grad_norm": 0.12660321593284607, "tokens_per_second": 1445839.2031781038, "eta_seconds": 2712.0235385656356, "data_wait_seconds": 1.8715858459472656e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16135336, "gate_a2b/L1_absmax": 0.68002236, "gate_a2b/L1_rms": 0.17112632, "gate_a2b/L3_absmean": 0.15215722, "gate_a2b/L3_absmax": 0.49964479, "gate_a2b/L3_rms": 0.15772443, "gate_a2b/L5_absmean": 0.21992493, "gate_a2b/L5_absmax": 0.73471731, "gate_a2b/L5_rms": 0.2259002, "gate_c2b/L1_absmean": 0.04061371, "gate_c2b/L1_absmax": 0.62392557, "gate_c2b/L1_rms": 0.07097888, "gate_c2b/L3_absmean": 0.16537069, "gate_c2b/L3_absmax": 0.38841617, "gate_c2b/L3_rms": 0.16884799, "gate_c2b/L5_absmean": 0.46908885, "gate_c2b/L5_absmax": 0.67557883, "gate_c2b/L5_rms": 0.4781583, "gate_b2a/L7_absmean": 0.05151619, "gate_b2a/L7_absmax": 0.3345052, "gate_b2a/L7_rms": 0.05727789, "gate_b2a/L11_absmean": 0.08365706, "gate_b2a/L11_absmax": 0.61547613, "gate_b2a/L11_rms": 0.09283947, "gate_b2a/L14_absmean": 0.21731314, "gate_b2a/L14_absmax": 0.81280518, "gate_b2a/L14_rms": 0.22650439, "gate_b2c/L3_absmean": 0.03610002, "gate_b2c/L3_absmax": 0.55100399, "gate_b2c/L3_rms": 0.04638031, "gate_b2c/L5_absmean": 0.03766391, "gate_b2c/L5_absmax": 0.52933043, "gate_b2c/L5_rms": 0.04669679, "gate_b2c/L6_absmean": 0.0544466, "gate_b2c/L6_absmax": 0.33938086, "gate_b2c/L6_rms": 0.0602385}, "streams": {"cos_a2b/round0": 0.358057, "cos_c2b/round0": 0.234125, "cos_b2a/round0": 0.163581, "cos_b2c/round0": 0.146724, "rms_a/round0": 7.616051, "rms_b/round0": 13.065501, "rms_c/round0": 5.02459, "cos_a2b/round1": 0.407015, "cos_c2b/round1": 0.304273, "cos_b2a/round1": 0.25616, "cos_b2c/round1": 0.110867, "rms_a/round1": 14.812333, "rms_b/round1": 30.395746, "rms_c/round1": 10.773933, "cos_a2b/round2": 0.532679, "cos_c2b/round2": 0.519635, "cos_b2a/round2": 0.098233, "cos_b2c/round2": 0.186649, "rms_a/round2": 43.472748, "rms_b/round2": 83.918373, "rms_c/round2": 20.926525}, "alpha_mean": 0.4262274930434131, "alpha_hist": [0.01062, 0.07655, 0.18359, 0.1973, 0.19524, 0.1571, 0.10661, 0.06253, 0.01035, 0.00011], "alpha_by_type": {"special": {"share": 0.00085, "mean": 0.57004, "std": 0.12612}, "whitespace": {"share": 0.03152, "mean": 0.50795, "std": 0.11794}, "digit": {"share": 0.05568, "mean": 0.47211, "std": 0.10273}, "word_start": {"share": 0.53174, "mean": 0.43533, "std": 0.17115}, "word_cont": {"share": 0.26418, "mean": 0.32853, "std": 0.13044}, "punct": {"share": 0.11603, "mean": 0.56167, "std": 0.17179}}, "alpha_p05": 0.16027, "alpha_p50": 0.40733, "alpha_p95": 0.7345}
{"step": 65010, "total_steps": 72479, "tokens": 34083962880, "loss": 4.595557868480682, "l_mix": 2.8494311571121216, "l_a": 2.9320909082889557, "l_c": 2.8883314430713654, "z_loss": 43.437087535858154, "lr": 0.0017877653571125968, "wd": 0.01, "grad_norm": 0.21191199123859406, "tokens_per_second": 1410400.7686455382, "eta_seconds": 2776.4498992443087, "data_wait_seconds": 1.8715858459472656e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 65020, "total_steps": 72479, "tokens": 34089205760, "loss": 4.608704745769501, "l_mix": 2.8592610359191895, "l_a": 2.9379535913467407, "l_c": 2.8935251235961914, "z_loss": 42.75150537490845, "lr": 0.001784996965564662, "wd": 0.01, "grad_norm": 0.17254506051540375, "tokens_per_second": 1469247.259974373, "eta_seconds": 2661.6787374973296, "data_wait_seconds": 1.58846378326416e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 65030, "total_steps": 72479, "tokens": 34094448640, "loss": 2.7665342688560486, "l_mix": 1.7132272124290466, "l_a": 1.7746199369430542, "l_c": 1.7364037036895752, "z_loss": 54.542025089263916, "lr": 0.0017822232699130447, "wd": 0.01, "grad_norm": 0.15026971697807312, "tokens_per_second": 1443353.7907274656, "eta_seconds": 2705.7962760686873, "data_wait_seconds": 2.822279930114746e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 65040, "total_steps": 72479, "tokens": 34099691520, "loss": 4.463379740715027, "l_mix": 2.7672020494937897, "l_a": 2.850807309150696, "l_c": 2.8031182289123535, "z_loss": 43.57089328765869, "lr": 0.0017794442933263118, "wd": 0.01, "grad_norm": 0.13222435116767883, "tokens_per_second": 1389191.9830711153, "eta_seconds": 2807.515793013573, "data_wait_seconds": 2.574920654296875e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 65050, "total_steps": 72479, "tokens": 34104934400, "loss": 4.406311631202698, "l_mix": 2.7320134341716766, "l_a": 2.813468098640442, "l_c": 2.7675259709358215, "z_loss": 44.12992238998413, "lr": 0.001776660059017141, "wd": 0.01, "grad_norm": 0.14146804809570312, "tokens_per_second": 1422713.234607525, "eta_seconds": 2737.681394433975, "data_wait_seconds": 1.946091651916504e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 65060, "total_steps": 72479, "tokens": 34110177280, "loss": 4.5313674211502075, "l_mix": 2.809484899044037, "l_a": 2.8903496265411377, "l_c": 2.849258601665497, "z_loss": 43.28809356689453, "lr": 0.0017738705902421276, "wd": 0.01, "grad_norm": 0.20275025069713593, "tokens_per_second": 1243809.373694021, "eta_seconds": 3127.241805911064, "data_wait_seconds": 2.181529998779297e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 65070, "total_steps": 72479, "tokens": 34115420160, "loss": 4.4971407651901245, "l_mix": 2.789260983467102, "l_a": 2.8670763671398163, "l_c": 2.8258558213710785, "z_loss": 42.17485761642456, "lr": 0.0017710759103015897, "wd": 0.01, "grad_norm": 0.18567027151584625, "tokens_per_second": 1436650.651710501, "eta_seconds": 2703.8234990358355, "data_wait_seconds": 1.531839370727539e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 65080, "total_steps": 72479, "tokens": 34120663040, "loss": 4.403582751750946, "l_mix": 2.7298783361911774, "l_a": 2.8133463859558105, "l_c": 2.7656679153442383, "z_loss": 43.71396255493164, "lr": 0.0017682760425393748, "wd": 0.01, "grad_norm": 0.11385801434516907, "tokens_per_second": 1441076.143883542, "eta_seconds": 2691.8819858789443, "data_wait_seconds": 1.6391277313232422e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 65090, "total_steps": 72479, "tokens": 34125905920, "loss": 4.467282712459564, "l_mix": 2.7705098390579224, "l_a": 2.8492385149002075, "l_c": 2.8066709339618683, "z_loss": 43.77736473083496, "lr": 0.0017654710103426645, "wd": 0.01, "grad_norm": 0.11983854323625565, "tokens_per_second": 1435394.7068842733, "eta_seconds": 2698.8841559886932, "data_wait_seconds": 2.205371856689453e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 65100, "total_steps": 72479, "tokens": 34131148800, "loss": 4.7307248711586, "l_mix": 2.9361496567726135, "l_a": 3.007849782705307, "l_c": 2.9740671813488007, "z_loss": 42.186742305755615, "lr": 0.0017626608371417774, "wd": 0.01, "grad_norm": 0.18421374261379242, "tokens_per_second": 1403559.5371353417, "eta_seconds": 2756.3641225337983, "data_wait_seconds": 1.6033649444580078e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.16137597, "gate_a2b/L1_absmax": 0.67981291, "gate_a2b/L1_rms": 0.1710375, "gate_a2b/L3_absmean": 0.15170705, "gate_a2b/L3_absmax": 0.50252193, "gate_a2b/L3_rms": 0.15728673, "gate_a2b/L5_absmean": 0.21989678, "gate_a2b/L5_absmax": 0.73539269, "gate_a2b/L5_rms": 0.22594477, "gate_c2b/L1_absmean": 0.04088223, "gate_c2b/L1_absmax": 0.62246555, "gate_c2b/L1_rms": 0.07103513, "gate_c2b/L3_absmean": 0.1652772, "gate_c2b/L3_absmax": 0.38859877, "gate_c2b/L3_rms": 0.1687896, "gate_c2b/L5_absmean": 0.46906924, "gate_c2b/L5_absmax": 0.67607087, "gate_c2b/L5_rms": 0.4781546, "gate_b2a/L7_absmean": 0.05128406, "gate_b2a/L7_absmax": 0.33321232, "gate_b2a/L7_rms": 0.05710125, "gate_b2a/L11_absmean": 0.0836827, "gate_b2a/L11_absmax": 0.61483508, "gate_b2a/L11_rms": 0.09280385, "gate_b2a/L14_absmean": 0.21724471, "gate_b2a/L14_absmax": 0.81120038, "gate_b2a/L14_rms": 0.22644582, "gate_b2c/L3_absmean": 0.0362578, "gate_b2c/L3_absmax": 0.55202556, "gate_b2c/L3_rms": 0.04655284, "gate_b2c/L5_absmean": 0.03778851, "gate_b2c/L5_absmax": 0.52719766, "gate_b2c/L5_rms": 0.04671958, "gate_b2c/L6_absmean": 0.05455443, "gate_b2c/L6_absmax": 0.3399654, "gate_b2c/L6_rms": 0.06031403}, "streams": {"cos_a2b/round0": 0.355026, "cos_c2b/round0": 0.237051, "cos_b2a/round0": 0.169975, "cos_b2c/round0": 0.151477, "rms_a/round0": 7.606358, "rms_b/round0": 12.925855, "rms_c/round0": 4.945184, "cos_a2b/round1": 0.412007, "cos_c2b/round1": 0.316325, "cos_b2a/round1": 0.257767, "cos_b2c/round1": 0.115447, "rms_a/round1": 14.836678, "rms_b/round1": 29.916176, "rms_c/round1": 10.744023, "cos_a2b/round2": 0.549483, "cos_c2b/round2": 0.513599, "cos_b2a/round2": 0.099892, "cos_b2c/round2": 0.19113, "rms_a/round2": 43.416233, "rms_b/round2": 82.370155, "rms_c/round2": 20.895321}, "alpha_mean": 0.44688198831110526, "alpha_hist": [0.00891, 0.05977, 0.16071, 0.18275, 0.18958, 0.18758, 0.13832, 0.06609, 0.00621, 9e-05], "alpha_by_type": {"special": {"share": 0.0006, "mean": 0.54302, "std": 0.11348}, "whitespace": {"share": 0.04648, "mean": 0.52286, "std": 0.11516}, "digit": {"share": 0.02707, "mean": 0.4326, "std": 0.14077}, "word_start": {"share": 0.59981, "mean": 0.46629, "std": 0.15943}, "word_cont": {"share": 0.22816, "mean": 0.32314, "std": 0.13216}, "punct": {"share": 0.09789, "mean": 0.58369, "std": 0.15617}}, "alpha_p05": 0.17925, "alpha_p50": 0.43686, "alpha_p95": 0.71423}
{"step": 65110, "total_steps": 72479, "tokens": 34136391680, "loss": 4.601581633090973, "l_mix": 2.8551336526870728, "l_a": 2.9306953847408295, "l_c": 2.8907978236675262, "z_loss": 43.556992530822754, "lr": 0.0017598455464099753, "wd": 0.01, "grad_norm": 0.1338447779417038, "tokens_per_second": 1404281.1510692316, "eta_seconds": 2751.2142202138903, "data_wait_seconds": 1.5735626220703125e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 65120, "total_steps": 72479, "tokens": 34141634560, "loss": 4.654699742794037, "l_mix": 2.8888576924800873, "l_a": 2.9610739648342133, "l_c": 2.925066113471985, "z_loss": 42.913761615753174, "lr": 0.0017570251616632663, "wd": 0.01, "grad_norm": 0.16440749168395996, "tokens_per_second": 1297966.484930851, "eta_seconds": 2972.523125052452, "data_wait_seconds": 1.8924474716186523e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 65130, "total_steps": 72479, "tokens": 34146877440, "loss": 4.678833067417145, "l_mix": 2.903503030538559, "l_a": 2.9770801961421967, "l_c": 2.9406864941120148, "z_loss": 43.8919620513916, "lr": 0.0017541997064602097, "wd": 0.01, "grad_norm": 0.15628884732723236, "tokens_per_second": 1484597.217339291, "eta_seconds": 2595.3116892576218, "data_wait_seconds": 2.2113323211669922e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 65140, "total_steps": 72479, "tokens": 34152120320, "loss": 2.873904764652252, "l_mix": 1.7790871262550354, "l_a": 1.8452042639255524, "l_c": 1.8041876405477524, "z_loss": 54.92452430725098, "lr": 0.0017513692044017163, "wd": 0.01, "grad_norm": 0.1552385538816452, "tokens_per_second": 1338562.9802147388, "eta_seconds": 2874.5376115083695, "data_wait_seconds": 2.4586915969848633e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 65150, "total_steps": 72479, "tokens": 34157363200, "loss": 4.329696834087372, "l_mix": 2.6840879321098328, "l_a": 2.7636574804782867, "l_c": 2.7217052280902863, "z_loss": 44.08907508850098, "lr": 0.0017485336791308544, "wd": 0.01, "grad_norm": 0.20335759222507477, "tokens_per_second": 1426935.6368974529, "eta_seconds": 2692.8381719827653, "data_wait_seconds": 2.2113323211669922e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 65160, "total_steps": 72479, "tokens": 34162606080, "loss": 4.387651085853577, "l_mix": 2.7204026877880096, "l_a": 2.8030342757701874, "l_c": 2.75446018576622, "z_loss": 43.55914926528931, "lr": 0.0017456931543326506, "wd": 0.01, "grad_norm": 0.1480143517255783, "tokens_per_second": 1432203.6578263924, "eta_seconds": 2679.272498035431, "data_wait_seconds": 1.6629695892333984e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 65170, "total_steps": 72479, "tokens": 34167848960, "loss": 4.406046807765961, "l_mix": 2.730951428413391, "l_a": 2.81740403175354, "l_c": 2.7662470638751984, "z_loss": 43.906044483184814, "lr": 0.0017428476537338926, "wd": 0.01, "grad_norm": 0.16106711328029633, "tokens_per_second": 1345196.9943710922, "eta_seconds": 2848.6690113306045, "data_wait_seconds": 2.5540590286254883e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 65180, "total_steps": 72479, "tokens": 34173091840, "loss": 4.3974146246910095, "l_mix": 2.7265374064445496, "l_a": 2.807700902223587, "l_c": 2.761889159679413, "z_loss": 43.807639598846436, "lr": 0.0017399972011029302, "wd": 0.01, "grad_norm": 0.14596816897392273, "tokens_per_second": 1373490.4135580247, "eta_seconds": 2786.1702376842495, "data_wait_seconds": 2.181529998779297e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 65190, "total_steps": 72479, "tokens": 34178334720, "loss": 4.41157203912735, "l_mix": 2.7347055971622467, "l_a": 2.816376894712448, "l_c": 2.7731775045394897, "z_loss": 44.45678520202637, "lr": 0.0017371418202494775, "wd": 0.01, "grad_norm": 0.16503192484378815, "tokens_per_second": 1462812.918618172, "eta_seconds": 2612.4565782546997, "data_wait_seconds": 2.1219253540039062e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 65200, "total_steps": 72479, "tokens": 34183577600, "loss": 4.402404606342316, "l_mix": 2.7291159629821777, "l_a": 2.8132359981536865, "l_c": 2.764392524957657, "z_loss": 44.755000591278076, "lr": 0.0017342815350244146, "wd": 0.01, "grad_norm": 0.14585258066654205, "tokens_per_second": 1436061.651095565, "eta_seconds": 2657.4711114168167, "data_wait_seconds": 4.7206878662109375e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.1614586, "gate_a2b/L1_absmax": 0.67849904, "gate_a2b/L1_rms": 0.17118157, "gate_a2b/L3_absmean": 0.15190861, "gate_a2b/L3_absmax": 0.49964273, "gate_a2b/L3_rms": 0.157509, "gate_a2b/L5_absmean": 0.21977845, "gate_a2b/L5_absmax": 0.73345739, "gate_a2b/L5_rms": 0.22579016, "gate_c2b/L1_absmean": 0.04088533, "gate_c2b/L1_absmax": 0.62173617, "gate_c2b/L1_rms": 0.07098043, "gate_c2b/L3_absmean": 0.16511701, "gate_c2b/L3_absmax": 0.38680106, "gate_c2b/L3_rms": 0.16863409, "gate_c2b/L5_absmean": 0.46881175, "gate_c2b/L5_absmax": 0.67443717, "gate_c2b/L5_rms": 0.47790679, "gate_b2a/L7_absmean": 0.05132647, "gate_b2a/L7_absmax": 0.33339375, "gate_b2a/L7_rms": 0.05710039, "gate_b2a/L11_absmean": 0.08386676, "gate_b2a/L11_absmax": 0.61553001, "gate_b2a/L11_rms": 0.09297431, "gate_b2a/L14_absmean": 0.21737111, "gate_b2a/L14_absmax": 0.81028634, "gate_b2a/L14_rms": 0.22649945, "gate_b2c/L3_absmean": 0.03611642, "gate_b2c/L3_absmax": 0.55226833, "gate_b2c/L3_rms": 0.04647207, "gate_b2c/L5_absmean": 0.03760745, "gate_b2c/L5_absmax": 0.52799392, "gate_b2c/L5_rms": 0.04660294, "gate_b2c/L6_absmean": 0.05456763, "gate_b2c/L6_absmax": 0.34092233, "gate_b2c/L6_rms": 0.06032555}, "streams": {"cos_a2b/round0": 0.356187, "cos_c2b/round0": 0.241641, "cos_b2a/round0": 0.160904, "cos_b2c/round0": 0.148372, "rms_a/round0": 7.675289, "rms_b/round0": 13.061057, "rms_c/round0": 4.976927, "cos_a2b/round1": 0.4072, "cos_c2b/round1": 0.302792, "cos_b2a/round1": 0.252015, "cos_b2c/round1": 0.10717, "rms_a/round1": 15.078979, "rms_b/round1": 30.683907, "rms_c/round1": 10.842434, "cos_a2b/round2": 0.53422, "cos_c2b/round2": 0.51977, "cos_b2a/round2": 0.096791, "cos_b2c/round2": 0.18176, "rms_a/round2": 44.2075, "rms_b/round2": 84.741287, "rms_c/round2": 21.217726}, "alpha_mean": 0.4393349274414149, "alpha_hist": [0.00948, 0.0641, 0.16896, 0.20308, 0.18697, 0.16345, 0.12329, 0.06955, 0.01096, 0.00017], "alpha_by_type": {"special": {"share": 0.00116, "mean": 0.58895, "std": 0.12666}, "whitespace": {"share": 0.02843, "mean": 0.52348, "std": 0.12379}, "digit": {"share": 0.04039, "mean": 0.47319, "std": 0.13438}, "word_start": {"share": 0.54735, "mean": 0.45422, "std": 0.16865}, "word_cont": {"share": 0.26205, "mean": 0.33083, "std": 0.12822}, "punct": {"share": 0.12062, "mean": 0.5749, "std": 0.15798}}, "alpha_p05": 0.18596, "alpha_p50": 0.42895, "alpha_p95": 0.73412}
{"step": 65210, "total_steps": 72479, "tokens": 34188820480, "loss": 4.683519065380096, "l_mix": 2.906018942594528, "l_a": 2.9833831787109375, "l_c": 2.941617041826248, "z_loss": 43.45116662979126, "lr": 0.0017314163693195868, "wd": 0.01, "grad_norm": 0.14379580318927765, "tokens_per_second": 1371439.9103328516, "eta_seconds": 2778.8672644615176, "data_wait_seconds": 2.09808349609375e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 65220, "total_steps": 72479, "tokens": 34194063360, "loss": 4.7231733202934265, "l_mix": 2.930755466222763, "l_a": 3.0067814588546753, "l_c": 2.967944234609604, "z_loss": 42.18007564544678, "lr": 0.0017285463470676056, "wd": 0.01, "grad_norm": 0.1606462597846985, "tokens_per_second": 1473022.7712255714, "eta_seconds": 2583.671255016327, "data_wait_seconds": 1.5914440155029297e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 65230, "total_steps": 72479, "tokens": 34199306240, "loss": 4.774013996124268, "l_mix": 2.963555932044983, "l_a": 3.0358481407165527, "l_c": 2.999011904001236, "z_loss": 40.95533609390259, "lr": 0.00172567149224165, "wd": 0.01, "grad_norm": 0.14770616590976715, "tokens_per_second": 1384712.479711721, "eta_seconds": 2744.659102654457, "data_wait_seconds": 2.54213809967041e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 65240, "total_steps": 72479, "tokens": 34204549120, "loss": 4.617931365966797, "l_mix": 2.866477131843567, "l_a": 2.935695171356201, "l_c": 2.9024857580661774, "z_loss": 42.71060371398926, "lr": 0.0017227918288552644, "wd": 0.01, "grad_norm": 0.11901076883077621, "tokens_per_second": 1325599.6986248707, "eta_seconds": 2863.097235113382, "data_wait_seconds": 3.1113624572753906e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 65250, "total_steps": 72479, "tokens": 34209792000, "loss": 4.730223000049591, "l_mix": 2.936027228832245, "l_a": 3.007087081670761, "l_c": 2.973564952611923, "z_loss": 43.150577545166016, "lr": 0.0017199073809621593, "wd": 0.01, "grad_norm": 0.12610934674739838, "tokens_per_second": 1423818.6448161053, "eta_seconds": 2661.910606241226, "data_wait_seconds": 1.9043684005737305e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 65260, "total_steps": 72479, "tokens": 34215034880, "loss": 4.366439700126648, "l_mix": 2.707168161869049, "l_a": 2.7876838743686676, "l_c": 2.7432211339473724, "z_loss": 44.78059482574463, "lr": 0.0017170181726560104, "wd": 0.01, "grad_norm": 0.10425330698490143, "tokens_per_second": 1358450.2436888176, "eta_seconds": 2786.1418477296825, "data_wait_seconds": 1.749396324157715e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 65270, "total_steps": 72479, "tokens": 34220277760, "loss": 4.366410493850708, "l_mix": 2.7067892849445343, "l_a": 2.7906340062618256, "l_c": 2.74143648147583, "z_loss": 44.4600567817688, "lr": 0.0017141242280702559, "wd": 0.01, "grad_norm": 0.12859848141670227, "tokens_per_second": 1469014.153325378, "eta_seconds": 2572.876635289192, "data_wait_seconds": 1.6301870346069336e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 65280, "total_steps": 72479, "tokens": 34225520640, "loss": 4.347825825214386, "l_mix": 2.6950342655181885, "l_a": 2.7784793972969055, "l_c": 2.73082572221756, "z_loss": 44.613811016082764, "lr": 0.0017112255713778968, "wd": 0.01, "grad_norm": 0.10834804177284241, "tokens_per_second": 1423117.1550538666, "eta_seconds": 2652.17048265934, "data_wait_seconds": 1.96993350982666e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 65290, "total_steps": 72479, "tokens": 34230763520, "loss": 4.434061944484711, "l_mix": 2.749469965696335, "l_a": 2.8317043483257294, "l_c": 2.7836018800735474, "z_loss": 43.65626764297485, "lr": 0.0017083222267912947, "wd": 0.01, "grad_norm": 0.1445661038160324, "tokens_per_second": 1415823.8862171227, "eta_seconds": 2662.1294277429583, "data_wait_seconds": 3.090500831604004e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 65300, "total_steps": 72479, "tokens": 34236006400, "loss": 4.420068144798279, "l_mix": 2.740014910697937, "l_a": 2.8243499994277954, "l_c": 2.7758270502090454, "z_loss": 43.59882640838623, "lr": 0.001705414218561967, "wd": 0.01, "grad_norm": 0.2040325254201889, "tokens_per_second": 1308884.4491363002, "eta_seconds": 2875.6270689010616, "data_wait_seconds": 2.950429916381836e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16129787, "gate_a2b/L1_absmax": 0.68148756, "gate_a2b/L1_rms": 0.17114201, "gate_a2b/L3_absmean": 0.15213081, "gate_a2b/L3_absmax": 0.50211972, "gate_a2b/L3_rms": 0.15772839, "gate_a2b/L5_absmean": 0.2195324, "gate_a2b/L5_absmax": 0.73210984, "gate_a2b/L5_rms": 0.22557123, "gate_c2b/L1_absmean": 0.04089576, "gate_c2b/L1_absmax": 0.62565392, "gate_c2b/L1_rms": 0.07136489, "gate_c2b/L3_absmean": 0.16516475, "gate_c2b/L3_absmax": 0.38859612, "gate_c2b/L3_rms": 0.16866587, "gate_c2b/L5_absmean": 0.46905908, "gate_c2b/L5_absmax": 0.67394024, "gate_c2b/L5_rms": 0.47804725, "gate_b2a/L7_absmean": 0.05126464, "gate_b2a/L7_absmax": 0.33557698, "gate_b2a/L7_rms": 0.05709369, "gate_b2a/L11_absmean": 0.08389619, "gate_b2a/L11_absmax": 0.61733735, "gate_b2a/L11_rms": 0.09305643, "gate_b2a/L14_absmean": 0.21824861, "gate_b2a/L14_absmax": 0.80972379, "gate_b2a/L14_rms": 0.22734265, "gate_b2c/L3_absmean": 0.03635044, "gate_b2c/L3_absmax": 0.55108708, "gate_b2c/L3_rms": 0.04664269, "gate_b2c/L5_absmean": 0.03760922, "gate_b2c/L5_absmax": 0.52612644, "gate_b2c/L5_rms": 0.04659558, "gate_b2c/L6_absmean": 0.05477216, "gate_b2c/L6_absmax": 0.34041804, "gate_b2c/L6_rms": 0.06056836}, "streams": {"cos_a2b/round0": 0.353169, "cos_c2b/round0": 0.234919, "cos_b2a/round0": 0.162487, "cos_b2c/round0": 0.14632, "rms_a/round0": 7.648131, "rms_b/round0": 13.270803, "rms_c/round0": 5.027518, "cos_a2b/round1": 0.409785, "cos_c2b/round1": 0.306084, "cos_b2a/round1": 0.255918, "cos_b2c/round1": 0.110052, "rms_a/round1": 15.118154, "rms_b/round1": 30.751202, "rms_c/round1": 11.02456, "cos_a2b/round2": 0.533894, "cos_c2b/round2": 0.520648, "cos_b2a/round2": 0.098522, "cos_b2c/round2": 0.183127, "rms_a/round2": 44.264111, "rms_b/round2": 84.709389, "rms_c/round2": 21.594948}, "alpha_mean": 0.43917176634717947, "alpha_hist": [0.00885, 0.06937, 0.17006, 0.19524, 0.18127, 0.16911, 0.12737, 0.06927, 0.00935, 0.00011], "alpha_by_type": {"special": {"share": 0.00148, "mean": 0.57006, "std": 0.12224}, "whitespace": {"share": 0.02312, "mean": 0.51178, "std": 0.13153}, "digit": {"share": 0.02467, "mean": 0.47328, "std": 0.11763}, "word_start": {"share": 0.57539, "mean": 0.45737, "std": 0.16588}, "word_cont": {"share": 0.27875, "mean": 0.33569, "std": 0.13666}, "punct": {"share": 0.09659, "mean": 0.6013, "std": 0.15522}}, "alpha_p05": 0.16968, "alpha_p50": 0.42179, "alpha_p95": 0.69781}
{"step": 65310, "total_steps": 72479, "tokens": 34241249280, "loss": 4.456004619598389, "l_mix": 2.7621675431728363, "l_a": 2.8448339700698853, "l_c": 2.8012893199920654, "z_loss": 43.80909013748169, "lr": 0.0017025015709803881, "wd": 0.01, "grad_norm": 0.1755705624818802, "tokens_per_second": 1460696.9114870208, "eta_seconds": 2573.1694525003436, "data_wait_seconds": 3.331899642944336e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 65320, "total_steps": 72479, "tokens": 34246492160, "loss": 2.862209677696228, "l_mix": 1.7733352184295654, "l_a": 1.8329234421253204, "l_c": 1.796657845377922, "z_loss": 53.79468584060669, "lr": 0.001699584308375784, "wd": 0.01, "grad_norm": 0.1399233639240265, "tokens_per_second": 1426855.8260502706, "eta_seconds": 2630.5235073328017, "data_wait_seconds": 3.6150217056274414e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 65330, "total_steps": 72479, "tokens": 34251735040, "loss": 4.703119516372681, "l_mix": 2.9186462461948395, "l_a": 2.9939660727977753, "l_c": 2.9542776942253113, "z_loss": 43.21894550323486, "lr": 0.00169666245511593, "wd": 0.01, "grad_norm": 0.18544167280197144, "tokens_per_second": 1440522.1038553703, "eta_seconds": 2601.9280800819397, "data_wait_seconds": 1.8388032913208008e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 65340, "total_steps": 72479, "tokens": 34256977920, "loss": 4.687189102172852, "l_mix": 2.9086197316646576, "l_a": 2.9822165071964264, "l_c": 2.946348339319229, "z_loss": 43.26008081436157, "lr": 0.001693736035606947, "wd": 0.01, "grad_norm": 0.12989196181297302, "tokens_per_second": 1475142.010635534, "eta_seconds": 2537.309631896019, "data_wait_seconds": 1.519918441772461e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 65350, "total_steps": 72479, "tokens": 34262220800, "loss": 4.677558779716492, "l_mix": 2.903453767299652, "l_a": 2.9745399057865143, "l_c": 2.9391435384750366, "z_loss": 41.32480716705322, "lr": 0.0016908050742930967, "wd": 0.01, "grad_norm": 0.15086135268211365, "tokens_per_second": 1380294.261031912, "eta_seconds": 2707.864009523392, "data_wait_seconds": 3.999471664428711e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 65360, "total_steps": 72479, "tokens": 34267463680, "loss": 4.779488205909729, "l_mix": 2.9663453698158264, "l_a": 3.0404884815216064, "l_c": 3.003320664167404, "z_loss": 43.11008834838867, "lr": 0.0016878695956565799, "wd": 0.01, "grad_norm": 0.12741535902023315, "tokens_per_second": 1437825.0269479821, "eta_seconds": 2595.8695961236954, "data_wait_seconds": 1.5079975128173828e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 65370, "total_steps": 72479, "tokens": 34272706560, "loss": 4.456908166408539, "l_mix": 2.7636856138706207, "l_a": 2.8447051644325256, "l_c": 2.7993699312210083, "z_loss": 44.110055446624756, "lr": 0.0016849296242173295, "wd": 0.01, "grad_norm": 0.13044381141662598, "tokens_per_second": 1389531.959245882, "eta_seconds": 2682.31570148468, "data_wait_seconds": 1.537799835205078e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 65380, "total_steps": 72479, "tokens": 34277949440, "loss": 4.416397869586945, "l_mix": 2.7380438148975372, "l_a": 2.8201264441013336, "l_c": 2.7743864357471466, "z_loss": 43.82297992706299, "lr": 0.0016819851845328063, "wd": 0.01, "grad_norm": 0.13772830367088318, "tokens_per_second": 1355654.5581827343, "eta_seconds": 2745.4785509586336, "data_wait_seconds": 3.552436828613281e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 65390, "total_steps": 72479, "tokens": 34283192320, "loss": 4.4646958112716675, "l_mix": 2.7681837379932404, "l_a": 2.850550264120102, "l_c": 2.8044895827770233, "z_loss": 43.51551055908203, "lr": 0.0016790363011977949, "wd": 0.01, "grad_norm": 0.13694244623184204, "tokens_per_second": 1225411.850522288, "eta_seconds": 3033.002847504616, "data_wait_seconds": 2.0295381546020508e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 65400, "total_steps": 72479, "tokens": 34288435200, "loss": 4.477595508098602, "l_mix": 2.775877445936203, "l_a": 2.858647972345352, "l_c": 2.813745528459549, "z_loss": 44.33677530288696, "lr": 0.001676082998844197, "wd": 0.01, "grad_norm": 0.1052287146449089, "tokens_per_second": 1320087.3298781707, "eta_seconds": 2811.5069874525075, "data_wait_seconds": 1.761317253112793e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.1611013, "gate_a2b/L1_absmax": 0.68203247, "gate_a2b/L1_rms": 0.17103486, "gate_a2b/L3_absmean": 0.15202573, "gate_a2b/L3_absmax": 0.50198674, "gate_a2b/L3_rms": 0.15767071, "gate_a2b/L5_absmean": 0.21998642, "gate_a2b/L5_absmax": 0.73148835, "gate_a2b/L5_rms": 0.22598511, "gate_c2b/L1_absmean": 0.04107105, "gate_c2b/L1_absmax": 0.623465, "gate_c2b/L1_rms": 0.07129081, "gate_c2b/L3_absmean": 0.16510302, "gate_c2b/L3_absmax": 0.38895679, "gate_c2b/L3_rms": 0.16861755, "gate_c2b/L5_absmean": 0.46914968, "gate_c2b/L5_absmax": 0.67359763, "gate_c2b/L5_rms": 0.47817695, "gate_b2a/L7_absmean": 0.0517026, "gate_b2a/L7_absmax": 0.33460835, "gate_b2a/L7_rms": 0.05745674, "gate_b2a/L11_absmean": 0.08382952, "gate_b2a/L11_absmax": 0.616243, "gate_b2a/L11_rms": 0.09296083, "gate_b2a/L14_absmean": 0.21831854, "gate_b2a/L14_absmax": 0.80987704, "gate_b2a/L14_rms": 0.22740792, "gate_b2c/L3_absmean": 0.03635607, "gate_b2c/L3_absmax": 0.55165619, "gate_b2c/L3_rms": 0.0465888, "gate_b2c/L5_absmean": 0.03785233, "gate_b2c/L5_absmax": 0.52742928, "gate_b2c/L5_rms": 0.04679734, "gate_b2c/L6_absmean": 0.05486777, "gate_b2c/L6_absmax": 0.34042999, "gate_b2c/L6_rms": 0.06060325}, "streams": {"cos_a2b/round0": 0.35619, "cos_c2b/round0": 0.239062, "cos_b2a/round0": 0.163079, "cos_b2c/round0": 0.14945, "rms_a/round0": 7.54443, "rms_b/round0": 12.996578, "rms_c/round0": 4.930542, "cos_a2b/round1": 0.417945, "cos_c2b/round1": 0.307626, "cos_b2a/round1": 0.255776, "cos_b2c/round1": 0.104203, "rms_a/round1": 14.732642, "rms_b/round1": 30.223209, "rms_c/round1": 10.516369, "cos_a2b/round2": 0.535638, "cos_c2b/round2": 0.519636, "cos_b2a/round2": 0.102996, "cos_b2c/round2": 0.182388, "rms_a/round2": 43.050983, "rms_b/round2": 83.339584, "rms_c/round2": 20.424318}, "alpha_mean": 0.4302953640473106, "alpha_hist": [0.00533, 0.07089, 0.19296, 0.21043, 0.16782, 0.15437, 0.12372, 0.06705, 0.00743, 0.0], "alpha_by_type": {"special": {"share": 0.00021, "mean": 0.54482, "std": 0.16225}, "whitespace": {"share": 0.02953, "mean": 0.51772, "std": 0.11531}, "digit": {"share": 0.03604, "mean": 0.40092, "std": 0.10056}, "word_start": {"share": 0.56816, "mean": 0.45256, "std": 0.16919}, "word_cont": {"share": 0.27332, "mean": 0.32661, "std": 0.12559}, "punct": {"share": 0.09274, "mean": 0.58275, "std": 0.16545}}, "alpha_p05": 0.17413, "alpha_p50": 0.40169, "alpha_p95": 0.72311}
{"step": 65410, "total_steps": 72479, "tokens": 34293678080, "loss": 4.283131897449493, "l_mix": 2.65417942404747, "l_a": 2.7388400733470917, "l_c": 2.691000998020172, "z_loss": 46.09846258163452, "lr": 0.0016731253021408266, "wd": 0.01, "grad_norm": 0.12372630089521408, "tokens_per_second": 1429637.8910184063, "eta_seconds": 2592.3990230560303, "data_wait_seconds": 1.6421079635620117e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 65420, "total_steps": 72479, "tokens": 34298920960, "loss": 4.345407545566559, "l_mix": 2.6937645375728607, "l_a": 2.7750673592090607, "l_c": 2.7304091453552246, "z_loss": 44.725170612335205, "lr": 0.001670163235793203, "wd": 0.01, "grad_norm": 0.14902447164058685, "tokens_per_second": 1438114.7355904586, "eta_seconds": 2573.472686433792, "data_wait_seconds": 1.5497207641601562e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 65430, "total_steps": 72479, "tokens": 34304163840, "loss": 3.156414359807968, "l_mix": 1.9554727971553802, "l_a": 2.0221232920885086, "l_c": 1.9810149371623993, "z_loss": 52.5994873046875, "lr": 0.0016671968245433458, "wd": 0.01, "grad_norm": 0.19024573266506195, "tokens_per_second": 1278774.598880738, "eta_seconds": 2890.0371615409854, "data_wait_seconds": 1.9043684005737305e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 65440, "total_steps": 72479, "tokens": 34309406720, "loss": 2.7748906314373016, "l_mix": 1.7169388830661774, "l_a": 1.7848235666751862, "l_c": 1.7416822165250778, "z_loss": 55.51425504684448, "lr": 0.0016642260931695658, "wd": 0.01, "grad_norm": 0.2028971165418625, "tokens_per_second": 1410958.7639644302, "eta_seconds": 2615.5712882995604, "data_wait_seconds": 2.148747444152832e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 65450, "total_steps": 72479, "tokens": 34314649600, "loss": 4.346375465393066, "l_mix": 2.694169193506241, "l_a": 2.7771686911582947, "l_c": 2.7301854491233826, "z_loss": 45.618794441223145, "lr": 0.001661251066486262, "wd": 0.01, "grad_norm": 0.15476219356060028, "tokens_per_second": 1432383.4273048998, "eta_seconds": 2572.7890184640883, "data_wait_seconds": 1.8984079360961914e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 65460, "total_steps": 72479, "tokens": 34319892480, "loss": 4.482052683830261, "l_mix": 2.7791768312454224, "l_a": 2.860464930534363, "l_c": 2.815787374973297, "z_loss": 43.847190856933594, "lr": 0.0016582717693437096, "wd": 0.01, "grad_norm": 0.13196153938770294, "tokens_per_second": 1395001.4660695016, "eta_seconds": 2637.9739100694655, "data_wait_seconds": 2.0116567611694336e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 65470, "total_steps": 72479, "tokens": 34325135360, "loss": 4.337943851947784, "l_mix": 2.6896019279956818, "l_a": 2.7684580981731415, "l_c": 2.7260145246982574, "z_loss": 44.5453667640686, "lr": 0.001655288226627856, "wd": 0.01, "grad_norm": 0.19133493304252625, "tokens_per_second": 1356280.6450819464, "eta_seconds": 2709.4205062389374, "data_wait_seconds": 2.1159648895263672e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 65480, "total_steps": 72479, "tokens": 34330378240, "loss": 4.3454559445381165, "l_mix": 2.6933901607990265, "l_a": 2.778028756380081, "l_c": 2.728856861591339, "z_loss": 45.86449575424194, "lr": 0.0016523004632601115, "wd": 0.01, "grad_norm": 0.12948328256607056, "tokens_per_second": 1445257.9383859998, "eta_seconds": 2538.9874115467073, "data_wait_seconds": 1.823902130126953e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 65490, "total_steps": 72479, "tokens": 34335621120, "loss": 4.482158541679382, "l_mix": 2.7790800034999847, "l_a": 2.860237956047058, "l_c": 2.8166903257369995, "z_loss": 45.561562061309814, "lr": 0.0016493085041971414, "wd": 0.01, "grad_norm": 0.17012758553028107, "tokens_per_second": 1412547.7464044327, "eta_seconds": 2594.0707783699036, "data_wait_seconds": 1.7911195755004883e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 65500, "total_steps": 72479, "tokens": 34340864000, "loss": 4.405171632766724, "l_mix": 2.7303668558597565, "l_a": 2.8171038925647736, "l_c": 2.7655783593654633, "z_loss": 44.04767417907715, "lr": 0.0016463123744306575, "wd": 0.01, "grad_norm": 0.12221080809831619, "tokens_per_second": 1355961.0076552099, "eta_seconds": 2698.459565830231, "data_wait_seconds": 1.6182661056518555e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16141181, "gate_a2b/L1_absmax": 0.67966944, "gate_a2b/L1_rms": 0.17125325, "gate_a2b/L3_absmean": 0.15218501, "gate_a2b/L3_absmax": 0.50103909, "gate_a2b/L3_rms": 0.1577571, "gate_a2b/L5_absmean": 0.21965857, "gate_a2b/L5_absmax": 0.73222929, "gate_a2b/L5_rms": 0.22570179, "gate_c2b/L1_absmean": 0.04108511, "gate_c2b/L1_absmax": 0.62285817, "gate_c2b/L1_rms": 0.07097803, "gate_c2b/L3_absmean": 0.16504784, "gate_c2b/L3_absmax": 0.38914672, "gate_c2b/L3_rms": 0.168593, "gate_c2b/L5_absmean": 0.46910039, "gate_c2b/L5_absmax": 0.67562437, "gate_c2b/L5_rms": 0.47811416, "gate_b2a/L7_absmean": 0.0512646, "gate_b2a/L7_absmax": 0.33425149, "gate_b2a/L7_rms": 0.05708081, "gate_b2a/L11_absmean": 0.08397792, "gate_b2a/L11_absmax": 0.61454761, "gate_b2a/L11_rms": 0.09304508, "gate_b2a/L14_absmean": 0.21812962, "gate_b2a/L14_absmax": 0.81021267, "gate_b2a/L14_rms": 0.22726078, "gate_b2c/L3_absmean": 0.03621634, "gate_b2c/L3_absmax": 0.55157268, "gate_b2c/L3_rms": 0.04653528, "gate_b2c/L5_absmean": 0.03790886, "gate_b2c/L5_absmax": 0.52829558, "gate_b2c/L5_rms": 0.04685061, "gate_b2c/L6_absmean": 0.05479223, "gate_b2c/L6_absmax": 0.33976391, "gate_b2c/L6_rms": 0.06055534}, "streams": {"cos_a2b/round0": 0.35848, "cos_c2b/round0": 0.238776, "cos_b2a/round0": 0.157681, "cos_b2c/round0": 0.149468, "rms_a/round0": 7.660334, "rms_b/round0": 13.043435, "rms_c/round0": 4.973131, "cos_a2b/round1": 0.412342, "cos_c2b/round1": 0.299058, "cos_b2a/round1": 0.251028, "cos_b2c/round1": 0.102731, "rms_a/round1": 15.098411, "rms_b/round1": 30.672857, "rms_c/round1": 10.790009, "cos_a2b/round2": 0.527896, "cos_c2b/round2": 0.523567, "cos_b2a/round2": 0.095092, "cos_b2c/round2": 0.17886, "rms_a/round2": 43.791012, "rms_b/round2": 84.563301, "rms_c/round2": 21.388531}, "alpha_mean": 0.43241851893803585, "alpha_hist": [0.01045, 0.07295, 0.18613, 0.19769, 0.17442, 0.15538, 0.12062, 0.07101, 0.01094, 0.0004], "alpha_by_type": {"special": {"share": 0.00101, "mean": 0.57141, "std": 0.12919}, "whitespace": {"share": 0.02753, "mean": 0.52881, "std": 0.12275}, "digit": {"share": 0.04681, "mean": 0.45649, "std": 0.12411}, "word_start": {"share": 0.55893, "mean": 0.44231, "std": 0.17236}, "word_cont": {"share": 0.25467, "mean": 0.32598, "std": 0.1337}, "punct": {"share": 0.11105, "mean": 0.59143, "std": 0.15873}}, "alpha_p05": 0.16914, "alpha_p50": 0.41751, "alpha_p95": 0.73905}
{"step": 65510, "total_steps": 72479, "tokens": 34346106880, "loss": 4.421092927455902, "l_mix": 2.7407815158367157, "l_a": 2.822299599647522, "l_c": 2.778738170862198, "z_loss": 44.33374261856079, "lr": 0.0016433120989872091, "wd": 0.01, "grad_norm": 0.1145167276263237, "tokens_per_second": 1457457.1211582758, "eta_seconds": 2506.943785142899, "data_wait_seconds": 1.58846378326416e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 65520, "total_steps": 72479, "tokens": 34351349760, "loss": 4.326282978057861, "l_mix": 2.681719094514847, "l_a": 2.763071656227112, "l_c": 2.7188078463077545, "z_loss": 44.76105308532715, "lr": 0.0016403077029279742, "wd": 0.01, "grad_norm": 0.12075766175985336, "tokens_per_second": 1462162.8021950694, "eta_seconds": 2495.2899817466737, "data_wait_seconds": 1.6003847122192383e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 65530, "total_steps": 72479, "tokens": 34356592640, "loss": 4.46468186378479, "l_mix": 2.768337905406952, "l_a": 2.848662346601486, "l_c": 2.805817186832428, "z_loss": 44.487149238586426, "lr": 0.00163729921134855, "wd": 0.01, "grad_norm": 0.13477486371994019, "tokens_per_second": 1192174.848484257, "eta_seconds": 3055.992429828644, "data_wait_seconds": 2.378225326538086e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 65540, "total_steps": 72479, "tokens": 34361835520, "loss": 3.0640448331832886, "l_mix": 1.8980241566896439, "l_a": 1.9639778286218643, "l_c": 1.922757476568222, "z_loss": 52.948904037475586, "lr": 0.0016342866493787441, "wd": 0.01, "grad_norm": 0.13462790846824646, "tokens_per_second": 1462333.2510862148, "eta_seconds": 2487.828563904762, "data_wait_seconds": 2.2292137145996094e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 65550, "total_steps": 72479, "tokens": 34367078400, "loss": 2.95735239982605, "l_mix": 1.830528363585472, "l_a": 1.8975846767425537, "l_c": 1.8584952503442764, "z_loss": 56.131954193115234, "lr": 0.001631270042182363, "wd": 0.01, "grad_norm": 0.1502862572669983, "tokens_per_second": 1257343.2187360113, "eta_seconds": 2889.2600666761396, "data_wait_seconds": 2.154707908630371e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 65560, "total_steps": 72479, "tokens": 34372321280, "loss": 4.444170236587524, "l_mix": 2.7555585503578186, "l_a": 2.8387617468833923, "l_c": 2.789943903684616, "z_loss": 44.26783847808838, "lr": 0.0016282494149570037, "wd": 0.01, "grad_norm": 0.1068963035941124, "tokens_per_second": 1450621.2211779486, "eta_seconds": 2500.6863397836687, "data_wait_seconds": 2.995133399963379e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 65570, "total_steps": 72479, "tokens": 34377564160, "loss": 4.4140585064888, "l_mix": 2.7371300160884857, "l_a": 2.8180135786533356, "l_c": 2.7717479169368744, "z_loss": 44.57116508483887, "lr": 0.0016252247929338414, "wd": 0.01, "grad_norm": 0.11752372235059738, "tokens_per_second": 1463095.5551321043, "eta_seconds": 2475.7821041107177, "data_wait_seconds": 2.199411392211914e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 65580, "total_steps": 72479, "tokens": 34382807040, "loss": 4.3876489996910095, "l_mix": 2.720409244298935, "l_a": 2.7988509833812714, "l_c": 2.7586146891117096, "z_loss": 43.91419458389282, "lr": 0.0016221962013774204, "wd": 0.01, "grad_norm": 0.19866910576820374, "tokens_per_second": 1301153.84650322, "eta_seconds": 2779.8887285470964, "data_wait_seconds": 4.1157007217407227e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 65590, "total_steps": 72479, "tokens": 34388049920, "loss": 4.322244167327881, "l_mix": 2.6797311305999756, "l_a": 2.7624592185020447, "l_c": 2.7125844061374664, "z_loss": 44.2913122177124, "lr": 0.001619163665585442, "wd": 0.01, "grad_norm": 0.14265693724155426, "tokens_per_second": 1462875.7821725823, "eta_seconds": 2468.986140871048, "data_wait_seconds": 2.130866050720215e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 65600, "total_steps": 72479, "tokens": 34393292800, "loss": 4.361716032028198, "l_mix": 2.703880161046982, "l_a": 2.785394608974457, "l_c": 2.7407247722148895, "z_loss": 44.566762924194336, "lr": 0.0016161272108885536, "wd": 0.01, "grad_norm": 0.13941562175750732, "tokens_per_second": 1372028.0172470645, "eta_seconds": 2628.6468692064286, "data_wait_seconds": 1.856684684753418e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.16153029, "gate_a2b/L1_absmax": 0.67879099, "gate_a2b/L1_rms": 0.1711892, "gate_a2b/L3_absmean": 0.15233819, "gate_a2b/L3_absmax": 0.49856198, "gate_a2b/L3_rms": 0.15791206, "gate_a2b/L5_absmean": 0.22004271, "gate_a2b/L5_absmax": 0.73137349, "gate_a2b/L5_rms": 0.22607186, "gate_c2b/L1_absmean": 0.04121755, "gate_c2b/L1_absmax": 0.61965859, "gate_c2b/L1_rms": 0.07118995, "gate_c2b/L3_absmean": 0.16476238, "gate_c2b/L3_absmax": 0.38724649, "gate_c2b/L3_rms": 0.16828512, "gate_c2b/L5_absmean": 0.46909663, "gate_c2b/L5_absmax": 0.67579925, "gate_c2b/L5_rms": 0.47809556, "gate_b2a/L7_absmean": 0.05143099, "gate_b2a/L7_absmax": 0.33389693, "gate_b2a/L7_rms": 0.05718613, "gate_b2a/L11_absmean": 0.08428718, "gate_b2a/L11_absmax": 0.61592138, "gate_b2a/L11_rms": 0.09336982, "gate_b2a/L14_absmean": 0.21865737, "gate_b2a/L14_absmax": 0.80881906, "gate_b2a/L14_rms": 0.22770402, "gate_b2c/L3_absmean": 0.03652008, "gate_b2c/L3_absmax": 0.55083525, "gate_b2c/L3_rms": 0.04670908, "gate_b2c/L5_absmean": 0.03766171, "gate_b2c/L5_absmax": 0.52873385, "gate_b2c/L5_rms": 0.04663708, "gate_b2c/L6_absmean": 0.05492133, "gate_b2c/L6_absmax": 0.3386097, "gate_b2c/L6_rms": 0.06062312}, "streams": {"cos_a2b/round0": 0.356853, "cos_c2b/round0": 0.242884, "cos_b2a/round0": 0.163919, "cos_b2c/round0": 0.150491, "rms_a/round0": 7.642458, "rms_b/round0": 13.034167, "rms_c/round0": 4.958469, "cos_a2b/round1": 0.416899, "cos_c2b/round1": 0.302829, "cos_b2a/round1": 0.257268, "cos_b2c/round1": 0.108609, "rms_a/round1": 14.937429, "rms_b/round1": 30.284168, "rms_c/round1": 10.692962, "cos_a2b/round2": 0.54196, "cos_c2b/round2": 0.517968, "cos_b2a/round2": 0.107319, "cos_b2c/round2": 0.186123, "rms_a/round2": 43.721981, "rms_b/round2": 84.005692, "rms_c/round2": 20.806501}, "alpha_mean": 0.43470062475363136, "alpha_hist": [0.00616, 0.06345, 0.17628, 0.20207, 0.19083, 0.17839, 0.11771, 0.05794, 0.00688, 0.00029], "alpha_by_type": {"special": {"share": 0.00089, "mean": 0.5575, "std": 0.1299}, "whitespace": {"share": 0.02733, "mean": 0.49627, "std": 0.12962}, "digit": {"share": 0.02345, "mean": 0.46328, "std": 0.10668}, "word_start": {"share": 0.57883, "mean": 0.44899, "std": 0.16128}, "word_cont": {"share": 0.26241, "mean": 0.33581, "std": 0.1287}, "punct": {"share": 0.1071, "mean": 0.57679, "std": 0.15113}}, "alpha_p05": 0.17838, "alpha_p50": 0.44324, "alpha_p95": 0.71938}
{"step": 65610, "total_steps": 72479, "tokens": 34398535680, "loss": 4.728496015071869, "l_mix": 2.9344597160816193, "l_a": 3.010026752948761, "l_c": 2.9700938165187836, "z_loss": 41.808865547180176, "lr": 0.0016130868626501368, "wd": 0.01, "grad_norm": 0.14003823697566986, "tokens_per_second": 1378177.0235890145, "eta_seconds": 2613.114433312416, "data_wait_seconds": 1.7315149307250977e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 65620, "total_steps": 72479, "tokens": 34403778560, "loss": 4.733494162559509, "l_mix": 2.9376558363437653, "l_a": 3.011346846818924, "l_c": 2.9747806787490845, "z_loss": 41.86236095428467, "lr": 0.001610042646266097, "wd": 0.01, "grad_norm": 0.15915480256080627, "tokens_per_second": 1427457.5902563646, "eta_seconds": 2519.228183412552, "data_wait_seconds": 5.4895877838134766e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 65630, "total_steps": 72479, "tokens": 34409021440, "loss": 4.670670509338379, "l_mix": 2.8982475996017456, "l_a": 2.9732953011989594, "l_c": 2.9347805976867676, "z_loss": 43.28562927246094, "lr": 0.0016069945871646485, "wd": 0.01, "grad_norm": 0.11864304542541504, "tokens_per_second": 1458338.2279341966, "eta_seconds": 2462.2878583431243, "data_wait_seconds": 1.940131187438965e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 65640, "total_steps": 72479, "tokens": 34414264320, "loss": 4.578923761844635, "l_mix": 2.841430217027664, "l_a": 2.9128664135932922, "l_c": 2.8787783086299896, "z_loss": 44.73160457611084, "lr": 0.001603942710806104, "wd": 0.01, "grad_norm": 0.15308155119419098, "tokens_per_second": 1444739.689143136, "eta_seconds": 2481.8350730895995, "data_wait_seconds": 1.6987323760986328e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 65650, "total_steps": 72479, "tokens": 34419507200, "loss": 2.8693897128105164, "l_mix": 1.7772071808576584, "l_a": 1.8397576063871384, "l_c": 1.8008508533239365, "z_loss": 53.9727087020874, "lr": 0.0016008870426826628, "wd": 0.01, "grad_norm": 0.18062615394592285, "tokens_per_second": 1450839.6245063553, "eta_seconds": 2467.786715722084, "data_wait_seconds": 3.0428171157836914e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 65660, "total_steps": 72479, "tokens": 34424750080, "loss": 2.8607405722141266, "l_mix": 1.771606683731079, "l_a": 1.8327016234397888, "l_c": 1.7977445423603058, "z_loss": 53.82350254058838, "lr": 0.001597827608318195, "wd": 0.01, "grad_norm": 0.1693526804447174, "tokens_per_second": 1441466.8428781705, "eta_seconds": 2480.1957045793533, "data_wait_seconds": 1.7642974853515625e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 65670, "total_steps": 72479, "tokens": 34429992960, "loss": 4.307617485523224, "l_mix": 2.6699196100234985, "l_a": 2.755206137895584, "l_c": 2.703786462545395, "z_loss": 44.91049003601074, "lr": 0.0015947644332680313, "wd": 0.01, "grad_norm": 0.12890948355197906, "tokens_per_second": 1379037.8403304287, "eta_seconds": 2588.672252202034, "data_wait_seconds": 2.5033950805664062e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 65680, "total_steps": 72479, "tokens": 34435235840, "loss": 4.416783094406128, "l_mix": 2.7377224564552307, "l_a": 2.8242736756801605, "l_c": 2.772594600915909, "z_loss": 44.50327396392822, "lr": 0.0015916975431187472, "wd": 0.01, "grad_norm": 0.12000003457069397, "tokens_per_second": 1254980.135713338, "eta_seconds": 2840.3908640146255, "data_wait_seconds": 2.0563602447509766e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 65690, "total_steps": 72479, "tokens": 34440478720, "loss": 4.345469951629639, "l_mix": 2.694076120853424, "l_a": 2.7746050357818604, "l_c": 2.730041056871414, "z_loss": 44.25170373916626, "lr": 0.0015886269634879505, "wd": 0.01, "grad_norm": 0.11148514598608017, "tokens_per_second": 1450985.518573375, "eta_seconds": 2453.0852902650836, "data_wait_seconds": 1.9252300262451172e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 65700, "total_steps": 72479, "tokens": 34445721600, "loss": 4.317447245121002, "l_mix": 2.6759895384311676, "l_a": 2.75979745388031, "l_c": 2.7117278575897217, "z_loss": 44.3787055015564, "lr": 0.0015855527200240676, "wd": 0.01, "grad_norm": 0.11633359640836716, "tokens_per_second": 1467860.6044811662, "eta_seconds": 2421.3119019269943, "data_wait_seconds": 1.4185905456542969e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16149294, "gate_a2b/L1_absmax": 0.6796189, "gate_a2b/L1_rms": 0.17118075, "gate_a2b/L3_absmean": 0.15227675, "gate_a2b/L3_absmax": 0.49989468, "gate_a2b/L3_rms": 0.15785509, "gate_a2b/L5_absmean": 0.22008438, "gate_a2b/L5_absmax": 0.73145211, "gate_a2b/L5_rms": 0.22606538, "gate_c2b/L1_absmean": 0.0413118, "gate_c2b/L1_absmax": 0.61905903, "gate_c2b/L1_rms": 0.07114124, "gate_c2b/L3_absmean": 0.1653409, "gate_c2b/L3_absmax": 0.38762373, "gate_c2b/L3_rms": 0.1688754, "gate_c2b/L5_absmean": 0.46933433, "gate_c2b/L5_absmax": 0.67579538, "gate_c2b/L5_rms": 0.47830287, "gate_b2a/L7_absmean": 0.05150133, "gate_b2a/L7_absmax": 0.33333617, "gate_b2a/L7_rms": 0.05723947, "gate_b2a/L11_absmean": 0.08413842, "gate_b2a/L11_absmax": 0.61387771, "gate_b2a/L11_rms": 0.09318122, "gate_b2a/L14_absmean": 0.2184768, "gate_b2a/L14_absmax": 0.80885309, "gate_b2a/L14_rms": 0.22753876, "gate_b2c/L3_absmean": 0.03669181, "gate_b2c/L3_absmax": 0.55030483, "gate_b2c/L3_rms": 0.04678974, "gate_b2c/L5_absmean": 0.03802449, "gate_b2c/L5_absmax": 0.52906281, "gate_b2c/L5_rms": 0.04696108, "gate_b2c/L6_absmean": 0.05494366, "gate_b2c/L6_absmax": 0.3390176, "gate_b2c/L6_rms": 0.06064203}, "streams": {"cos_a2b/round0": 0.357107, "cos_c2b/round0": 0.243256, "cos_b2a/round0": 0.157115, "cos_b2c/round0": 0.149931, "rms_a/round0": 7.580427, "rms_b/round0": 12.817146, "rms_c/round0": 4.925807, "cos_a2b/round1": 0.408982, "cos_c2b/round1": 0.29462, "cos_b2a/round1": 0.245918, "cos_b2c/round1": 0.097202, "rms_a/round1": 14.831656, "rms_b/round1": 30.205557, "rms_c/round1": 10.551132, "cos_a2b/round2": 0.531799, "cos_c2b/round2": 0.517439, "cos_b2a/round2": 0.094414, "cos_b2c/round2": 0.176991, "rms_a/round2": 43.241203, "rms_b/round2": 84.285629, "rms_c/round2": 20.701704}, "alpha_mean": 0.4427448608204685, "alpha_hist": [0.00549, 0.0611, 0.17015, 0.20473, 0.18437, 0.16603, 0.12726, 0.06856, 0.01187, 0.00044], "alpha_by_type": {"special": {"share": 0.00066, "mean": 0.57112, "std": 0.12413}, "whitespace": {"share": 0.03825, "mean": 0.54636, "std": 0.14624}, "digit": {"share": 0.06226, "mean": 0.45921, "std": 0.13381}, "word_start": {"share": 0.53624, "mean": 0.45362, "std": 0.16707}, "word_cont": {"share": 0.2529, "mean": 0.33478, "std": 0.12583}, "punct": {"share": 0.1097, "mean": 0.59226, "std": 0.14843}}, "alpha_p05": 0.20025, "alpha_p50": 0.43134, "alpha_p95": 0.72056}
{"step": 65710, "total_steps": 72479, "tokens": 34450964480, "loss": 4.409798741340637, "l_mix": 2.7342343032360077, "l_a": 2.8163035213947296, "l_c": 2.7689110338687897, "z_loss": 44.532522201538086, "lr": 0.001582474838406128, "wd": 0.01, "grad_norm": 0.12807905673980713, "tokens_per_second": 1419295.2376868404, "eta_seconds": 2500.470217728615, "data_wait_seconds": 3.787875175476074e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 65720, "total_steps": 72479, "tokens": 34456207360, "loss": 4.828528702259064, "l_mix": 2.995862513780594, "l_a": 3.0786082446575165, "l_c": 3.0302789211273193, "z_loss": 41.15569019317627, "lr": 0.0015793933443435523, "wd": 0.01, "grad_norm": 0.15206097066402435, "tokens_per_second": 1437062.9955303827, "eta_seconds": 2465.9062288999557, "data_wait_seconds": 2.7865171432495117e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 65730, "total_steps": 72479, "tokens": 34461450240, "loss": 4.727077484130859, "l_mix": 2.9336737394332886, "l_a": 3.0074155032634735, "l_c": 2.9705969393253326, "z_loss": 42.41377878189087, "lr": 0.0015763082635759328, "wd": 0.01, "grad_norm": 0.11304297298192978, "tokens_per_second": 1386684.0488324547, "eta_seconds": 2551.712998342514, "data_wait_seconds": 5.754828453063965e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 65740, "total_steps": 72479, "tokens": 34466693120, "loss": 4.674173653125763, "l_mix": 2.9006617665290833, "l_a": 2.975255101919174, "l_c": 2.9364510476589203, "z_loss": 43.37348031997681, "lr": 0.0015732196218728237, "wd": 0.01, "grad_norm": 0.13018319010734558, "tokens_per_second": 1440444.3514557963, "eta_seconds": 2452.8381318092347, "data_wait_seconds": 2.872943878173828e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 65750, "total_steps": 72479, "tokens": 34471936000, "loss": 4.643889486789703, "l_mix": 2.881350100040436, "l_a": 2.9583799242973328, "l_c": 2.9167510271072388, "z_loss": 44.253803730010986, "lr": 0.0015701274450335228, "wd": 0.01, "grad_norm": 0.1648399531841278, "tokens_per_second": 1424732.8252846391, "eta_seconds": 2476.2073908805846, "data_wait_seconds": 1.6361474990844727e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 65760, "total_steps": 72479, "tokens": 34477178880, "loss": 4.672662436962128, "l_mix": 2.899298906326294, "l_a": 2.9736447930336, "l_c": 2.9375669360160828, "z_loss": 43.94338798522949, "lr": 0.0015670317588868565, "wd": 0.01, "grad_norm": 0.1225203424692154, "tokens_per_second": 1388457.9109454155, "eta_seconds": 2537.1248521327975, "data_wait_seconds": 2.446770668029785e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 65770, "total_steps": 72479, "tokens": 34482421760, "loss": 2.8285476863384247, "l_mix": 1.7512123137712479, "l_a": 1.8158996403217316, "l_c": 1.7752180993556976, "z_loss": 54.888121128082275, "lr": 0.0015639325892909648, "wd": 0.01, "grad_norm": 0.12242747098207474, "tokens_per_second": 1477541.1123999797, "eta_seconds": 2380.609353256226, "data_wait_seconds": 2.3990869522094727e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 65780, "total_steps": 72479, "tokens": 34487664640, "loss": 4.341181576251984, "l_mix": 2.691330075263977, "l_a": 2.773193895816803, "l_c": 2.726311206817627, "z_loss": 45.085089683532715, "lr": 0.0015608299621330843, "wd": 0.01, "grad_norm": 0.1405322551727295, "tokens_per_second": 1380847.412858084, "eta_seconds": 2543.514424037933, "data_wait_seconds": 1.6480684280395508e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 65790, "total_steps": 72479, "tokens": 34492907520, "loss": 4.377001106739044, "l_mix": 2.713272362947464, "l_a": 2.795119822025299, "l_c": 2.75064280629158, "z_loss": 45.85795879364014, "lr": 0.0015577239033293332, "wd": 0.01, "grad_norm": 0.1298554241657257, "tokens_per_second": 1325255.4407681266, "eta_seconds": 2646.253940272331, "data_wait_seconds": 2.1666288375854492e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 65800, "total_steps": 72479, "tokens": 34498150400, "loss": 4.423953354358673, "l_mix": 2.742991179227829, "l_a": 2.8224398493766785, "l_c": 2.780767172574997, "z_loss": 44.81697702407837, "lr": 0.0015546144388244932, "wd": 0.01, "grad_norm": 0.13008223474025726, "tokens_per_second": 1334001.0682523903, "eta_seconds": 2624.975073361397, "data_wait_seconds": 1.5854835510253906e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16185944, "gate_a2b/L1_absmax": 0.67869914, "gate_a2b/L1_rms": 0.17157187, "gate_a2b/L3_absmean": 0.15244518, "gate_a2b/L3_absmax": 0.50126439, "gate_a2b/L3_rms": 0.15802963, "gate_a2b/L5_absmean": 0.21989799, "gate_a2b/L5_absmax": 0.73154449, "gate_a2b/L5_rms": 0.22590828, "gate_c2b/L1_absmean": 0.04117959, "gate_c2b/L1_absmax": 0.61832905, "gate_c2b/L1_rms": 0.07097482, "gate_c2b/L3_absmean": 0.16536205, "gate_c2b/L3_absmax": 0.38636303, "gate_c2b/L3_rms": 0.16888157, "gate_c2b/L5_absmean": 0.46954963, "gate_c2b/L5_absmax": 0.67641485, "gate_c2b/L5_rms": 0.47857746, "gate_b2a/L7_absmean": 0.05157049, "gate_b2a/L7_absmax": 0.33434531, "gate_b2a/L7_rms": 0.05736773, "gate_b2a/L11_absmean": 0.08394689, "gate_b2a/L11_absmax": 0.61659384, "gate_b2a/L11_rms": 0.09307756, "gate_b2a/L14_absmean": 0.21856922, "gate_b2a/L14_absmax": 0.8089726, "gate_b2a/L14_rms": 0.22763741, "gate_b2c/L3_absmean": 0.03640178, "gate_b2c/L3_absmax": 0.55099177, "gate_b2c/L3_rms": 0.04664008, "gate_b2c/L5_absmean": 0.03791492, "gate_b2c/L5_absmax": 0.53137869, "gate_b2c/L5_rms": 0.04694941, "gate_b2c/L6_absmean": 0.05473859, "gate_b2c/L6_absmax": 0.33925816, "gate_b2c/L6_rms": 0.06048241}, "streams": {"cos_a2b/round0": 0.358163, "cos_c2b/round0": 0.240097, "cos_b2a/round0": 0.160084, "cos_b2c/round0": 0.153554, "rms_a/round0": 7.540156, "rms_b/round0": 12.876718, "rms_c/round0": 4.932864, "cos_a2b/round1": 0.420553, "cos_c2b/round1": 0.307078, "cos_b2a/round1": 0.253163, "cos_b2c/round1": 0.101032, "rms_a/round1": 14.788657, "rms_b/round1": 29.948835, "rms_c/round1": 10.446088, "cos_a2b/round2": 0.543975, "cos_c2b/round2": 0.511238, "cos_b2a/round2": 0.106796, "cos_b2c/round2": 0.18708, "rms_a/round2": 43.460361, "rms_b/round2": 83.574318, "rms_c/round2": 20.378048}, "alpha_mean": 0.4510935638767677, "alpha_hist": [0.00374, 0.04317, 0.16171, 0.22144, 0.18277, 0.16307, 0.14279, 0.0742, 0.00708, 3e-05], "alpha_by_type": {"special": {"share": 0.00038, "mean": 0.5769, "std": 0.08042}, "whitespace": {"share": 0.02144, "mean": 0.51712, "std": 0.11259}, "digit": {"share": 0.02603, "mean": 0.43959, "std": 0.12164}, "word_start": {"share": 0.61826, "mean": 0.47394, "std": 0.15811}, "word_cont": {"share": 0.24176, "mean": 0.3303, "std": 0.11763}, "punct": {"share": 0.09213, "mean": 0.60211, "std": 0.14442}}, "alpha_p05": 0.18477, "alpha_p50": 0.43806, "alpha_p95": 0.73412}
{"step": 65810, "total_steps": 72479, "tokens": 34503393280, "loss": 4.227348327636719, "l_mix": 2.619901120662689, "l_a": 2.7038075923919678, "l_c": 2.654349237680435, "z_loss": 45.246583461761475, "lr": 0.001551501594591795, "wd": 0.01, "grad_norm": 0.12358018755912781, "tokens_per_second": 1466531.0382456447, "eta_seconds": 2384.181841921806, "data_wait_seconds": 2.250075340270996e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 65820, "total_steps": 72479, "tokens": 34508636160, "loss": 4.455793082714081, "l_mix": 2.7629359364509583, "l_a": 2.8438029289245605, "l_c": 2.799054265022278, "z_loss": 44.610408782958984, "lr": 0.0015483853966326976, "wd": 0.01, "grad_norm": 0.12651757895946503, "tokens_per_second": 1450626.6756751016, "eta_seconds": 2406.70728764534, "data_wait_seconds": 2.4884939193725586e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 65830, "total_steps": 72479, "tokens": 34513879040, "loss": 3.006159871816635, "l_mix": 1.8616153299808502, "l_a": 1.9291472733020782, "l_c": 1.886001080274582, "z_loss": 53.329355239868164, "lr": 0.0015452658709766763, "wd": 0.01, "grad_norm": 0.138754203915596, "tokens_per_second": 1448581.9169035272, "eta_seconds": 2406.485177898407, "data_wait_seconds": 2.8371810913085938e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 65840, "total_steps": 72479, "tokens": 34519121920, "loss": 4.738144636154175, "l_mix": 2.9404630064964294, "l_a": 3.0151415169239044, "l_c": 2.977130174636841, "z_loss": 43.72099781036377, "lr": 0.0015421430436810015, "wd": 0.01, "grad_norm": 0.1585870087146759, "tokens_per_second": 1371681.0647276288, "eta_seconds": 2537.5782472372052, "data_wait_seconds": 2.5153160095214844e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 65850, "total_steps": 72479, "tokens": 34524364800, "loss": 4.651091158390045, "l_mix": 2.885888934135437, "l_a": 2.962250739336014, "l_c": 2.9217564463615417, "z_loss": 43.80067491531372, "lr": 0.0015390169408305212, "wd": 0.01, "grad_norm": 0.17356592416763306, "tokens_per_second": 1407945.540776303, "eta_seconds": 2468.4940229177478, "data_wait_seconds": 2.1249055862426758e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 65860, "total_steps": 72479, "tokens": 34529607680, "loss": 4.586469292640686, "l_mix": 2.846007317304611, "l_a": 2.920130342245102, "l_c": 2.881409376859665, "z_loss": 43.73131847381592, "lr": 0.0015358875885374446, "wd": 0.01, "grad_norm": 0.16436196863651276, "tokens_per_second": 1465268.6074498293, "eta_seconds": 2368.345472192764, "data_wait_seconds": 2.1964311599731445e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 65870, "total_steps": 72479, "tokens": 34534850560, "loss": 4.679872930049896, "l_mix": 2.903790146112442, "l_a": 2.97984379529953, "l_c": 2.940432131290436, "z_loss": 43.05513286590576, "lr": 0.0015327550129411239, "wd": 0.01, "grad_norm": 0.11608271300792694, "tokens_per_second": 1411424.3390216962, "eta_seconds": 2454.9806151151656, "data_wait_seconds": 1.7881393432617188e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 65880, "total_steps": 72479, "tokens": 34540093440, "loss": 4.7263911962509155, "l_mix": 2.9341457784175873, "l_a": 3.0075856745243073, "l_c": 2.96656534075737, "z_loss": 42.318819522857666, "lr": 0.0015296192402078344, "wd": 0.01, "grad_norm": 0.17616797983646393, "tokens_per_second": 1474292.278840898, "eta_seconds": 2346.7371847867967, "data_wait_seconds": 2.22623348236084e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 65890, "total_steps": 72479, "tokens": 34545336320, "loss": 4.3393861055374146, "l_mix": 2.6899361312389374, "l_a": 2.7720451951026917, "l_c": 2.7261213660240173, "z_loss": 45.759868144989014, "lr": 0.0015264802965305583, "wd": 0.01, "grad_norm": 0.12114737927913666, "tokens_per_second": 1447216.1657087763, "eta_seconds": 2387.0197927951813, "data_wait_seconds": 1.564621925354004e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 65900, "total_steps": 72479, "tokens": 34550579200, "loss": 4.209549695253372, "l_mix": 2.6087394654750824, "l_a": 2.6926023066043854, "l_c": 2.643431305885315, "z_loss": 46.18795871734619, "lr": 0.001523338208128763, "wd": 0.01, "grad_norm": 0.12030909210443497, "tokens_per_second": 1408488.9645204805, "eta_seconds": 2448.9299092054366, "data_wait_seconds": 1.5348196029663086e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.1616897, "gate_a2b/L1_absmax": 0.67742056, "gate_a2b/L1_rms": 0.17139684, "gate_a2b/L3_absmean": 0.15273882, "gate_a2b/L3_absmax": 0.49935102, "gate_a2b/L3_rms": 0.1583183, "gate_a2b/L5_absmean": 0.22022429, "gate_a2b/L5_absmax": 0.73099267, "gate_a2b/L5_rms": 0.22622833, "gate_c2b/L1_absmean": 0.04162922, "gate_c2b/L1_absmax": 0.6161412, "gate_c2b/L1_rms": 0.0711747, "gate_c2b/L3_absmean": 0.16519497, "gate_c2b/L3_absmax": 0.38433865, "gate_c2b/L3_rms": 0.16870886, "gate_c2b/L5_absmean": 0.46965471, "gate_c2b/L5_absmax": 0.67575759, "gate_c2b/L5_rms": 0.47863111, "gate_b2a/L7_absmean": 0.05138196, "gate_b2a/L7_absmax": 0.33377904, "gate_b2a/L7_rms": 0.0571725, "gate_b2a/L11_absmean": 0.08405189, "gate_b2a/L11_absmax": 0.61536688, "gate_b2a/L11_rms": 0.09314319, "gate_b2a/L14_absmean": 0.21876062, "gate_b2a/L14_absmax": 0.80755669, "gate_b2a/L14_rms": 0.227807, "gate_b2c/L3_absmean": 0.03662293, "gate_b2c/L3_absmax": 0.55083245, "gate_b2c/L3_rms": 0.04675817, "gate_b2c/L5_absmean": 0.03791981, "gate_b2c/L5_absmax": 0.53233886, "gate_b2c/L5_rms": 0.04696357, "gate_b2c/L6_absmean": 0.05494218, "gate_b2c/L6_absmax": 0.33813673, "gate_b2c/L6_rms": 0.06064434}, "streams": {"cos_a2b/round0": 0.354917, "cos_c2b/round0": 0.24334, "cos_b2a/round0": 0.164251, "cos_b2c/round0": 0.153274, "rms_a/round0": 7.410274, "rms_b/round0": 12.751086, "rms_c/round0": 4.911371, "cos_a2b/round1": 0.400098, "cos_c2b/round1": 0.293904, "cos_b2a/round1": 0.245628, "cos_b2c/round1": 0.103366, "rms_a/round1": 14.721403, "rms_b/round1": 30.44256, "rms_c/round1": 10.636722, "cos_a2b/round2": 0.523897, "cos_c2b/round2": 0.521815, "cos_b2a/round2": 0.10297, "cos_b2c/round2": 0.180588, "rms_a/round2": 43.22377, "rms_b/round2": 84.957497, "rms_c/round2": 20.879963}, "alpha_mean": 0.406244280954418, "alpha_hist": [0.01521, 0.10048, 0.20357, 0.19997, 0.17412, 0.14436, 0.10429, 0.0511, 0.00679, 0.00011], "alpha_by_type": {"special": {"share": 0.00072, "mean": 0.58255, "std": 0.17191}, "whitespace": {"share": 0.03593, "mean": 0.50843, "std": 0.10556}, "digit": {"share": 0.06648, "mean": 0.40116, "std": 0.14462}, "word_start": {"share": 0.49814, "mean": 0.42798, "std": 0.17019}, "word_cont": {"share": 0.27548, "mean": 0.29038, "std": 0.11906}, "punct": {"share": 0.12325, "mean": 0.54931, "std": 0.16051}}, "alpha_p05": 0.15714, "alpha_p50": 0.39467, "alpha_p95": 0.72253}
{"step": 65910, "total_steps": 72479, "tokens": 34555822080, "loss": 4.410449266433716, "l_mix": 2.7348800897598267, "l_a": 2.8156126737594604, "l_c": 2.769617795944214, "z_loss": 44.772645473480225, "lr": 0.0015201930012481845, "wd": 0.01, "grad_norm": 0.12144973874092102, "tokens_per_second": 1419120.661443919, "eta_seconds": 2426.888682246208, "data_wait_seconds": 2.0265579223632812e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 65920, "total_steps": 72479, "tokens": 34561064960, "loss": 4.330882906913757, "l_mix": 2.6846006512641907, "l_a": 2.7673164904117584, "l_c": 2.7202906608581543, "z_loss": 44.78646802902222, "lr": 0.0015170447021606074, "wd": 0.01, "grad_norm": 0.11758207529783249, "tokens_per_second": 1414693.8076258074, "eta_seconds": 2430.776874446869, "data_wait_seconds": 2.3990869522094727e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 65930, "total_steps": 72479, "tokens": 34566307840, "loss": 4.380701303482056, "l_mix": 2.715568780899048, "l_a": 2.799134314060211, "l_c": 2.7513073086738586, "z_loss": 44.96836614608765, "lr": 0.0015138933371636454, "wd": 0.01, "grad_norm": 0.09880688786506653, "tokens_per_second": 1467303.6985283948, "eta_seconds": 2340.0486998319625, "data_wait_seconds": 2.2143125534057617e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 65940, "total_steps": 72479, "tokens": 34571550720, "loss": 3.0430728793144226, "l_mix": 1.8854502886533737, "l_a": 1.9467645585536957, "l_c": 1.9119772613048553, "z_loss": 55.17898082733154, "lr": 0.0015107389325805212, "wd": 0.01, "grad_norm": 0.13073310256004333, "tokens_per_second": 1456911.5547778376, "eta_seconds": 2353.1416308403013, "data_wait_seconds": 3.11732292175293e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 65950, "total_steps": 72479, "tokens": 34576793600, "loss": 3.0564280450344086, "l_mix": 1.8802907317876816, "l_a": 1.992772564291954, "l_c": 1.9276849627494812, "z_loss": 50.50917911529541, "lr": 0.0015075815147598474, "wd": 0.01, "grad_norm": 0.20132577419281006, "tokens_per_second": 1386975.8201721294, "eta_seconds": 2468.014439916611, "data_wait_seconds": 1.6987323760986328e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 65960, "total_steps": 72479, "tokens": 34582036480, "loss": 3.0035471618175507, "l_mix": 1.847974568605423, "l_a": 1.9588779360055923, "l_c": 1.8930304199457169, "z_loss": 49.42993354797363, "lr": 0.0015044211100754057, "wd": 0.01, "grad_norm": 0.12494363635778427, "tokens_per_second": 1221669.9487992928, "eta_seconds": 2797.673361253738, "data_wait_seconds": 2.041459083557129e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 65970, "total_steps": 72479, "tokens": 34587279360, "loss": 2.9767561554908752, "l_mix": 1.8316369205713272, "l_a": 1.9409839063882828, "l_c": 1.8760800659656525, "z_loss": 48.907408237457275, "lr": 0.0015012577449259273, "wd": 0.01, "grad_norm": 0.14674270153045654, "tokens_per_second": 1356390.9055981652, "eta_seconds": 2515.934438896179, "data_wait_seconds": 2.491474151611328e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 65980, "total_steps": 72479, "tokens": 34592522240, "loss": 4.469637036323547, "l_mix": 2.7715014219284058, "l_a": 2.8526935279369354, "l_c": 2.8077581226825714, "z_loss": 45.64970588684082, "lr": 0.001498091445734872, "wd": 0.01, "grad_norm": 0.11254292726516724, "tokens_per_second": 1412310.1505311457, "eta_seconds": 2412.605836415291, "data_wait_seconds": 2.1249055862426758e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 65990, "total_steps": 72479, "tokens": 34597765120, "loss": 4.3594478368759155, "l_mix": 2.7022103667259216, "l_a": 2.7850941717624664, "l_c": 2.739030420780182, "z_loss": 45.37475252151489, "lr": 0.0014949222389502074, "wd": 0.01, "grad_norm": 0.1236320286989212, "tokens_per_second": 1471304.0570910787, "eta_seconds": 2312.3057505369184, "data_wait_seconds": 1.919269561767578e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 66000, "total_steps": 72479, "tokens": 34603008000, "loss": 4.420567035675049, "l_mix": 2.740909665822983, "l_a": 2.8225900530815125, "l_c": 2.776267498731613, "z_loss": 44.81316614151001, "lr": 0.0014917501510441878, "wd": 0.01, "grad_norm": 0.14935685694217682, "tokens_per_second": 1453481.9351767132, "eta_seconds": 2337.05137283802, "data_wait_seconds": 2.086162567138672e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.16227661, "gate_a2b/L1_absmax": 0.67674482, "gate_a2b/L1_rms": 0.17190285, "gate_a2b/L3_absmean": 0.15295795, "gate_a2b/L3_absmax": 0.49984562, "gate_a2b/L3_rms": 0.15850271, "gate_a2b/L5_absmean": 0.22076666, "gate_a2b/L5_absmax": 0.73096818, "gate_a2b/L5_rms": 0.22672948, "gate_c2b/L1_absmean": 0.04210377, "gate_c2b/L1_absmax": 0.61596817, "gate_c2b/L1_rms": 0.0713206, "gate_c2b/L3_absmean": 0.16475511, "gate_c2b/L3_absmax": 0.38505757, "gate_c2b/L3_rms": 0.16827859, "gate_c2b/L5_absmean": 0.4697338, "gate_c2b/L5_absmax": 0.67554957, "gate_c2b/L5_rms": 0.47874239, "gate_b2a/L7_absmean": 0.05185637, "gate_b2a/L7_absmax": 0.33275673, "gate_b2a/L7_rms": 0.05756085, "gate_b2a/L11_absmean": 0.08417533, "gate_b2a/L11_absmax": 0.61453229, "gate_b2a/L11_rms": 0.09324922, "gate_b2a/L14_absmean": 0.21922238, "gate_b2a/L14_absmax": 0.80510944, "gate_b2a/L14_rms": 0.22820373, "gate_b2c/L3_absmean": 0.03686578, "gate_b2c/L3_absmax": 0.55095619, "gate_b2c/L3_rms": 0.04701447, "gate_b2c/L5_absmean": 0.03787331, "gate_b2c/L5_absmax": 0.53274959, "gate_b2c/L5_rms": 0.04697539, "gate_b2c/L6_absmean": 0.05525351, "gate_b2c/L6_absmax": 0.33906078, "gate_b2c/L6_rms": 0.06091614}, "streams": {"cos_a2b/round0": 0.354044, "cos_c2b/round0": 0.234624, "cos_b2a/round0": 0.160544, "cos_b2c/round0": 0.145682, "rms_a/round0": 7.509902, "rms_b/round0": 12.931198, "rms_c/round0": 4.980109, "cos_a2b/round1": 0.402277, "cos_c2b/round1": 0.295612, "cos_b2a/round1": 0.249124, "cos_b2c/round1": 0.108105, "rms_a/round1": 14.980261, "rms_b/round1": 30.672215, "rms_c/round1": 10.881038, "cos_a2b/round2": 0.528145, "cos_c2b/round2": 0.518797, "cos_b2a/round2": 0.093006, "cos_b2c/round2": 0.179527, "rms_a/round2": 43.543751, "rms_b/round2": 84.369476, "rms_c/round2": 21.493065}, "alpha_mean": 0.4189276665313173, "alpha_hist": [0.01686, 0.09291, 0.18822, 0.19525, 0.17232, 0.14787, 0.1115, 0.06401, 0.01077, 0.00029], "alpha_by_type": {"special": {"share": 0.00146, "mean": 0.5623, "std": 0.11718}, "whitespace": {"share": 0.03401, "mean": 0.52189, "std": 0.137}, "digit": {"share": 0.0544, "mean": 0.46877, "std": 0.12321}, "word_start": {"share": 0.51036, "mean": 0.43794, "std": 0.17405}, "word_cont": {"share": 0.28212, "mean": 0.30164, "std": 0.13332}, "punct": {"share": 0.11765, "mean": 0.56311, "std": 0.16461}}, "alpha_p05": 0.16318, "alpha_p50": 0.39304, "alpha_p95": 0.72096}
{"step": 66010, "total_steps": 72479, "tokens": 34608250880, "loss": 4.332248866558075, "l_mix": 2.6855059266090393, "l_a": 2.7692519426345825, "l_c": 2.719891130924225, "z_loss": 45.23072528839111, "lr": 0.0014885752085131345, "wd": 0.01, "grad_norm": 0.10781702399253845, "tokens_per_second": 1413485.11964424, "eta_seconds": 2399.47278175354, "data_wait_seconds": 1.7821788787841797e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 66020, "total_steps": 72479, "tokens": 34613493760, "loss": 4.412529706954956, "l_mix": 2.7358461916446686, "l_a": 2.8177265524864197, "l_c": 2.7712183594703674, "z_loss": 44.360400676727295, "lr": 0.0014853974378772115, "wd": 0.01, "grad_norm": 0.13841165602207184, "tokens_per_second": 1331119.9821889678, "eta_seconds": 2544.0052266597745, "data_wait_seconds": 1.919269561767578e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 66030, "total_steps": 72479, "tokens": 34618736640, "loss": 4.420488119125366, "l_mix": 2.740756332874298, "l_a": 2.822183281183243, "l_c": 2.7769225239753723, "z_loss": 45.55917167663574, "lr": 0.001482216865680208, "wd": 0.01, "grad_norm": 0.13471238315105438, "tokens_per_second": 1380493.5595552686, "eta_seconds": 2449.2206346035005, "data_wait_seconds": 1.6957521438598633e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 66040, "total_steps": 72479, "tokens": 34623979520, "loss": 4.3896602392196655, "l_mix": 2.721385896205902, "l_a": 2.802700102329254, "l_c": 2.7582144141197205, "z_loss": 45.42837953567505, "lr": 0.0014790335184893132, "wd": 0.01, "grad_norm": 0.11374027281999588, "tokens_per_second": 1176131.8547809683, "eta_seconds": 2870.333303427696, "data_wait_seconds": 1.84476375579834e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 66050, "total_steps": 72479, "tokens": 34629222400, "loss": 2.850979655981064, "l_mix": 1.765265628695488, "l_a": 1.8271831274032593, "l_c": 1.7918633818626404, "z_loss": 54.31960916519165, "lr": 0.0014758474228948957, "wd": 0.01, "grad_norm": 0.13514959812164307, "tokens_per_second": 1449176.7461930516, "eta_seconds": 2325.9050773859026, "data_wait_seconds": 2.0295381546020508e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 66060, "total_steps": 72479, "tokens": 34634465280, "loss": 3.0541476905345917, "l_mix": 1.87928007543087, "l_a": 1.9916878640651703, "l_c": 1.9245372712612152, "z_loss": 48.375723361968994, "lr": 0.001472658605510282, "wd": 0.01, "grad_norm": 0.15211854875087738, "tokens_per_second": 1390926.4329471122, "eta_seconds": 2419.541819238663, "data_wait_seconds": 2.428889274597168e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 66070, "total_steps": 72479, "tokens": 34639708160, "loss": 3.047918200492859, "l_mix": 1.8760134726762772, "l_a": 1.9848344922065735, "l_c": 1.921514317393303, "z_loss": 47.49269676208496, "lr": 0.0014694670929715334, "wd": 0.01, "grad_norm": 0.18167518079280853, "tokens_per_second": 1304641.9104644228, "eta_seconds": 2575.5433464527127, "data_wait_seconds": 2.5600194931030273e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 66080, "total_steps": 72479, "tokens": 34644951040, "loss": 3.0142035484313965, "l_mix": 1.854900524020195, "l_a": 1.9651314616203308, "l_c": 1.8992117792367935, "z_loss": 48.15399217605591, "lr": 0.0014662729119372226, "wd": 0.01, "grad_norm": 0.13478529453277588, "tokens_per_second": 1411640.6130371303, "eta_seconds": 2376.609797859192, "data_wait_seconds": 2.580881118774414e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 66090, "total_steps": 72479, "tokens": 34650193920, "loss": 4.284588783979416, "l_mix": 2.655573606491089, "l_a": 2.7401742935180664, "l_c": 2.689876079559326, "z_loss": 46.383413791656494, "lr": 0.0014630760890882142, "wd": 0.01, "grad_norm": 0.15742909908294678, "tokens_per_second": 1415842.0266350613, "eta_seconds": 2365.854360151291, "data_wait_seconds": 2.5033950805664062e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 66100, "total_steps": 72479, "tokens": 34655436800, "loss": 4.437559545040131, "l_mix": 2.7516975104808807, "l_a": 2.8317605555057526, "l_c": 2.7877791821956635, "z_loss": 45.78732490539551, "lr": 0.0014598766511274386, "wd": 0.01, "grad_norm": 0.16435599327087402, "tokens_per_second": 1350671.987864104, "eta_seconds": 2476.1253524541853, "data_wait_seconds": 1.5854835510253906e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16228592, "gate_a2b/L1_absmax": 0.67747611, "gate_a2b/L1_rms": 0.17199425, "gate_a2b/L3_absmean": 0.15315071, "gate_a2b/L3_absmax": 0.50077683, "gate_a2b/L3_rms": 0.15863617, "gate_a2b/L5_absmean": 0.2205613, "gate_a2b/L5_absmax": 0.73239583, "gate_a2b/L5_rms": 0.22654961, "gate_c2b/L1_absmean": 0.04244405, "gate_c2b/L1_absmax": 0.61616826, "gate_c2b/L1_rms": 0.07156223, "gate_c2b/L3_absmean": 0.16513203, "gate_c2b/L3_absmax": 0.38366622, "gate_c2b/L3_rms": 0.16862835, "gate_c2b/L5_absmean": 0.46948859, "gate_c2b/L5_absmax": 0.67775822, "gate_c2b/L5_rms": 0.47851783, "gate_b2a/L7_absmean": 0.05183602, "gate_b2a/L7_absmax": 0.33251217, "gate_b2a/L7_rms": 0.0575735, "gate_b2a/L11_absmean": 0.08414157, "gate_b2a/L11_absmax": 0.61568791, "gate_b2a/L11_rms": 0.0932536, "gate_b2a/L14_absmean": 0.21889919, "gate_b2a/L14_absmax": 0.80749547, "gate_b2a/L14_rms": 0.22791716, "gate_b2c/L3_absmean": 0.03669218, "gate_b2c/L3_absmax": 0.5516941, "gate_b2c/L3_rms": 0.0468419, "gate_b2c/L5_absmean": 0.03792142, "gate_b2c/L5_absmax": 0.53324455, "gate_b2c/L5_rms": 0.046982, "gate_b2c/L6_absmean": 0.05512873, "gate_b2c/L6_absmax": 0.33793774, "gate_b2c/L6_rms": 0.06081538}, "streams": {"cos_a2b/round0": 0.355197, "cos_c2b/round0": 0.237765, "cos_b2a/round0": 0.159929, "cos_b2c/round0": 0.149056, "rms_a/round0": 7.486361, "rms_b/round0": 12.802393, "rms_c/round0": 4.945174, "cos_a2b/round1": 0.415575, "cos_c2b/round1": 0.306494, "cos_b2a/round1": 0.253741, "cos_b2c/round1": 0.100042, "rms_a/round1": 14.66914, "rms_b/round1": 29.941158, "rms_c/round1": 10.642747, "cos_a2b/round2": 0.5317, "cos_c2b/round2": 0.521622, "cos_b2a/round2": 0.107865, "cos_b2c/round2": 0.180325, "rms_a/round2": 43.327057, "rms_b/round2": 83.48101, "rms_c/round2": 20.968845}, "alpha_mean": 0.43052311800287413, "alpha_hist": [0.00896, 0.07948, 0.16406, 0.18993, 0.19298, 0.19084, 0.12126, 0.0472, 0.00525, 5e-05], "alpha_by_type": {"special": {"share": 0.00079, "mean": 0.52736, "std": 0.11135}, "whitespace": {"share": 0.04176, "mean": 0.55068, "std": 0.1073}, "digit": {"share": 0.02826, "mean": 0.42607, "std": 0.1156}, "word_start": {"share": 0.53271, "mean": 0.43807, "std": 0.16142}, "word_cont": {"share": 0.28436, "mean": 0.34487, "std": 0.14062}, "punct": {"share": 0.11211, "mean": 0.56758, "std": 0.14661}}, "alpha_p05": 0.16132, "alpha_p50": 0.4347, "alpha_p95": 0.72272}
{"step": 66110, "total_steps": 72479, "tokens": 34660679680, "loss": 4.420102179050446, "l_mix": 2.740694671869278, "l_a": 2.824219971895218, "l_c": 2.7738046050071716, "z_loss": 44.831583976745605, "lr": 0.00145667462477967, "wd": 0.01, "grad_norm": 0.12162714451551437, "tokens_per_second": 1465179.179425717, "eta_seconds": 2279.0320248126986, "data_wait_seconds": 1.5169382095336914e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 66120, "total_steps": 72479, "tokens": 34665922560, "loss": 4.421916723251343, "l_mix": 2.741650313138962, "l_a": 2.8233282268047333, "l_c": 2.7775597274303436, "z_loss": 44.59829235076904, "lr": 0.0014534700367913036, "wd": 0.01, "grad_norm": 0.10266389697790146, "tokens_per_second": 1418225.3741711709, "eta_seconds": 2350.788141798973, "data_wait_seconds": 3.746151924133301e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 66130, "total_steps": 72479, "tokens": 34671165440, "loss": 4.724717974662781, "l_mix": 2.932429939508438, "l_a": 3.0064888894557953, "l_c": 2.9678046703338623, "z_loss": 43.31820249557495, "lr": 0.0014502629139301316, "wd": 0.01, "grad_norm": 0.11619244515895844, "tokens_per_second": 1458772.8908015557, "eta_seconds": 2281.852461743355, "data_wait_seconds": 2.327561378479004e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 66140, "total_steps": 72479, "tokens": 34676408320, "loss": 4.666810035705566, "l_mix": 2.896373838186264, "l_a": 2.971091538667679, "l_c": 2.9303620755672455, "z_loss": 42.95444440841675, "lr": 0.0014470532829851202, "wd": 0.01, "grad_norm": 0.10811594873666763, "tokens_per_second": 1393595.7225695224, "eta_seconds": 2384.8104426383975, "data_wait_seconds": 2.4497509002685547e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 66150, "total_steps": 72479, "tokens": 34681651200, "loss": 4.738088667392731, "l_mix": 2.9403414726257324, "l_a": 3.016543209552765, "l_c": 2.975946992635727, "z_loss": 42.44466018676758, "lr": 0.0014438411707661852, "wd": 0.01, "grad_norm": 0.10936258733272552, "tokens_per_second": 1436750.71159047, "eta_seconds": 2309.5299172163013, "data_wait_seconds": 2.9474496841430664e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 66160, "total_steps": 72479, "tokens": 34686894080, "loss": 4.6529746651649475, "l_mix": 2.887917995452881, "l_a": 2.9612579941749573, "l_c": 2.9222640991210938, "z_loss": 43.65074396133423, "lr": 0.0014406266041039695, "wd": 0.01, "grad_norm": 0.13594557344913483, "tokens_per_second": 1430410.3026128162, "eta_seconds": 2316.101796770096, "data_wait_seconds": 3.540515899658203e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 66170, "total_steps": 72479, "tokens": 34692136960, "loss": 3.0290148556232452, "l_mix": 1.8635694980621338, "l_a": 1.974022001028061, "l_c": 1.9107955247163773, "z_loss": 48.69315433502197, "lr": 0.0014374096098496154, "wd": 0.01, "grad_norm": 0.15395307540893555, "tokens_per_second": 1417211.190882408, "eta_seconds": 2333.973237919807, "data_wait_seconds": 4.360079765319824e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 66180, "total_steps": 72479, "tokens": 34697379840, "loss": 3.056850880384445, "l_mix": 1.880590781569481, "l_a": 1.994281068444252, "l_c": 1.9265857636928558, "z_loss": 47.57090997695923, "lr": 0.001434190214874546, "wd": 0.01, "grad_norm": 0.16761508584022522, "tokens_per_second": 1290022.619771819, "eta_seconds": 2560.024964976311, "data_wait_seconds": 1.659989356994629e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 66190, "total_steps": 72479, "tokens": 34702622720, "loss": 3.0232798159122467, "l_mix": 1.8603147864341736, "l_a": 1.9705371707677841, "l_c": 1.906012937426567, "z_loss": 48.88102388381958, "lr": 0.0014309684460702351, "wd": 0.01, "grad_norm": 0.14764507114887238, "tokens_per_second": 1429262.030584844, "eta_seconds": 2306.957829594612, "data_wait_seconds": 2.3245811462402344e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 66200, "total_steps": 72479, "tokens": 34707865600, "loss": 4.363023936748505, "l_mix": 2.704132527112961, "l_a": 2.789452701807022, "l_c": 2.7401849925518036, "z_loss": 45.534016609191895, "lr": 0.0014277443303479852, "wd": 0.01, "grad_norm": 0.11216431856155396, "tokens_per_second": 1416733.8500343484, "eta_seconds": 2323.657581782341, "data_wait_seconds": 2.60770320892334e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16270903, "gate_a2b/L1_absmax": 0.67657715, "gate_a2b/L1_rms": 0.17225744, "gate_a2b/L3_absmean": 0.15332873, "gate_a2b/L3_absmax": 0.50078857, "gate_a2b/L3_rms": 0.15886773, "gate_a2b/L5_absmean": 0.22081326, "gate_a2b/L5_absmax": 0.73144692, "gate_a2b/L5_rms": 0.22677295, "gate_c2b/L1_absmean": 0.04231789, "gate_c2b/L1_absmax": 0.61425698, "gate_c2b/L1_rms": 0.07128993, "gate_c2b/L3_absmean": 0.16481261, "gate_c2b/L3_absmax": 0.38368261, "gate_c2b/L3_rms": 0.16830859, "gate_c2b/L5_absmean": 0.46952578, "gate_c2b/L5_absmax": 0.6771782, "gate_c2b/L5_rms": 0.47856107, "gate_b2a/L7_absmean": 0.05144138, "gate_b2a/L7_absmax": 0.33338568, "gate_b2a/L7_rms": 0.05720146, "gate_b2a/L11_absmean": 0.08428705, "gate_b2a/L11_absmax": 0.61465377, "gate_b2a/L11_rms": 0.09332013, "gate_b2a/L14_absmean": 0.21932445, "gate_b2a/L14_absmax": 0.80535686, "gate_b2a/L14_rms": 0.22825225, "gate_b2c/L3_absmean": 0.03717278, "gate_b2c/L3_absmax": 0.55054104, "gate_b2c/L3_rms": 0.04716944, "gate_b2c/L5_absmean": 0.03826382, "gate_b2c/L5_absmax": 0.53284764, "gate_b2c/L5_rms": 0.04725583, "gate_b2c/L6_absmean": 0.05548564, "gate_b2c/L6_absmax": 0.33743545, "gate_b2c/L6_rms": 0.06108306}, "streams": {"cos_a2b/round0": 0.356405, "cos_c2b/round0": 0.238529, "cos_b2a/round0": 0.171901, "cos_b2c/round0": 0.151177, "rms_a/round0": 7.476386, "rms_b/round0": 12.823802, "rms_c/round0": 4.96247, "cos_a2b/round1": 0.414039, "cos_c2b/round1": 0.306727, "cos_b2a/round1": 0.260776, "cos_b2c/round1": 0.111062, "rms_a/round1": 14.81623, "rms_b/round1": 30.150259, "rms_c/round1": 10.777446, "cos_a2b/round2": 0.541008, "cos_c2b/round2": 0.517511, "cos_b2a/round2": 0.095196, "cos_b2c/round2": 0.179934, "rms_a/round2": 43.619259, "rms_b/round2": 83.006729, "rms_c/round2": 21.321619}, "alpha_mean": 0.43475728816979853, "alpha_hist": [0.01309, 0.09171, 0.18118, 0.17691, 0.16319, 0.14835, 0.12749, 0.08199, 0.01596, 0.00014], "alpha_by_type": {"special": {"share": 0.00119, "mean": 0.5606, "std": 0.1175}, "whitespace": {"share": 0.02663, "mean": 0.5039, "std": 0.12721}, "digit": {"share": 0.02773, "mean": 0.45448, "std": 0.13056}, "word_start": {"share": 0.57364, "mean": 0.45504, "std": 0.17943}, "word_cont": {"share": 0.25949, "mean": 0.31377, "std": 0.14002}, "punct": {"share": 0.11133, "mean": 0.58947, "std": 0.17674}}, "alpha_p05": 0.15817, "alpha_p50": 0.42895, "alpha_p95": 0.74798}
{"step": 66210, "total_steps": 72479, "tokens": 34713108480, "loss": 4.404447555541992, "l_mix": 2.730683773756027, "l_a": 2.814308673143387, "l_c": 2.7649038434028625, "z_loss": 44.58378458023071, "lr": 0.001424517894638704, "wd": 0.01, "grad_norm": 0.13258683681488037, "tokens_per_second": 1462787.7163264873, "eta_seconds": 2246.916237616539, "data_wait_seconds": 1.7255544662475586e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 66220, "total_steps": 72479, "tokens": 34718351360, "loss": 4.397312045097351, "l_mix": 2.726026326417923, "l_a": 2.8089740574359894, "l_c": 2.761978179216385, "z_loss": 45.21000003814697, "lr": 0.0014212891658926764, "wd": 0.01, "grad_norm": 0.1015813946723938, "tokens_per_second": 1438294.581036872, "eta_seconds": 2281.5344194889067, "data_wait_seconds": 1.7195940017700195e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 66230, "total_steps": 72479, "tokens": 34723594240, "loss": 3.048463523387909, "l_mix": 1.8753228336572647, "l_a": 1.9898263663053513, "l_c": 1.920642450451851, "z_loss": 49.5096640586853, "lr": 0.001418058171079342, "wd": 0.01, "grad_norm": 0.1503356397151947, "tokens_per_second": 1363447.5274503292, "eta_seconds": 2402.9349469184876, "data_wait_seconds": 3.8743019104003906e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 66240, "total_steps": 72479, "tokens": 34728837120, "loss": 3.0053901374340057, "l_mix": 1.8491201400756836, "l_a": 1.9611341208219528, "l_c": 1.8930991291999817, "z_loss": 50.493130683898926, "lr": 0.0014148249371870678, "wd": 0.01, "grad_norm": 0.14229004085063934, "tokens_per_second": 1317497.284631464, "eta_seconds": 2482.762484717369, "data_wait_seconds": 1.6629695892333984e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 66250, "total_steps": 72479, "tokens": 34734080000, "loss": 3.070512682199478, "l_mix": 1.8895482122898102, "l_a": 2.0006738752126694, "l_c": 1.9358743876218796, "z_loss": 47.67062044143677, "lr": 0.0014115894912229245, "wd": 0.01, "grad_norm": 0.14755697548389435, "tokens_per_second": 1349551.2961551328, "eta_seconds": 2419.907980751991, "data_wait_seconds": 2.6613473892211914e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 66260, "total_steps": 72479, "tokens": 34739322880, "loss": 4.616972982883453, "l_mix": 2.8651272654533386, "l_a": 2.9397140443325043, "l_c": 2.89977166056633, "z_loss": 41.600192070007324, "lr": 0.0014083518602124605, "wd": 0.01, "grad_norm": 0.17614518105983734, "tokens_per_second": 1445005.0341708206, "eta_seconds": 2256.4260987997054, "data_wait_seconds": 2.0205974578857422e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 66270, "total_steps": 72479, "tokens": 34744565760, "loss": 4.673996150493622, "l_mix": 2.9010449647903442, "l_a": 2.9730552434921265, "l_c": 2.9367816150188446, "z_loss": 41.66724443435669, "lr": 0.001405112071199474, "wd": 0.01, "grad_norm": 0.12721280753612518, "tokens_per_second": 1344018.332455768, "eta_seconds": 2422.068295788765, "data_wait_seconds": 2.771615982055664e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 66280, "total_steps": 72479, "tokens": 34749808640, "loss": 4.4866980612277985, "l_mix": 2.7847082912921906, "l_a": 2.8564301431179047, "l_c": 2.8168687522411346, "z_loss": 43.31364583969116, "lr": 0.0014018701512457914, "wd": 0.01, "grad_norm": 0.12271241098642349, "tokens_per_second": 1411795.04442196, "eta_seconds": 2302.0772914886475, "data_wait_seconds": 2.7358531951904297e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 66290, "total_steps": 72479, "tokens": 34755051520, "loss": 3.0466296672821045, "l_mix": 1.8749813884496689, "l_a": 1.9853285253047943, "l_c": 1.9201656430959702, "z_loss": 48.92776441574097, "lr": 0.001398626127431037, "wd": 0.01, "grad_norm": 0.12933440506458282, "tokens_per_second": 1427525.9772912166, "eta_seconds": 2273.036346530914, "data_wait_seconds": 3.4928321838378906e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 66300, "total_steps": 72479, "tokens": 34760294400, "loss": 3.003838986158371, "l_mix": 1.8481775373220444, "l_a": 1.9594131410121918, "l_c": 1.8927915394306183, "z_loss": 49.37972068786621, "lr": 0.0013953800268524082, "wd": 0.01, "grad_norm": 0.11343558877706528, "tokens_per_second": 1431963.4137230003, "eta_seconds": 2262.3312306404114, "data_wait_seconds": 1.7911195755004883e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16286719, "gate_a2b/L1_absmax": 0.67776382, "gate_a2b/L1_rms": 0.17244425, "gate_a2b/L3_absmean": 0.15315269, "gate_a2b/L3_absmax": 0.50188375, "gate_a2b/L3_rms": 0.15870737, "gate_a2b/L5_absmean": 0.22033767, "gate_a2b/L5_absmax": 0.73300588, "gate_a2b/L5_rms": 0.22632338, "gate_c2b/L1_absmean": 0.04222327, "gate_c2b/L1_absmax": 0.61493784, "gate_c2b/L1_rms": 0.07130792, "gate_c2b/L3_absmean": 0.16489851, "gate_c2b/L3_absmax": 0.38714254, "gate_c2b/L3_rms": 0.16834596, "gate_c2b/L5_absmean": 0.46946692, "gate_c2b/L5_absmax": 0.67927188, "gate_c2b/L5_rms": 0.47843713, "gate_b2a/L7_absmean": 0.05188937, "gate_b2a/L7_absmax": 0.33393624, "gate_b2a/L7_rms": 0.05757156, "gate_b2a/L11_absmean": 0.08458545, "gate_b2a/L11_absmax": 0.61478472, "gate_b2a/L11_rms": 0.09359272, "gate_b2a/L14_absmean": 0.21967989, "gate_b2a/L14_absmax": 0.80817646, "gate_b2a/L14_rms": 0.22864388, "gate_b2c/L3_absmean": 0.03702829, "gate_b2c/L3_absmax": 0.55127972, "gate_b2c/L3_rms": 0.04710643, "gate_b2c/L5_absmean": 0.03823482, "gate_b2c/L5_absmax": 0.53333855, "gate_b2c/L5_rms": 0.04723993, "gate_b2c/L6_absmean": 0.05537876, "gate_b2c/L6_absmax": 0.33653924, "gate_b2c/L6_rms": 0.06101193}, "streams": {"cos_a2b/round0": 0.353075, "cos_c2b/round0": 0.207848, "cos_b2a/round0": 0.154018, "cos_b2c/round0": 0.129499, "rms_a/round0": 7.749528, "rms_b/round0": 13.032555, "rms_c/round0": 5.071301, "cos_a2b/round1": 0.40818, "cos_c2b/round1": 0.306298, "cos_b2a/round1": 0.245419, "cos_b2c/round1": 0.117824, "rms_a/round1": 15.628716, "rms_b/round1": 31.78677, "rms_c/round1": 11.517559, "cos_a2b/round2": 0.518228, "cos_c2b/round2": 0.508743, "cos_b2a/round2": 0.091502, "cos_b2c/round2": 0.173607, "rms_a/round2": 44.487743, "rms_b/round2": 84.801399, "rms_c/round2": 22.873951}, "alpha_mean": 0.40791592360223916, "alpha_hist": [0.0463, 0.17841, 0.18637, 0.1337, 0.10695, 0.10057, 0.10078, 0.08984, 0.05182, 0.00526], "alpha_by_type": {"special": {"share": 0.0011, "mean": 0.65809, "std": 0.12468}, "whitespace": {"share": 0.03357, "mean": 0.63118, "std": 0.16758}, "digit": {"share": 0.00891, "mean": 0.37534, "std": 0.17935}, "word_start": {"share": 0.59146, "mean": 0.41498, "std": 0.21805}, "word_cont": {"share": 0.24704, "mean": 0.2552, "std": 0.14491}, "punct": {"share": 0.11792, "mean": 0.62902, "std": 0.19626}}, "alpha_p05": 0.10895, "alpha_p50": 0.35399, "alpha_p95": 0.82559}
{"step": 66310, "total_steps": 72479, "tokens": 34765537280, "loss": 4.429818272590637, "l_mix": 2.7471050024032593, "l_a": 2.827114224433899, "l_c": 2.781929612159729, "z_loss": 44.52990007400513, "lr": 0.0013921318766244519, "wd": 0.01, "grad_norm": 0.11242173612117767, "tokens_per_second": 1447091.0260984285, "eta_seconds": 2235.058205509186, "data_wait_seconds": 1.6748905181884766e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 66320, "total_steps": 72479, "tokens": 34770780160, "loss": 4.336328506469727, "l_mix": 2.687491685152054, "l_a": 2.771992266178131, "l_c": 2.7241307199001312, "z_loss": 45.3984899520874, "lr": 0.0013888817038788324, "wd": 0.01, "grad_norm": 0.13965649902820587, "tokens_per_second": 1429491.4259711988, "eta_seconds": 2258.9081216812133, "data_wait_seconds": 1.6033649444580078e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 66330, "total_steps": 72479, "tokens": 34776023040, "loss": 4.3050937950611115, "l_mix": 2.6691696643829346, "l_a": 2.7502410113811493, "l_c": 2.7028392553329468, "z_loss": 43.547929763793945, "lr": 0.0013856295357641114, "wd": 0.01, "grad_norm": 0.13415583968162537, "tokens_per_second": 1417499.8696622977, "eta_seconds": 2274.319018292427, "data_wait_seconds": 1.6629695892333984e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 66340, "total_steps": 72479, "tokens": 34781265920, "loss": 3.0315018594264984, "l_mix": 1.8658469915390015, "l_a": 1.9757945239543915, "l_c": 1.9097216129302979, "z_loss": 48.45779323577881, "lr": 0.0013823753994455148, "wd": 0.01, "grad_norm": 0.1270413100719452, "tokens_per_second": 1452305.4497941863, "eta_seconds": 2216.203232216835, "data_wait_seconds": 2.816319465637207e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 66350, "total_steps": 72479, "tokens": 34786508800, "loss": 3.0694854259490967, "l_mix": 1.8891898840665817, "l_a": 2.000575542449951, "l_c": 1.933742731809616, "z_loss": 47.939818382263184, "lr": 0.001379119322104711, "wd": 0.01, "grad_norm": 0.13039007782936096, "tokens_per_second": 1444677.235683032, "eta_seconds": 2224.2761723041535, "data_wait_seconds": 1.952052116394043e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 66360, "total_steps": 72479, "tokens": 34791751680, "loss": 2.9821219444274902, "l_mix": 1.834041103720665, "l_a": 1.9460051655769348, "l_c": 1.8809309303760529, "z_loss": 50.26294946670532, "lr": 0.0013758613309395808, "wd": 0.01, "grad_norm": 0.15605297684669495, "tokens_per_second": 1456097.0559220775, "eta_seconds": 2203.231068253517, "data_wait_seconds": 1.9758939743041992e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 66370, "total_steps": 72479, "tokens": 34796994560, "loss": 3.0064487159252167, "l_mix": 1.850592166185379, "l_a": 1.9606004655361176, "l_c": 1.892254650592804, "z_loss": 48.157026290893555, "lr": 0.0013726014531639907, "wd": 0.01, "grad_norm": 0.1838514506816864, "tokens_per_second": 1411046.2226339174, "eta_seconds": 2269.858591890335, "data_wait_seconds": 2.4557113647460938e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 66380, "total_steps": 72479, "tokens": 34802237440, "loss": 3.0435523688793182, "l_mix": 1.8725573867559433, "l_a": 1.98458693921566, "l_c": 1.918729454278946, "z_loss": 49.58571767807007, "lr": 0.0013693397160075668, "wd": 0.01, "grad_norm": 0.11448927223682404, "tokens_per_second": 1470058.655022196, "eta_seconds": 2175.1734198331833, "data_wait_seconds": 3.343820571899414e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 66390, "total_steps": 72479, "tokens": 34807480320, "loss": 3.0915678441524506, "l_mix": 1.9024966210126877, "l_a": 2.0161159336566925, "l_c": 1.94745472073555, "z_loss": 48.89749240875244, "lr": 0.0013660761467154657, "wd": 0.01, "grad_norm": 0.13816533982753754, "tokens_per_second": 1368934.1609717575, "eta_seconds": 2332.0256905078886, "data_wait_seconds": 2.288818359375e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 66400, "total_steps": 72479, "tokens": 34812723200, "loss": 3.0463914573192596, "l_mix": 1.874629631638527, "l_a": 1.9853217005729675, "l_c": 1.9205508381128311, "z_loss": 49.13378286361694, "lr": 0.0013628107725481488, "wd": 0.01, "grad_norm": 0.11275386065244675, "tokens_per_second": 1247702.4307993478, "eta_seconds": 2554.4125532865523, "data_wait_seconds": 2.9206275939941406e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.16289864, "gate_a2b/L1_absmax": 0.67696631, "gate_a2b/L1_rms": 0.17250463, "gate_a2b/L3_absmean": 0.1530669, "gate_a2b/L3_absmax": 0.50147891, "gate_a2b/L3_rms": 0.15863524, "gate_a2b/L5_absmean": 0.22088301, "gate_a2b/L5_absmax": 0.73229086, "gate_a2b/L5_rms": 0.22685313, "gate_c2b/L1_absmean": 0.04261196, "gate_c2b/L1_absmax": 0.6141082, "gate_c2b/L1_rms": 0.07129271, "gate_c2b/L3_absmean": 0.16489111, "gate_c2b/L3_absmax": 0.38735375, "gate_c2b/L3_rms": 0.16841291, "gate_c2b/L5_absmean": 0.46931157, "gate_c2b/L5_absmax": 0.67771298, "gate_c2b/L5_rms": 0.47828957, "gate_b2a/L7_absmean": 0.05198392, "gate_b2a/L7_absmax": 0.33366701, "gate_b2a/L7_rms": 0.05763756, "gate_b2a/L11_absmean": 0.08464496, "gate_b2a/L11_absmax": 0.61532569, "gate_b2a/L11_rms": 0.09366892, "gate_b2a/L14_absmean": 0.21969028, "gate_b2a/L14_absmax": 0.80545604, "gate_b2a/L14_rms": 0.22859417, "gate_b2c/L3_absmean": 0.03709411, "gate_b2c/L3_absmax": 0.55056125, "gate_b2c/L3_rms": 0.04713104, "gate_b2c/L5_absmean": 0.03830771, "gate_b2c/L5_absmax": 0.53270674, "gate_b2c/L5_rms": 0.04728101, "gate_b2c/L6_absmean": 0.05547566, "gate_b2c/L6_absmax": 0.33610436, "gate_b2c/L6_rms": 0.061077}, "streams": {"cos_a2b/round0": 0.351886, "cos_c2b/round0": 0.206082, "cos_b2a/round0": 0.157159, "cos_b2c/round0": 0.128458, "rms_a/round0": 7.666208, "rms_b/round0": 13.054587, "rms_c/round0": 5.060266, "cos_a2b/round1": 0.409622, "cos_c2b/round1": 0.30797, "cos_b2a/round1": 0.249713, "cos_b2c/round1": 0.118732, "rms_a/round1": 15.518531, "rms_b/round1": 31.764633, "rms_c/round1": 11.610147, "cos_a2b/round2": 0.51637, "cos_c2b/round2": 0.509311, "cos_b2a/round2": 0.090537, "cos_b2c/round2": 0.174524, "rms_a/round2": 44.2785, "rms_b/round2": 84.536903, "rms_c/round2": 22.966816}, "alpha_mean": 0.39955906373175054, "alpha_hist": [0.05031, 0.188, 0.18945, 0.13541, 0.10379, 0.09615, 0.09395, 0.08669, 0.05034, 0.00592], "alpha_by_type": {"special": {"share": 0.00111, "mean": 0.66479, "std": 0.11711}, "whitespace": {"share": 0.02968, "mean": 0.63733, "std": 0.18291}, "digit": {"share": 0.01074, "mean": 0.41094, "std": 0.18264}, "word_start": {"share": 0.59007, "mean": 0.40826, "std": 0.21968}, "word_cont": {"share": 0.26334, "mean": 0.25694, "std": 0.14375}, "punct": {"share": 0.10506, "mean": 0.63702, "std": 0.19898}}, "alpha_p05": 0.10088, "alpha_p50": 0.36886, "alpha_p95": 0.82727}
{"step": 66410, "total_steps": 72479, "tokens": 34817966080, "loss": 3.047032445669174, "l_mix": 1.875736027956009, "l_a": 1.983600839972496, "l_c": 1.9207203835248947, "z_loss": 48.53079843521118, "lr": 0.001359543620781153, "wd": 0.01, "grad_norm": 0.1671334207057953, "tokens_per_second": 1479840.9672660485, "eta_seconds": 2150.1660937786105, "data_wait_seconds": 2.3543834686279297e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 66420, "total_steps": 72479, "tokens": 34823208960, "loss": 4.337808221578598, "l_mix": 2.688795268535614, "l_a": 2.7724044024944305, "l_c": 2.724305272102356, "z_loss": 45.525174140930176, "lr": 0.001356274718704863, "wd": 0.01, "grad_norm": 0.11361098289489746, "tokens_per_second": 1467922.628780645, "eta_seconds": 2164.052062225342, "data_wait_seconds": 2.9593706130981445e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 66430, "total_steps": 72479, "tokens": 34828451840, "loss": 4.344410598278046, "l_mix": 2.6929794251918793, "l_a": 2.776259869337082, "l_c": 2.7285107374191284, "z_loss": 46.149447441101074, "lr": 0.0013530040936242842, "wd": 0.01, "grad_norm": 0.13489386439323425, "tokens_per_second": 1439590.665366008, "eta_seconds": 2202.9999140024183, "data_wait_seconds": 2.0444393157958984e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 66440, "total_steps": 72479, "tokens": 34833694720, "loss": 4.343355774879456, "l_mix": 2.692276567220688, "l_a": 2.776527374982834, "l_c": 2.727069675922394, "z_loss": 45.3803915977478, "lr": 0.001349731772858814, "wd": 0.01, "grad_norm": 0.11995743215084076, "tokens_per_second": 1446239.81332589, "eta_seconds": 2189.2463496208193, "data_wait_seconds": 1.6361474990844727e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 66450, "total_steps": 72479, "tokens": 34838937600, "loss": 3.0253439843654633, "l_mix": 1.8612103313207626, "l_a": 1.9736958593130112, "l_c": 1.9067494422197342, "z_loss": 47.63132572174072, "lr": 0.0013464577837420141, "wd": 0.01, "grad_norm": 0.28451547026634216, "tokens_per_second": 1474260.0573621215, "eta_seconds": 2144.0805753469467, "data_wait_seconds": 1.5914440155029297e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 66460, "total_steps": 72479, "tokens": 34844180480, "loss": 3.0190184712409973, "l_mix": 1.8573839217424393, "l_a": 1.9700099229812622, "l_c": 1.9021051228046417, "z_loss": 48.426610469818115, "lr": 0.0013431821536213817, "wd": 0.01, "grad_norm": 0.15815973281860352, "tokens_per_second": 1323502.1986828432, "eta_seconds": 2384.3477367401124, "data_wait_seconds": 3.698468208312988e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 66470, "total_steps": 72479, "tokens": 34849423360, "loss": 3.0487585365772247, "l_mix": 1.875998169183731, "l_a": 1.988956794142723, "l_c": 1.9202442914247513, "z_loss": 48.66652774810791, "lr": 0.0013399049098581212, "wd": 0.01, "grad_norm": 0.15458159148693085, "tokens_per_second": 1383150.5834091364, "eta_seconds": 2277.7321788311006, "data_wait_seconds": 2.390146255493164e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 66480, "total_steps": 72479, "tokens": 34854666240, "loss": 2.9985422492027283, "l_mix": 1.8448552042245865, "l_a": 1.9574895352125168, "l_c": 1.8881337493658066, "z_loss": 49.232502460479736, "lr": 0.0013366260798269162, "wd": 0.01, "grad_norm": 0.12868472933769226, "tokens_per_second": 1442715.7406705532, "eta_seconds": 2180.057805800438, "data_wait_seconds": 2.6702880859375e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 66490, "total_steps": 72479, "tokens": 34859909120, "loss": 2.974952816963196, "l_mix": 1.8300649374723434, "l_a": 1.9419335573911667, "l_c": 1.8743590712547302, "z_loss": 50.87270212173462, "lr": 0.0013333456909156989, "wd": 0.01, "grad_norm": 0.14404840767383575, "tokens_per_second": 1429650.7174326214, "eta_seconds": 2196.3132628917697, "data_wait_seconds": 2.0116567611694336e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 66500, "total_steps": 72479, "tokens": 34865152000, "loss": 3.030514717102051, "l_mix": 1.8642113506793976, "l_a": 1.9777006804943085, "l_c": 1.9099770933389664, "z_loss": 49.601627349853516, "lr": 0.0013300637705254243, "wd": 0.01, "grad_norm": 0.1442946344614029, "tokens_per_second": 1237488.0145584226, "eta_seconds": 2533.1299496412275, "data_wait_seconds": 1.817941665649414e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.16332097, "gate_a2b/L1_absmax": 0.67665195, "gate_a2b/L1_rms": 0.17283829, "gate_a2b/L3_absmean": 0.15367675, "gate_a2b/L3_absmax": 0.50098854, "gate_a2b/L3_rms": 0.15920256, "gate_a2b/L5_absmean": 0.22108932, "gate_a2b/L5_absmax": 0.73225313, "gate_a2b/L5_rms": 0.22703668, "gate_c2b/L1_absmean": 0.04303145, "gate_c2b/L1_absmax": 0.61394036, "gate_c2b/L1_rms": 0.0715737, "gate_c2b/L3_absmean": 0.16491468, "gate_c2b/L3_absmax": 0.38736939, "gate_c2b/L3_rms": 0.16841349, "gate_c2b/L5_absmean": 0.46949822, "gate_c2b/L5_absmax": 0.67731708, "gate_c2b/L5_rms": 0.47841048, "gate_b2a/L7_absmean": 0.05191119, "gate_b2a/L7_absmax": 0.33387685, "gate_b2a/L7_rms": 0.05758991, "gate_b2a/L11_absmean": 0.08466064, "gate_b2a/L11_absmax": 0.61579776, "gate_b2a/L11_rms": 0.09367653, "gate_b2a/L14_absmean": 0.21986723, "gate_b2a/L14_absmax": 0.80432189, "gate_b2a/L14_rms": 0.22873746, "gate_b2c/L3_absmean": 0.03730956, "gate_b2c/L3_absmax": 0.55104131, "gate_b2c/L3_rms": 0.04730338, "gate_b2c/L5_absmean": 0.03837942, "gate_b2c/L5_absmax": 0.53393686, "gate_b2c/L5_rms": 0.0473794, "gate_b2c/L6_absmean": 0.05559778, "gate_b2c/L6_absmax": 0.33624932, "gate_b2c/L6_rms": 0.06115913}, "streams": {"cos_a2b/round0": 0.3535, "cos_c2b/round0": 0.206736, "cos_b2a/round0": 0.156316, "cos_b2c/round0": 0.12815, "rms_a/round0": 7.700725, "rms_b/round0": 13.012776, "rms_c/round0": 5.086937, "cos_a2b/round1": 0.407356, "cos_c2b/round1": 0.307257, "cos_b2a/round1": 0.246696, "cos_b2c/round1": 0.114342, "rms_a/round1": 15.560175, "rms_b/round1": 31.638735, "rms_c/round1": 11.636848, "cos_a2b/round2": 0.516339, "cos_c2b/round2": 0.510188, "cos_b2a/round2": 0.08889, "cos_b2c/round2": 0.173407, "rms_a/round2": 44.511005, "rms_b/round2": 84.545677, "rms_c/round2": 22.917171}, "alpha_mean": 0.4012557290171941, "alpha_hist": [0.04599, 0.17572, 0.18669, 0.14209, 0.11488, 0.10889, 0.09729, 0.08179, 0.04333, 0.00333], "alpha_by_type": {"special": {"share": 0.00113, "mean": 0.63246, "std": 0.10981}, "whitespace": {"share": 0.0345, "mean": 0.61605, "std": 0.18009}, "digit": {"share": 0.01076, "mean": 0.38473, "std": 0.18609}, "word_start": {"share": 0.56519, "mean": 0.40542, "std": 0.21098}, "word_cont": {"share": 0.26726, "mean": 0.26569, "std": 0.1444}, "punct": {"share": 0.12117, "mean": 0.61902, "std": 0.18924}}, "alpha_p05": 0.11437, "alpha_p50": 0.38676, "alpha_p95": 0.80225}
{"step": 66510, "total_steps": 72479, "tokens": 34870394880, "loss": 4.34275758266449, "l_mix": 2.6924389004707336, "l_a": 2.7715169489383698, "l_c": 2.729545146226883, "z_loss": 46.21508550643921, "lr": 0.0013267803460698393, "wd": 0.01, "grad_norm": 0.11599169671535492, "tokens_per_second": 1452945.2921680866, "eta_seconds": 2153.883624434471, "data_wait_seconds": 2.1249055862426758e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 66520, "total_steps": 72479, "tokens": 34875637760, "loss": 4.423719763755798, "l_mix": 2.7425725162029266, "l_a": 2.826383113861084, "l_c": 2.7774409353733063, "z_loss": 45.82369422912598, "lr": 0.0013234954449752535, "wd": 0.01, "grad_norm": 0.16621263325214386, "tokens_per_second": 1417936.5827006474, "eta_seconds": 2203.365249276161, "data_wait_seconds": 1.659989356994629e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 66530, "total_steps": 72479, "tokens": 34880880640, "loss": 4.329730272293091, "l_mix": 2.6839829087257385, "l_a": 2.765504390001297, "l_c": 2.7203201353549957, "z_loss": 45.17811870574951, "lr": 0.0013202090946803121, "wd": 0.01, "grad_norm": 0.1503012776374817, "tokens_per_second": 1407746.88911521, "eta_seconds": 2215.5895609617237, "data_wait_seconds": 1.6957521438598633e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 66540, "total_steps": 72479, "tokens": 34886123520, "loss": 4.46765524148941, "l_mix": 2.770432859659195, "l_a": 2.8500942289829254, "l_c": 2.8073137402534485, "z_loss": 44.86052703857422, "lr": 0.001316921322635764, "wd": 0.01, "grad_norm": 0.10967179387807846, "tokens_per_second": 1455197.8538950484, "eta_seconds": 2139.740945649147, "data_wait_seconds": 2.0831823348999023e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 66550, "total_steps": 72479, "tokens": 34891366400, "loss": 4.444571375846863, "l_mix": 2.755982220172882, "l_a": 2.836905062198639, "l_c": 2.7917251884937286, "z_loss": 44.77025604248047, "lr": 0.001313632156304235, "wd": 0.01, "grad_norm": 0.09981945157051086, "tokens_per_second": 1470611.4586318496, "eta_seconds": 2113.749035310745, "data_wait_seconds": 2.199411392211914e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 66560, "total_steps": 72479, "tokens": 34896609280, "loss": 3.023982733488083, "l_mix": 1.8612454682588577, "l_a": 1.9712495058774948, "l_c": 1.9045412242412567, "z_loss": 49.56203317642212, "lr": 0.0013103416231599972, "wd": 0.01, "grad_norm": 0.1552460789680481, "tokens_per_second": 1454805.1630628149, "eta_seconds": 2133.110845899582, "data_wait_seconds": 1.671910285949707e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 66570, "total_steps": 72479, "tokens": 34901852160, "loss": 3.057503879070282, "l_mix": 1.8806950598955154, "l_a": 1.9957069158554077, "l_c": 1.926989197731018, "z_loss": 49.778624057769775, "lr": 0.0013070497506887386, "wd": 0.01, "grad_norm": 0.17890574038028717, "tokens_per_second": 1475893.1646882033, "eta_seconds": 2099.0799782276154, "data_wait_seconds": 2.1457672119140625e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 66580, "total_steps": 72479, "tokens": 34907095040, "loss": 3.029638558626175, "l_mix": 1.8636902868747711, "l_a": 1.9759317934513092, "l_c": 1.9105622619390488, "z_loss": 50.69302034378052, "lr": 0.0013037565663873362, "wd": 0.01, "grad_norm": 0.15346617996692657, "tokens_per_second": 1372828.7142736656, "eta_seconds": 2252.8483559846877, "data_wait_seconds": 1.77919864654541e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 66590, "total_steps": 72479, "tokens": 34912337920, "loss": 3.0255796909332275, "l_mix": 1.8623791635036469, "l_a": 1.9724611341953278, "l_c": 1.904873862862587, "z_loss": 49.53350830078125, "lr": 0.0013004620977636232, "wd": 0.01, "grad_norm": 0.16340278089046478, "tokens_per_second": 1391388.2994985108, "eta_seconds": 2219.0297511577605, "data_wait_seconds": 2.8818845748901367e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 66600, "total_steps": 72479, "tokens": 34917580800, "loss": 3.011603683233261, "l_mix": 1.8534461706876755, "l_a": 1.9639655202627182, "l_c": 1.8965590447187424, "z_loss": 49.58134174346924, "lr": 0.0012971663723361619, "wd": 0.01, "grad_norm": 0.125202476978302, "tokens_per_second": 1344465.2670957742, "eta_seconds": 2292.5762587070467, "data_wait_seconds": 2.9742717742919922e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16365023, "gate_a2b/L1_absmax": 0.67523521, "gate_a2b/L1_rms": 0.17315605, "gate_a2b/L3_absmean": 0.15388489, "gate_a2b/L3_absmax": 0.50074774, "gate_a2b/L3_rms": 0.15939218, "gate_a2b/L5_absmean": 0.22129984, "gate_a2b/L5_absmax": 0.73181641, "gate_a2b/L5_rms": 0.22721362, "gate_c2b/L1_absmean": 0.04276285, "gate_c2b/L1_absmax": 0.61221254, "gate_c2b/L1_rms": 0.07109871, "gate_c2b/L3_absmean": 0.16466439, "gate_c2b/L3_absmax": 0.38579968, "gate_c2b/L3_rms": 0.16807675, "gate_c2b/L5_absmean": 0.46945596, "gate_c2b/L5_absmax": 0.67769617, "gate_c2b/L5_rms": 0.47844872, "gate_b2a/L7_absmean": 0.05174107, "gate_b2a/L7_absmax": 0.33297753, "gate_b2a/L7_rms": 0.05740985, "gate_b2a/L11_absmean": 0.08489205, "gate_b2a/L11_absmax": 0.61461937, "gate_b2a/L11_rms": 0.09385795, "gate_b2a/L14_absmean": 0.21998179, "gate_b2a/L14_absmax": 0.80383092, "gate_b2a/L14_rms": 0.2288361, "gate_b2c/L3_absmean": 0.03739773, "gate_b2c/L3_absmax": 0.55047822, "gate_b2c/L3_rms": 0.047342, "gate_b2c/L5_absmean": 0.03860038, "gate_b2c/L5_absmax": 0.53312367, "gate_b2c/L5_rms": 0.04751008, "gate_b2c/L6_absmean": 0.05576469, "gate_b2c/L6_absmax": 0.33611268, "gate_b2c/L6_rms": 0.06133851}, "streams": {"cos_a2b/round0": 0.355043, "cos_c2b/round0": 0.207346, "cos_b2a/round0": 0.152255, "cos_b2c/round0": 0.128406, "rms_a/round0": 7.703313, "rms_b/round0": 12.982178, "rms_c/round0": 5.053004, "cos_a2b/round1": 0.405708, "cos_c2b/round1": 0.303604, "cos_b2a/round1": 0.242884, "cos_b2c/round1": 0.113533, "rms_a/round1": 15.542716, "rms_b/round1": 31.51474, "rms_c/round1": 11.536329, "cos_a2b/round2": 0.512951, "cos_c2b/round2": 0.513966, "cos_b2a/round2": 0.087617, "cos_b2c/round2": 0.173501, "rms_a/round2": 44.260094, "rms_b/round2": 84.593742, "rms_c/round2": 22.770472}, "alpha_mean": 0.39435109529315326, "alpha_hist": [0.04413, 0.18375, 0.19508, 0.14168, 0.11168, 0.10811, 0.09605, 0.07849, 0.03743, 0.0036], "alpha_by_type": {"special": {"share": 0.00104, "mean": 0.61693, "std": 0.14916}, "whitespace": {"share": 0.0457, "mean": 0.5917, "std": 0.17197}, "digit": {"share": 0.01176, "mean": 0.33851, "std": 0.20262}, "word_start": {"share": 0.54688, "mean": 0.39328, "std": 0.20923}, "word_cont": {"share": 0.26303, "mean": 0.25979, "std": 0.13847}, "punct": {"share": 0.13159, "mean": 0.60249, "std": 0.18671}}, "alpha_p05": 0.10781, "alpha_p50": 0.35533, "alpha_p95": 0.80131}
{"step": 66610, "total_steps": 72479, "tokens": 34922823680, "loss": 3.0430499613285065, "l_mix": 1.8723461329936981, "l_a": 1.985139936208725, "l_c": 1.9172061085700989, "z_loss": 50.12189435958862, "lr": 0.001293869417634011, "wd": 0.01, "grad_norm": 0.13444840908050537, "tokens_per_second": 1210046.5351528558, "eta_seconds": 2542.915650439262, "data_wait_seconds": 2.6524066925048828e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 66620, "total_steps": 72479, "tokens": 34928066560, "loss": 4.3360098004341125, "l_mix": 2.6883994936943054, "l_a": 2.769913285970688, "l_c": 2.7221208214759827, "z_loss": 45.45896291732788, "lr": 0.0012905712611964992, "wd": 0.01, "grad_norm": 0.10447598248720169, "tokens_per_second": 1364657.818650747, "eta_seconds": 2250.969693660736, "data_wait_seconds": 4.13060188293457e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 66630, "total_steps": 72479, "tokens": 34933309440, "loss": 4.406885385513306, "l_mix": 2.73228919506073, "l_a": 2.814313590526581, "l_c": 2.7676733434200287, "z_loss": 44.430527210235596, "lr": 0.001287271930572992, "wd": 0.01, "grad_norm": 0.1267174333333969, "tokens_per_second": 1396648.3560225426, "eta_seconds": 2195.6568371534345, "data_wait_seconds": 2.822279930114746e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 66640, "total_steps": 72479, "tokens": 34938552320, "loss": 4.335951507091522, "l_mix": 2.687956392765045, "l_a": 2.7705421149730682, "l_c": 2.722774714231491, "z_loss": 46.38150501251221, "lr": 0.0012839714533226627, "wd": 0.01, "grad_norm": 0.10255418717861176, "tokens_per_second": 1453944.757395783, "eta_seconds": 2105.5254103899, "data_wait_seconds": 1.7374753952026367e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 66650, "total_steps": 72479, "tokens": 34943795200, "loss": 4.315269887447357, "l_mix": 2.67547070980072, "l_a": 2.755092442035675, "l_c": 2.7109043896198273, "z_loss": 45.96071100234985, "lr": 0.0012806698570142635, "wd": 0.01, "grad_norm": 0.10950363427400589, "tokens_per_second": 1420976.5162379583, "eta_seconds": 2150.686318230629, "data_wait_seconds": 2.22623348236084e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 66660, "total_steps": 72479, "tokens": 34949038080, "loss": 4.491070032119751, "l_mix": 2.7848813831806183, "l_a": 2.866608113050461, "l_c": 2.820687234401703, "z_loss": 44.307634353637695, "lr": 0.001277367169225893, "wd": 0.01, "grad_norm": 0.10969984531402588, "tokens_per_second": 1427691.2266685485, "eta_seconds": 2136.8989421606066, "data_wait_seconds": 1.811981201171875e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 66670, "total_steps": 72479, "tokens": 34954280960, "loss": 4.620532333850861, "l_mix": 2.866812616586685, "l_a": 2.9422413110733032, "l_c": 2.9034906923770905, "z_loss": 43.48981189727783, "lr": 0.001274063417544767, "wd": 0.01, "grad_norm": 0.12489026039838791, "tokens_per_second": 1436036.424363279, "eta_seconds": 2120.82990398407, "data_wait_seconds": 1.7374753952026367e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 66680, "total_steps": 72479, "tokens": 34959523840, "loss": 3.037910431623459, "l_mix": 1.8695061206817627, "l_a": 1.9790005087852478, "l_c": 1.9156804978847504, "z_loss": 49.02656269073486, "lr": 0.0012707586295669887, "wd": 0.01, "grad_norm": 0.13309992849826813, "tokens_per_second": 1440686.8836457497, "eta_seconds": 2110.3448268413545, "data_wait_seconds": 1.8984079360961914e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 66690, "total_steps": 72479, "tokens": 34964766720, "loss": 3.016268312931061, "l_mix": 1.8560222536325455, "l_a": 1.9664008766412735, "l_c": 1.9010858237743378, "z_loss": 48.61511182785034, "lr": 0.0012674528328973165, "wd": 0.01, "grad_norm": 0.14566558599472046, "tokens_per_second": 1406926.8280079253, "eta_seconds": 2157.257343864441, "data_wait_seconds": 2.4527311325073242e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 66700, "total_steps": 72479, "tokens": 34970009600, "loss": 3.021966576576233, "l_mix": 1.8593086153268814, "l_a": 1.9709298610687256, "l_c": 1.9045963287353516, "z_loss": 49.23753881454468, "lr": 0.0012641460551489352, "wd": 0.01, "grad_norm": 0.11256029456853867, "tokens_per_second": 1462974.3701123684, "eta_seconds": 2071.027636504173, "data_wait_seconds": 1.7076730728149414e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16370703, "gate_a2b/L1_absmax": 0.67293531, "gate_a2b/L1_rms": 0.17313613, "gate_a2b/L3_absmean": 0.15368536, "gate_a2b/L3_absmax": 0.50278312, "gate_a2b/L3_rms": 0.15925638, "gate_a2b/L5_absmean": 0.22107406, "gate_a2b/L5_absmax": 0.73081034, "gate_a2b/L5_rms": 0.22702742, "gate_c2b/L1_absmean": 0.04269166, "gate_c2b/L1_absmax": 0.61055666, "gate_c2b/L1_rms": 0.070858, "gate_c2b/L3_absmean": 0.16462569, "gate_c2b/L3_absmax": 0.38634849, "gate_c2b/L3_rms": 0.16803478, "gate_c2b/L5_absmean": 0.4692671, "gate_c2b/L5_absmax": 0.67579967, "gate_c2b/L5_rms": 0.47816402, "gate_b2a/L7_absmean": 0.05189455, "gate_b2a/L7_absmax": 0.33258289, "gate_b2a/L7_rms": 0.05759705, "gate_b2a/L11_absmean": 0.08484012, "gate_b2a/L11_absmax": 0.61244684, "gate_b2a/L11_rms": 0.09378164, "gate_b2a/L14_absmean": 0.21985576, "gate_b2a/L14_absmax": 0.80388087, "gate_b2a/L14_rms": 0.22879535, "gate_b2c/L3_absmean": 0.03747568, "gate_b2c/L3_absmax": 0.54763967, "gate_b2c/L3_rms": 0.04734847, "gate_b2c/L5_absmean": 0.03846566, "gate_b2c/L5_absmax": 0.53072375, "gate_b2c/L5_rms": 0.04738292, "gate_b2c/L6_absmean": 0.05590295, "gate_b2c/L6_absmax": 0.33615708, "gate_b2c/L6_rms": 0.06153611}, "streams": {"cos_a2b/round0": 0.354414, "cos_c2b/round0": 0.21005, "cos_b2a/round0": 0.152993, "cos_b2c/round0": 0.129465, "rms_a/round0": 7.657213, "rms_b/round0": 12.787849, "rms_c/round0": 5.012648, "cos_a2b/round1": 0.406329, "cos_c2b/round1": 0.301607, "cos_b2a/round1": 0.242405, "cos_b2c/round1": 0.113329, "rms_a/round1": 15.402523, "rms_b/round1": 31.331661, "rms_c/round1": 11.392309, "cos_a2b/round2": 0.516904, "cos_c2b/round2": 0.512083, "cos_b2a/round2": 0.086927, "cos_b2c/round2": 0.17049, "rms_a/round2": 44.011482, "rms_b/round2": 84.593124, "rms_c/round2": 22.5116}, "alpha_mean": 0.40341034233392037, "alpha_hist": [0.04848, 0.17331, 0.18785, 0.13901, 0.11325, 0.10559, 0.097, 0.08379, 0.04813, 0.0036], "alpha_by_type": {"special": {"share": 0.00104, "mean": 0.62588, "std": 0.12729}, "whitespace": {"share": 0.04045, "mean": 0.60884, "std": 0.17491}, "digit": {"share": 0.01541, "mean": 0.39294, "std": 0.17013}, "word_start": {"share": 0.54553, "mean": 0.40743, "std": 0.21594}, "word_cont": {"share": 0.26332, "mean": 0.25906, "std": 0.14262}, "punct": {"share": 0.13425, "mean": 0.60781, "std": 0.19315}}, "alpha_p05": 0.10339, "alpha_p50": 0.37938, "alpha_p95": 0.801}
{"step": 66710, "total_steps": 72479, "tokens": 34975252480, "loss": 3.0306376516819, "l_mix": 1.8649763315916061, "l_a": 1.9759569019079208, "l_c": 1.9095807820558548, "z_loss": 49.699941635131836, "lr": 0.0012608383239432234, "wd": 0.01, "grad_norm": 0.10853341966867447, "tokens_per_second": 1446585.4501303327, "eta_seconds": 2090.866786837578, "data_wait_seconds": 3.30805778503418e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 66720, "total_steps": 72479, "tokens": 34980495360, "loss": 3.0258898735046387, "l_mix": 1.8619864732027054, "l_a": 1.9721461832523346, "l_c": 1.9075317084789276, "z_loss": 49.083720684051514, "lr": 0.001257529666909525, "wd": 0.01, "grad_norm": 0.13843730092048645, "tokens_per_second": 1448237.9007239796, "eta_seconds": 2084.860913038254, "data_wait_seconds": 3.725290298461914e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 66730, "total_steps": 72479, "tokens": 34985738240, "loss": 4.32006961107254, "l_mix": 2.6776695251464844, "l_a": 2.7621864080429077, "l_c": 2.7124800980091095, "z_loss": 46.48046541213989, "lr": 0.001254220111684917, "wd": 0.01, "grad_norm": 0.13931554555892944, "tokens_per_second": 1338767.3617399433, "eta_seconds": 2251.4230613470077, "data_wait_seconds": 1.5944242477416992e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 66740, "total_steps": 72479, "tokens": 34990981120, "loss": 4.322840690612793, "l_mix": 2.680320769548416, "l_a": 2.759417086839676, "l_c": 2.7156489193439484, "z_loss": 45.049785137176514, "lr": 0.001250909685913979, "wd": 0.01, "grad_norm": 0.16955919563770294, "tokens_per_second": 1393920.892911306, "eta_seconds": 2158.5793335199355, "data_wait_seconds": 1.767277717590332e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 66750, "total_steps": 72479, "tokens": 34996224000, "loss": 4.421514630317688, "l_mix": 2.741728335618973, "l_a": 2.8223359286785126, "l_c": 2.7769514620304108, "z_loss": 44.78929376602173, "lr": 0.0012475984172485614, "wd": 0.01, "grad_norm": 0.16012102365493774, "tokens_per_second": 1416786.608326687, "eta_seconds": 2120.0411793470385, "data_wait_seconds": 2.0176172256469727e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 66760, "total_steps": 72479, "tokens": 35001466880, "loss": 4.466767191886902, "l_mix": 2.7705888748168945, "l_a": 2.8471811413764954, "l_c": 2.806746244430542, "z_loss": 45.26118803024292, "lr": 0.0012442863333475562, "wd": 0.01, "grad_norm": 0.19058528542518616, "tokens_per_second": 1365683.3036591727, "eta_seconds": 2195.533227920532, "data_wait_seconds": 1.6123056411743164e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 66770, "total_steps": 72479, "tokens": 35006709760, "loss": 4.674875617027283, "l_mix": 2.899096757173538, "l_a": 2.983369827270508, "l_c": 2.935892730951309, "z_loss": 44.53358173370361, "lr": 0.001240973461876665, "wd": 0.01, "grad_norm": 0.1456141620874405, "tokens_per_second": 1457518.7523824447, "eta_seconds": 2053.599781894684, "data_wait_seconds": 1.6003847122192383e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 66780, "total_steps": 72479, "tokens": 35011952640, "loss": 3.068567544221878, "l_mix": 1.8885009437799454, "l_a": 1.9994468241930008, "l_c": 1.9341081827878952, "z_loss": 47.53868532180786, "lr": 0.0012376598305081673, "wd": 0.01, "grad_norm": 0.20091032981872559, "tokens_per_second": 1454859.4474768215, "eta_seconds": 2053.7498087406157, "data_wait_seconds": 3.319978713989258e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 66790, "total_steps": 72479, "tokens": 35017195520, "loss": 2.9907467365264893, "l_mix": 1.8393625319004059, "l_a": 1.9524851143360138, "l_c": 1.885462149977684, "z_loss": 49.30300998687744, "lr": 0.0012343454669206909, "wd": 0.01, "grad_norm": 0.21575039625167847, "tokens_per_second": 1429967.9190850903, "eta_seconds": 2085.8331100940704, "data_wait_seconds": 2.0444393157958984e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 66800, "total_steps": 72479, "tokens": 35022438400, "loss": 4.524092733860016, "l_mix": 2.807031065225601, "l_a": 2.8838823437690735, "l_c": 2.839656412601471, "z_loss": 44.18629598617554, "lr": 0.001231030398798979, "wd": 0.01, "grad_norm": 0.10308697074651718, "tokens_per_second": 1404158.3051075384, "eta_seconds": 2120.438657927513, "data_wait_seconds": 2.4378299713134766e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16395055, "gate_a2b/L1_absmax": 0.67341208, "gate_a2b/L1_rms": 0.17342342, "gate_a2b/L3_absmean": 0.15417603, "gate_a2b/L3_absmax": 0.50254732, "gate_a2b/L3_rms": 0.15966588, "gate_a2b/L5_absmean": 0.22142427, "gate_a2b/L5_absmax": 0.73247004, "gate_a2b/L5_rms": 0.22737721, "gate_c2b/L1_absmean": 0.04297893, "gate_c2b/L1_absmax": 0.61010653, "gate_c2b/L1_rms": 0.07101298, "gate_c2b/L3_absmean": 0.16449524, "gate_c2b/L3_absmax": 0.38796303, "gate_c2b/L3_rms": 0.16800271, "gate_c2b/L5_absmean": 0.46915004, "gate_c2b/L5_absmax": 0.67737395, "gate_c2b/L5_rms": 0.4780612, "gate_b2a/L7_absmean": 0.05180909, "gate_b2a/L7_absmax": 0.33372784, "gate_b2a/L7_rms": 0.05751244, "gate_b2a/L11_absmean": 0.08478981, "gate_b2a/L11_absmax": 0.6136387, "gate_b2a/L11_rms": 0.09374078, "gate_b2a/L14_absmean": 0.22010842, "gate_b2a/L14_absmax": 0.80380654, "gate_b2a/L14_rms": 0.22902177, "gate_b2c/L3_absmean": 0.0376357, "gate_b2c/L3_absmax": 0.54958707, "gate_b2c/L3_rms": 0.04751862, "gate_b2c/L5_absmean": 0.03866254, "gate_b2c/L5_absmax": 0.53459126, "gate_b2c/L5_rms": 0.04762299, "gate_b2c/L6_absmean": 0.05585747, "gate_b2c/L6_absmax": 0.33625025, "gate_b2c/L6_rms": 0.06144859}, "streams": {"cos_a2b/round0": 0.351955, "cos_c2b/round0": 0.241911, "cos_b2a/round0": 0.174376, "cos_b2c/round0": 0.150721, "rms_a/round0": 7.272533, "rms_b/round0": 12.427635, "rms_c/round0": 4.865805, "cos_a2b/round1": 0.411323, "cos_c2b/round1": 0.306675, "cos_b2a/round1": 0.254631, "cos_b2c/round1": 0.114183, "rms_a/round1": 14.380021, "rms_b/round1": 29.2714, "rms_c/round1": 10.509655, "cos_a2b/round2": 0.538278, "cos_c2b/round2": 0.522941, "cos_b2a/round2": 0.100467, "cos_b2c/round2": 0.191099, "rms_a/round2": 42.722527, "rms_b/round2": 83.606247, "rms_c/round2": 20.685152}, "alpha_mean": 0.44474268723976706, "alpha_hist": [0.00664, 0.06343, 0.16565, 0.17468, 0.19667, 0.18951, 0.13545, 0.06271, 0.00519, 6e-05], "alpha_by_type": {"special": {"share": 0.00032, "mean": 0.53438, "std": 0.12863}, "whitespace": {"share": 0.05374, "mean": 0.54269, "std": 0.11335}, "digit": {"share": 0.05791, "mean": 0.35338, "std": 0.14923}, "word_start": {"share": 0.55194, "mean": 0.46812, "std": 0.15725}, "word_cont": {"share": 0.2289, "mean": 0.32905, "std": 0.12763}, "punct": {"share": 0.10719, "mean": 0.5714, "std": 0.15483}}, "alpha_p05": 0.18567, "alpha_p50": 0.44035, "alpha_p95": 0.71063}
{"step": 66810, "total_steps": 72479, "tokens": 35027681280, "loss": 3.018920600414276, "l_mix": 1.8571920990943909, "l_a": 1.9719323813915253, "l_c": 1.900495857000351, "z_loss": 50.85240030288696, "lr": 0.0012277146538336597, "wd": 0.01, "grad_norm": 0.1480274200439453, "tokens_per_second": 1378788.9918701216, "eta_seconds": 2155.6515823125837, "data_wait_seconds": 2.60770320892334e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 66820, "total_steps": 72479, "tokens": 35032924160, "loss": 2.9792622923851013, "l_mix": 1.833313450217247, "l_a": 1.94144606590271, "l_c": 1.8783832043409348, "z_loss": 50.32809638977051, "lr": 0.001224398259721016, "wd": 0.01, "grad_norm": 0.1398395150899887, "tokens_per_second": 1369792.2550759173, "eta_seconds": 2165.9823093652726, "data_wait_seconds": 2.0742416381835938e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 66830, "total_steps": 72479, "tokens": 35038167040, "loss": 3.0150607526302338, "l_mix": 1.8553231358528137, "l_a": 1.9652607142925262, "l_c": 1.9005314260721207, "z_loss": 49.65746831893921, "lr": 0.0012210812441627514, "wd": 0.01, "grad_norm": 0.12282677739858627, "tokens_per_second": 1377155.2066637825, "eta_seconds": 2150.5948622703554, "data_wait_seconds": 1.823902130126953e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 66840, "total_steps": 72479, "tokens": 35043409920, "loss": 4.4203134179115295, "l_mix": 2.74112132191658, "l_a": 2.821536362171173, "l_c": 2.7757703363895416, "z_loss": 45.0582013130188, "lr": 0.0012177636348657617, "wd": 0.01, "grad_norm": 0.09740010648965836, "tokens_per_second": 1462075.4054270948, "eta_seconds": 2022.0981907129287, "data_wait_seconds": 2.2351741790771484e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 66850, "total_steps": 72479, "tokens": 35048652800, "loss": 4.458586275577545, "l_mix": 2.764577865600586, "l_a": 2.8450958728790283, "l_c": 2.801598399877548, "z_loss": 45.284661293029785, "lr": 0.0012144454595419015, "wd": 0.01, "grad_norm": 0.13885675370693207, "tokens_per_second": 1440918.9227623204, "eta_seconds": 2048.149347877503, "data_wait_seconds": 1.7821788787841797e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 66860, "total_steps": 72479, "tokens": 35053895680, "loss": 4.357781529426575, "l_mix": 2.702271819114685, "l_a": 2.782040536403656, "l_c": 2.736324965953827, "z_loss": 45.06991100311279, "lr": 0.0012111267459077532, "wd": 0.01, "grad_norm": 0.12577402591705322, "tokens_per_second": 1238674.4754823907, "eta_seconds": 2378.3280678749084, "data_wait_seconds": 1.761317253112793e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 66870, "total_steps": 72479, "tokens": 35059138560, "loss": 4.370938777923584, "l_mix": 2.7096937894821167, "l_a": 2.791404604911804, "l_c": 2.7460786402225494, "z_loss": 45.85742139816284, "lr": 0.0012078075216843966, "wd": 0.01, "grad_norm": 0.1327195167541504, "tokens_per_second": 1402375.7841101345, "eta_seconds": 2096.9638989210125, "data_wait_seconds": 1.8656253814697266e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 66880, "total_steps": 72479, "tokens": 35064381440, "loss": 4.368872344493866, "l_mix": 2.708344727754593, "l_a": 2.7916097342967987, "l_c": 2.74348184466362, "z_loss": 46.059528827667236, "lr": 0.0012044878145971757, "wd": 0.01, "grad_norm": 0.13612031936645508, "tokens_per_second": 1446292.0335617757, "eta_seconds": 2029.665132546425, "data_wait_seconds": 1.621246337890625e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 66890, "total_steps": 72479, "tokens": 35069624320, "loss": 3.004332095384598, "l_mix": 1.8486346453428268, "l_a": 1.9573230296373367, "l_c": 1.8950015008449554, "z_loss": 51.168521881103516, "lr": 0.0012011676523754678, "wd": 0.01, "grad_norm": 0.16343410313129425, "tokens_per_second": 1233202.6318094048, "eta_seconds": 2376.126644897461, "data_wait_seconds": 2.3126602172851562e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 66900, "total_steps": 72479, "tokens": 35074867200, "loss": 3.044213831424713, "l_mix": 1.8733484596014023, "l_a": 1.9856461584568024, "l_c": 1.9172382950782776, "z_loss": 49.9860520362854, "lr": 0.0011978470627524527, "wd": 0.01, "grad_norm": 0.16119244694709778, "tokens_per_second": 1410170.9479461324, "eta_seconds": 2074.2185592889787, "data_wait_seconds": 2.0503997802734375e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.1639102, "gate_a2b/L1_absmax": 0.67495865, "gate_a2b/L1_rms": 0.17338082, "gate_a2b/L3_absmean": 0.15410975, "gate_a2b/L3_absmax": 0.50352126, "gate_a2b/L3_rms": 0.15965593, "gate_a2b/L5_absmean": 0.22154026, "gate_a2b/L5_absmax": 0.73148388, "gate_a2b/L5_rms": 0.2274631, "gate_c2b/L1_absmean": 0.04329279, "gate_c2b/L1_absmax": 0.60928005, "gate_c2b/L1_rms": 0.07115471, "gate_c2b/L3_absmean": 0.16444393, "gate_c2b/L3_absmax": 0.3877815, "gate_c2b/L3_rms": 0.16792983, "gate_c2b/L5_absmean": 0.46906438, "gate_c2b/L5_absmax": 0.67745405, "gate_c2b/L5_rms": 0.47799572, "gate_b2a/L7_absmean": 0.05187839, "gate_b2a/L7_absmax": 0.33327621, "gate_b2a/L7_rms": 0.05754501, "gate_b2a/L11_absmean": 0.08497708, "gate_b2a/L11_absmax": 0.61380595, "gate_b2a/L11_rms": 0.09392538, "gate_b2a/L14_absmean": 0.22035754, "gate_b2a/L14_absmax": 0.8022719, "gate_b2a/L14_rms": 0.22919898, "gate_b2c/L3_absmean": 0.03777842, "gate_b2c/L3_absmax": 0.54958045, "gate_b2c/L3_rms": 0.04764871, "gate_b2c/L5_absmean": 0.03864562, "gate_b2c/L5_absmax": 0.53155023, "gate_b2c/L5_rms": 0.04756884, "gate_b2c/L6_absmean": 0.05596472, "gate_b2c/L6_absmax": 0.33520246, "gate_b2c/L6_rms": 0.06148227}, "streams": {"cos_a2b/round0": 0.351609, "cos_c2b/round0": 0.205244, "cos_b2a/round0": 0.157362, "cos_b2c/round0": 0.128709, "rms_a/round0": 7.587162, "rms_b/round0": 12.946253, "rms_c/round0": 5.062454, "cos_a2b/round1": 0.409037, "cos_c2b/round1": 0.309633, "cos_b2a/round1": 0.247338, "cos_b2c/round1": 0.11832, "rms_a/round1": 15.457531, "rms_b/round1": 31.439955, "rms_c/round1": 11.566185, "cos_a2b/round2": 0.517089, "cos_c2b/round2": 0.512027, "cos_b2a/round2": 0.091564, "cos_b2c/round2": 0.174254, "rms_a/round2": 44.189266, "rms_b/round2": 84.212631, "rms_c/round2": 22.826139}, "alpha_mean": 0.4065458280603025, "alpha_hist": [0.04541, 0.17816, 0.18576, 0.1347, 0.10829, 0.10306, 0.1031, 0.0909, 0.04726, 0.00336], "alpha_by_type": {"special": {"share": 0.00116, "mean": 0.65694, "std": 0.12357}, "whitespace": {"share": 0.032, "mean": 0.63509, "std": 0.17041}, "digit": {"share": 0.00769, "mean": 0.40646, "std": 0.18494}, "word_start": {"share": 0.58252, "mean": 0.41178, "std": 0.21396}, "word_cont": {"share": 0.25902, "mean": 0.2606, "std": 0.14719}, "punct": {"share": 0.11761, "mean": 0.63737, "std": 0.18688}}, "alpha_p05": 0.10596, "alpha_p50": 0.35757, "alpha_p95": 0.78432}
{"step": 66910, "total_steps": 72479, "tokens": 35080110080, "loss": 4.60660719871521, "l_mix": 2.8588586151599884, "l_a": 2.931725323200226, "l_c": 2.894102931022644, "z_loss": 43.943541049957275, "lr": 0.0011945260734648792, "wd": 0.01, "grad_norm": 0.115836501121521, "tokens_per_second": 1439606.8753116578, "eta_seconds": 2028.1647177934647, "data_wait_seconds": 1.633167266845703e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 66920, "total_steps": 72479, "tokens": 35085352960, "loss": 4.6362144947052, "l_mix": 2.876908779144287, "l_a": 2.950624316930771, "l_c": 2.9137277007102966, "z_loss": 44.46532392501831, "lr": 0.0011912047122528354, "wd": 0.01, "grad_norm": 0.1292891949415207, "tokens_per_second": 1465874.4865757248, "eta_seconds": 1988.2445725679397, "data_wait_seconds": 2.0712614059448242e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 66930, "total_steps": 72479, "tokens": 35090595840, "loss": 4.575287103652954, "l_mix": 2.838805317878723, "l_a": 2.9163532853126526, "l_c": 2.87191903591156, "z_loss": 44.36405849456787, "lr": 0.0011878830068595152, "wd": 0.01, "grad_norm": 0.1422693282365799, "tokens_per_second": 1467356.3739558542, "eta_seconds": 1982.6636280298233, "data_wait_seconds": 1.6957521438598633e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 66940, "total_steps": 72479, "tokens": 35095838720, "loss": 4.496430516242981, "l_mix": 2.7891630232334137, "l_a": 2.8669474720954895, "l_c": 2.823943853378296, "z_loss": 45.21107625961304, "lr": 0.001184560985030988, "wd": 0.01, "grad_norm": 0.15352562069892883, "tokens_per_second": 1422655.2479909654, "eta_seconds": 2041.2754503250123, "data_wait_seconds": 1.8358230590820312e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 66950, "total_steps": 72479, "tokens": 35101081600, "loss": 3.041535407304764, "l_mix": 1.8711233288049698, "l_a": 1.9837065935134888, "l_c": 1.9176670461893082, "z_loss": 50.53454065322876, "lr": 0.0011812386745159666, "wd": 0.01, "grad_norm": 0.13207146525382996, "tokens_per_second": 1432618.3990896486, "eta_seconds": 2023.4197423696519, "data_wait_seconds": 2.6673078536987305e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 66960, "total_steps": 72479, "tokens": 35106324480, "loss": 4.3635061383247375, "l_mix": 2.7052982449531555, "l_a": 2.7867608964443207, "l_c": 2.740598440170288, "z_loss": 45.98310565948486, "lr": 0.0011779161030655743, "wd": 0.01, "grad_norm": 0.10816876590251923, "tokens_per_second": 1345943.9349904326, "eta_seconds": 2149.826153063774, "data_wait_seconds": 1.8775463104248047e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 66970, "total_steps": 72479, "tokens": 35111567360, "loss": 4.450957357883453, "l_mix": 2.759732633829117, "l_a": 2.8408002853393555, "l_c": 2.796615242958069, "z_loss": 45.79682159423828, "lr": 0.0011745932984331155, "wd": 0.01, "grad_norm": 0.1217799037694931, "tokens_per_second": 1447533.397284801, "eta_seconds": 1995.3270835876467, "data_wait_seconds": 2.4110078811645508e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 66980, "total_steps": 72479, "tokens": 35116810240, "loss": 4.38381040096283, "l_mix": 2.718550145626068, "l_a": 2.7987272441387177, "l_c": 2.7521398067474365, "z_loss": 45.49743318557739, "lr": 0.0011712702883738399, "wd": 0.01, "grad_norm": 0.16860263049602509, "tokens_per_second": 1423349.9254558296, "eta_seconds": 2025.545272064209, "data_wait_seconds": 3.1828880310058594e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 66990, "total_steps": 72479, "tokens": 35122053120, "loss": 4.399033308029175, "l_mix": 2.726775735616684, "l_a": 2.8101506531238556, "l_c": 2.764041095972061, "z_loss": 45.4592022895813, "lr": 0.0011679471006447158, "wd": 0.01, "grad_norm": 0.12083763629198074, "tokens_per_second": 1052809.7253272892, "eta_seconds": 2733.4633816242217, "data_wait_seconds": 2.3990869522094727e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 67000, "total_steps": 72479, "tokens": 35127296000, "loss": 3.0109411478042603, "l_mix": 1.8525537103414536, "l_a": 1.963822290301323, "l_c": 1.8974691331386566, "z_loss": 49.794445514678955, "lr": 0.0011646237630041942, "wd": 0.01, "grad_norm": 0.1409115344285965, "tokens_per_second": 1356488.4651288688, "eta_seconds": 2117.6545365810393, "data_wait_seconds": 1.7970800399780273e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16425399, "gate_a2b/L1_absmax": 0.67366159, "gate_a2b/L1_rms": 0.17372754, "gate_a2b/L3_absmean": 0.1542068, "gate_a2b/L3_absmax": 0.50320578, "gate_a2b/L3_rms": 0.15971644, "gate_a2b/L5_absmean": 0.22184305, "gate_a2b/L5_absmax": 0.73238319, "gate_a2b/L5_rms": 0.22773455, "gate_c2b/L1_absmean": 0.04344731, "gate_c2b/L1_absmax": 0.60821652, "gate_c2b/L1_rms": 0.07131754, "gate_c2b/L3_absmean": 0.16441977, "gate_c2b/L3_absmax": 0.38849437, "gate_c2b/L3_rms": 0.16786768, "gate_c2b/L5_absmean": 0.46916953, "gate_c2b/L5_absmax": 0.67833728, "gate_c2b/L5_rms": 0.47807291, "gate_b2a/L7_absmean": 0.05196595, "gate_b2a/L7_absmax": 0.33384573, "gate_b2a/L7_rms": 0.05767471, "gate_b2a/L11_absmean": 0.08502173, "gate_b2a/L11_absmax": 0.61355525, "gate_b2a/L11_rms": 0.09394872, "gate_b2a/L14_absmean": 0.22070187, "gate_b2a/L14_absmax": 0.80147755, "gate_b2a/L14_rms": 0.22949134, "gate_b2c/L3_absmean": 0.03792463, "gate_b2c/L3_absmax": 0.5499534, "gate_b2c/L3_rms": 0.04779216, "gate_b2c/L5_absmean": 0.03866991, "gate_b2c/L5_absmax": 0.53145158, "gate_b2c/L5_rms": 0.04758284, "gate_b2c/L6_absmean": 0.05609621, "gate_b2c/L6_absmax": 0.33495736, "gate_b2c/L6_rms": 0.06161813}, "streams": {"cos_a2b/round0": 0.353844, "cos_c2b/round0": 0.209134, "cos_b2a/round0": 0.157021, "cos_b2c/round0": 0.12963, "rms_a/round0": 7.584233, "rms_b/round0": 12.877013, "rms_c/round0": 5.024539, "cos_a2b/round1": 0.40914, "cos_c2b/round1": 0.306002, "cos_b2a/round1": 0.246751, "cos_b2c/round1": 0.117213, "rms_a/round1": 15.335357, "rms_b/round1": 31.306456, "rms_c/round1": 11.522012, "cos_a2b/round2": 0.51755, "cos_c2b/round2": 0.512537, "cos_b2a/round2": 0.091028, "cos_b2c/round2": 0.172824, "rms_a/round2": 44.079449, "rms_b/round2": 84.153198, "rms_c/round2": 22.883772}, "alpha_mean": 0.3965109854178337, "alpha_hist": [0.05557, 0.185, 0.18256, 0.13512, 0.10594, 0.10699, 0.10025, 0.0838, 0.04146, 0.00331], "alpha_by_type": {"special": {"share": 0.00108, "mean": 0.64526, "std": 0.10637}, "whitespace": {"share": 0.0361, "mean": 0.62108, "std": 0.17437}, "digit": {"share": 0.00966, "mean": 0.39867, "std": 0.19544}, "word_start": {"share": 0.57472, "mean": 0.39722, "std": 0.21551}, "word_cont": {"share": 0.25426, "mean": 0.25502, "std": 0.14265}, "punct": {"share": 0.12418, "mean": 0.61531, "std": 0.18467}}, "alpha_p05": 0.1016, "alpha_p50": 0.34202, "alpha_p95": 0.77627}
{"step": 67010, "total_steps": 72479, "tokens": 35132538880, "loss": 3.012805789709091, "l_mix": 1.8533886224031448, "l_a": 1.965214729309082, "l_c": 1.899508848786354, "z_loss": 49.45290184020996, "lr": 0.0011613003032119784, "wd": 0.01, "grad_norm": 0.16050021350383759, "tokens_per_second": 1423885.946033027, "eta_seconds": 2013.7364793777467, "data_wait_seconds": 2.142786979675293e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 67020, "total_steps": 72479, "tokens": 35137781760, "loss": 2.8213962614536285, "l_mix": 1.7473581731319427, "l_a": 1.80939482152462, "l_c": 1.7707319557666779, "z_loss": 57.93327474594116, "lr": 0.0011579767490287921, "wd": 0.01, "grad_norm": 0.14543119072914124, "tokens_per_second": 1444841.5435264918, "eta_seconds": 1980.9010924577713, "data_wait_seconds": 2.950429916381836e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 67030, "total_steps": 72479, "tokens": 35143024640, "loss": 4.468684136867523, "l_mix": 2.7711006700992584, "l_a": 2.8525368571281433, "l_c": 2.8060742020606995, "z_loss": 45.63298940658569, "lr": 0.001154653128216147, "wd": 0.01, "grad_norm": 0.12969614565372467, "tokens_per_second": 1347544.593542811, "eta_seconds": 2120.0376786708835, "data_wait_seconds": 2.0265579223632812e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 67040, "total_steps": 72479, "tokens": 35148267520, "loss": 4.367920696735382, "l_mix": 2.707597702741623, "l_a": 2.789479821920395, "l_c": 2.744929939508438, "z_loss": 45.35947608947754, "lr": 0.0011513294685361124, "wd": 0.01, "grad_norm": 0.11661616712808609, "tokens_per_second": 1420285.0703214519, "eta_seconds": 2007.7676598787307, "data_wait_seconds": 2.771615982055664e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 67050, "total_steps": 72479, "tokens": 35153510400, "loss": 4.33011919260025, "l_mix": 2.6841318905353546, "l_a": 2.7671716809272766, "l_c": 2.71945258975029, "z_loss": 45.99052047729492, "lr": 0.0011480057977510817, "wd": 0.01, "grad_norm": 0.1012030616402626, "tokens_per_second": 1470794.999130508, "eta_seconds": 1935.2524000167846, "data_wait_seconds": 1.6301870346069336e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 67060, "total_steps": 72479, "tokens": 35158753280, "loss": 4.419223785400391, "l_mix": 2.7398585379123688, "l_a": 2.8242649137973785, "l_c": 2.7736188173294067, "z_loss": 45.73483657836914, "lr": 0.0011446821436235403, "wd": 0.01, "grad_norm": 0.1337125450372696, "tokens_per_second": 1341219.0909460739, "eta_seconds": 2118.309149622917, "data_wait_seconds": 1.5854835510253906e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 67070, "total_steps": 72479, "tokens": 35163996160, "loss": 4.380307734012604, "l_mix": 2.7152806222438812, "l_a": 2.798440933227539, "l_c": 2.7516492009162903, "z_loss": 46.217631340026855, "lr": 0.0011413585339158363, "wd": 0.01, "grad_norm": 0.10473498702049255, "tokens_per_second": 1470380.3768627297, "eta_seconds": 1928.6667835235594, "data_wait_seconds": 2.4616718292236328e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 67080, "total_steps": 72479, "tokens": 35169239040, "loss": 4.333138644695282, "l_mix": 2.6864284574985504, "l_a": 2.7676206827163696, "l_c": 2.7214129865169525, "z_loss": 45.78579092025757, "lr": 0.001138034996389945, "wd": 0.01, "grad_norm": 0.12705297768115997, "tokens_per_second": 1431650.8253923343, "eta_seconds": 1977.1796738386154, "data_wait_seconds": 1.52587890625e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 67090, "total_steps": 72479, "tokens": 35174481920, "loss": 4.38994687795639, "l_mix": 2.72189524769783, "l_a": 2.8040906488895416, "l_c": 2.7560811936855316, "z_loss": 45.528053283691406, "lr": 0.0011347115588072395, "wd": 0.01, "grad_norm": 0.12550899386405945, "tokens_per_second": 1328075.7162757793, "eta_seconds": 2127.4299329280852, "data_wait_seconds": 2.7388334274291992e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 67100, "total_steps": 72479, "tokens": 35179724800, "loss": 4.398504436016083, "l_mix": 2.726909965276718, "l_a": 2.809849262237549, "l_c": 2.7621322572231293, "z_loss": 46.291603088378906, "lr": 0.0011313882489282577, "wd": 0.01, "grad_norm": 0.15559320151805878, "tokens_per_second": 1374584.1242850942, "eta_seconds": 2051.63518345356, "data_wait_seconds": 1.5914440155029297e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.1643313, "gate_a2b/L1_absmax": 0.67385459, "gate_a2b/L1_rms": 0.173759, "gate_a2b/L3_absmean": 0.15427139, "gate_a2b/L3_absmax": 0.50296706, "gate_a2b/L3_rms": 0.15977287, "gate_a2b/L5_absmean": 0.22206075, "gate_a2b/L5_absmax": 0.7327112, "gate_a2b/L5_rms": 0.2279858, "gate_c2b/L1_absmean": 0.04369594, "gate_c2b/L1_absmax": 0.60810643, "gate_c2b/L1_rms": 0.07118553, "gate_c2b/L3_absmean": 0.16435048, "gate_c2b/L3_absmax": 0.38793877, "gate_c2b/L3_rms": 0.16782337, "gate_c2b/L5_absmean": 0.46892786, "gate_c2b/L5_absmax": 0.67870438, "gate_c2b/L5_rms": 0.47783008, "gate_b2a/L7_absmean": 0.05184592, "gate_b2a/L7_absmax": 0.33392152, "gate_b2a/L7_rms": 0.0575423, "gate_b2a/L11_absmean": 0.08507644, "gate_b2a/L11_absmax": 0.61352479, "gate_b2a/L11_rms": 0.0939914, "gate_b2a/L14_absmean": 0.22047082, "gate_b2a/L14_absmax": 0.80183059, "gate_b2a/L14_rms": 0.22929214, "gate_b2c/L3_absmean": 0.03779866, "gate_b2c/L3_absmax": 0.54938, "gate_b2c/L3_rms": 0.04768538, "gate_b2c/L5_absmean": 0.03892519, "gate_b2c/L5_absmax": 0.53273088, "gate_b2c/L5_rms": 0.04779276, "gate_b2c/L6_absmean": 0.05623848, "gate_b2c/L6_absmax": 0.33457601, "gate_b2c/L6_rms": 0.06175686}, "streams": {"cos_a2b/round0": 0.355215, "cos_c2b/round0": 0.23357, "cos_b2a/round0": 0.166803, "cos_b2c/round0": 0.150268, "rms_a/round0": 7.295602, "rms_b/round0": 12.585802, "rms_c/round0": 4.924222, "cos_a2b/round1": 0.410341, "cos_c2b/round1": 0.304457, "cos_b2a/round1": 0.255849, "cos_b2c/round1": 0.107236, "rms_a/round1": 14.415404, "rms_b/round1": 29.70866, "rms_c/round1": 10.675497, "cos_a2b/round2": 0.538739, "cos_c2b/round2": 0.518138, "cos_b2a/round2": 0.097129, "cos_b2c/round2": 0.183012, "rms_a/round2": 42.91235, "rms_b/round2": 82.903885, "rms_c/round2": 21.113422}, "alpha_mean": 0.4270165617817412, "alpha_hist": [0.01187, 0.08713, 0.18567, 0.19313, 0.17227, 0.15158, 0.11577, 0.07213, 0.01021, 0.00024], "alpha_by_type": {"special": {"share": 0.00104, "mean": 0.54089, "std": 0.14438}, "whitespace": {"share": 0.02554, "mean": 0.52445, "std": 0.13823}, "digit": {"share": 0.03654, "mean": 0.46175, "std": 0.1192}, "word_start": {"share": 0.56549, "mean": 0.44095, "std": 0.17685}, "word_cont": {"share": 0.26392, "mean": 0.32159, "std": 0.13258}, "punct": {"share": 0.10747, "mean": 0.57654, "std": 0.16452}}, "alpha_p05": 0.15279, "alpha_p50": 0.42298, "alpha_p95": 0.72389}
{"step": 67110, "total_steps": 72479, "tokens": 35184967680, "loss": 3.0400776863098145, "l_mix": 1.870583936572075, "l_a": 1.982708603143692, "l_c": 1.9156038016080856, "z_loss": 50.20235013961792, "lr": 0.0011280650945124713, "wd": 0.01, "grad_norm": 0.12385547161102295, "tokens_per_second": 1409715.3267719545, "eta_seconds": 1996.7877333402635, "data_wait_seconds": 2.142786979675293e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 67120, "total_steps": 72479, "tokens": 35190210560, "loss": 3.031562954187393, "l_mix": 1.8655771762132645, "l_a": 1.9760399013757706, "l_c": 1.9105792194604874, "z_loss": 49.97546148300171, "lr": 0.0011247421233180534, "wd": 0.01, "grad_norm": 0.14006678760051727, "tokens_per_second": 1469020.4339663594, "eta_seconds": 1912.6074267148972, "data_wait_seconds": 1.7523765563964844e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 67130, "total_steps": 72479, "tokens": 35195453440, "loss": 2.826540529727936, "l_mix": 1.7500471770763397, "l_a": 1.8137456476688385, "l_c": 1.7745652496814728, "z_loss": 57.537864208221436, "lr": 0.0011214193631016466, "wd": 0.01, "grad_norm": 0.12792953848838806, "tokens_per_second": 1428030.093615182, "eta_seconds": 1963.8357234477996, "data_wait_seconds": 5.754828453063965e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 67140, "total_steps": 72479, "tokens": 35200696320, "loss": 4.487134695053101, "l_mix": 2.7825475931167603, "l_a": 2.8636623919010162, "l_c": 2.818294733762741, "z_loss": 44.92930221557617, "lr": 0.001118096841618131, "wd": 0.01, "grad_norm": 0.11171477288007736, "tokens_per_second": 1227690.9752782276, "eta_seconds": 2280.031122136116, "data_wait_seconds": 1.7017126083374023e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 67150, "total_steps": 72479, "tokens": 35205939200, "loss": 4.37925785779953, "l_mix": 2.7149785459041595, "l_a": 2.7959910333156586, "l_c": 2.7516065537929535, "z_loss": 45.549922943115234, "lr": 0.0011147745866203923, "wd": 0.01, "grad_norm": 0.15605215728282928, "tokens_per_second": 1457386.31930473, "eta_seconds": 1917.083147406578, "data_wait_seconds": 1.767277717590332e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 67160, "total_steps": 72479, "tokens": 35211182080, "loss": 4.359062850475311, "l_mix": 2.7024524807929993, "l_a": 2.7864310443401337, "l_c": 2.735603243112564, "z_loss": 46.01923894882202, "lr": 0.0011114526258590907, "wd": 0.01, "grad_norm": 0.15856625139713287, "tokens_per_second": 1386392.4002239893, "eta_seconds": 2011.4708300113678, "data_wait_seconds": 1.806020736694336e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 67170, "total_steps": 72479, "tokens": 35216424960, "loss": 4.490752220153809, "l_mix": 2.784995287656784, "l_a": 2.8654446601867676, "l_c": 2.8204113245010376, "z_loss": 45.89794969558716, "lr": 0.001108130987082429, "wd": 0.01, "grad_norm": 0.10536947846412659, "tokens_per_second": 1183927.5568645322, "eta_seconds": 2351.026442337036, "data_wait_seconds": 1.7523765563964844e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 67180, "total_steps": 72479, "tokens": 35221667840, "loss": 4.418752014636993, "l_mix": 2.739366054534912, "l_a": 2.820615142583847, "l_c": 2.777337819337845, "z_loss": 46.155396938323975, "lr": 0.0011048096980359196, "wd": 0.01, "grad_norm": 0.16188763082027435, "tokens_per_second": 1446684.0435016383, "eta_seconds": 1920.3931393861772, "data_wait_seconds": 1.6391277313232422e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 67190, "total_steps": 72479, "tokens": 35226910720, "loss": 4.717692017555237, "l_mix": 2.9279501140117645, "l_a": 3.0021259486675262, "l_c": 2.96368008852005, "z_loss": 44.55748891830444, "lr": 0.0011014887864621544, "wd": 0.01, "grad_norm": 0.10869486629962921, "tokens_per_second": 1454056.7592902267, "eta_seconds": 1907.0501988887788, "data_wait_seconds": 1.6301870346069336e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 67200, "total_steps": 72479, "tokens": 35232153600, "loss": 4.7360804080963135, "l_mix": 2.9399574398994446, "l_a": 3.0140116214752197, "l_c": 2.9730647206306458, "z_loss": 43.06698274612427, "lr": 0.0010981682801005715, "wd": 0.01, "grad_norm": 0.11513091623783112, "tokens_per_second": 1294987.2307906665, "eta_seconds": 2137.2537784099577, "data_wait_seconds": 2.3126602172851562e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16447406, "gate_a2b/L1_absmax": 0.67382026, "gate_a2b/L1_rms": 0.17389607, "gate_a2b/L3_absmean": 0.15450652, "gate_a2b/L3_absmax": 0.50133473, "gate_a2b/L3_rms": 0.16000725, "gate_a2b/L5_absmean": 0.22211595, "gate_a2b/L5_absmax": 0.73257279, "gate_a2b/L5_rms": 0.2280439, "gate_c2b/L1_absmean": 0.04416309, "gate_c2b/L1_absmax": 0.60830063, "gate_c2b/L1_rms": 0.0716878, "gate_c2b/L3_absmean": 0.16415639, "gate_c2b/L3_absmax": 0.38793615, "gate_c2b/L3_rms": 0.16763099, "gate_c2b/L5_absmean": 0.46917954, "gate_c2b/L5_absmax": 0.67915821, "gate_c2b/L5_rms": 0.4780592, "gate_b2a/L7_absmean": 0.05225474, "gate_b2a/L7_absmax": 0.33399695, "gate_b2a/L7_rms": 0.05788163, "gate_b2a/L11_absmean": 0.08503959, "gate_b2a/L11_absmax": 0.61361355, "gate_b2a/L11_rms": 0.09398291, "gate_b2a/L14_absmean": 0.22082201, "gate_b2a/L14_absmax": 0.80021822, "gate_b2a/L14_rms": 0.22959463, "gate_b2c/L3_absmean": 0.03784429, "gate_b2c/L3_absmax": 0.55087918, "gate_b2c/L3_rms": 0.04774372, "gate_b2c/L5_absmean": 0.03901838, "gate_b2c/L5_absmax": 0.53345418, "gate_b2c/L5_rms": 0.04786348, "gate_b2c/L6_absmean": 0.05638552, "gate_b2c/L6_absmax": 0.33491957, "gate_b2c/L6_rms": 0.06187731}, "streams": {"cos_a2b/round0": 0.357161, "cos_c2b/round0": 0.235336, "cos_b2a/round0": 0.154885, "cos_b2c/round0": 0.151046, "rms_a/round0": 7.288007, "rms_b/round0": 12.262155, "rms_c/round0": 4.777787, "cos_a2b/round1": 0.401755, "cos_c2b/round1": 0.296755, "cos_b2a/round1": 0.24377, "cos_b2c/round1": 0.098883, "rms_a/round1": 14.237995, "rms_b/round1": 29.154089, "rms_c/round1": 10.475326, "cos_a2b/round2": 0.524197, "cos_c2b/round2": 0.519238, "cos_b2a/round2": 0.091405, "cos_b2c/round2": 0.174764, "rms_a/round2": 42.293835, "rms_b/round2": 84.870255, "rms_c/round2": 20.780003}, "alpha_mean": 0.4324523530742681, "alpha_hist": [0.00447, 0.06413, 0.17845, 0.22359, 0.18127, 0.16046, 0.11958, 0.05643, 0.01088, 0.00073], "alpha_by_type": {"special": {"share": 0.00038, "mean": 0.59906, "std": 0.13282}, "whitespace": {"share": 0.04198, "mean": 0.56701, "std": 0.13168}, "digit": {"share": 0.10951, "mean": 0.40883, "std": 0.16213}, "word_start": {"share": 0.44464, "mean": 0.44997, "std": 0.16417}, "word_cont": {"share": 0.26743, "mean": 0.33697, "std": 0.11641}, "punct": {"share": 0.13606, "mean": 0.5399, "std": 0.16428}}, "alpha_p05": 0.19777, "alpha_p50": 0.40263, "alpha_p95": 0.69864}
{"step": 67210, "total_steps": 72479, "tokens": 35237396480, "loss": 4.574581325054169, "l_mix": 2.8383314311504364, "l_a": 2.9124651849269867, "l_c": 2.8750343024730682, "z_loss": 44.39340353012085, "lr": 0.001094848206687225, "wd": 0.01, "grad_norm": 0.1094539612531662, "tokens_per_second": 1383545.6670010043, "eta_seconds": 1996.6622988224028, "data_wait_seconds": 1.811981201171875e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 67220, "total_steps": 72479, "tokens": 35242639360, "loss": 4.632410526275635, "l_mix": 2.874218851327896, "l_a": 2.948727935552597, "l_c": 2.9119107723236084, "z_loss": 45.35977745056152, "lr": 0.0010915285939545525, "wd": 0.01, "grad_norm": 0.10758642852306366, "tokens_per_second": 1354477.9470051788, "eta_seconds": 2035.6408150434493, "data_wait_seconds": 1.7762184143066406e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 67230, "total_steps": 72479, "tokens": 35247882240, "loss": 2.9822563230991364, "l_mix": 1.8348359912633896, "l_a": 1.9455392509698868, "l_c": 1.8791950196027756, "z_loss": 51.8243522644043, "lr": 0.0010882094696311431, "wd": 0.01, "grad_norm": 0.13180913031101227, "tokens_per_second": 1438939.8361680605, "eta_seconds": 1912.510615682602, "data_wait_seconds": 2.536177635192871e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 67240, "total_steps": 72479, "tokens": 35253125120, "loss": 2.919143885374069, "l_mix": 1.8067501336336136, "l_a": 1.8758321851491928, "l_c": 1.8321468830108643, "z_loss": 57.53776216506958, "lr": 0.0010848908614415071, "wd": 0.01, "grad_norm": 0.10012572258710861, "tokens_per_second": 1205006.3817274086, "eta_seconds": 2279.444220089912, "data_wait_seconds": 2.0205974578857422e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 67250, "total_steps": 72479, "tokens": 35258368000, "loss": 4.423540949821472, "l_mix": 2.742262691259384, "l_a": 2.826259583234787, "l_c": 2.7780010998249054, "z_loss": 44.53993844985962, "lr": 0.0010815727971058423, "wd": 0.01, "grad_norm": 0.1546371877193451, "tokens_per_second": 1289057.3236686124, "eta_seconds": 2126.749448347092, "data_wait_seconds": 0.04930388927459717, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 67260, "total_steps": 72479, "tokens": 35263610880, "loss": 4.342600226402283, "l_mix": 2.6926927268505096, "l_a": 2.7719717621803284, "l_c": 2.727719724178314, "z_loss": 46.12327337265015, "lr": 0.0010782553043398042, "wd": 0.01, "grad_norm": 0.13984233140945435, "tokens_per_second": 650523.3242248212, "eta_seconds": 4206.242835736275, "data_wait_seconds": 3.451108932495117e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 67270, "total_steps": 72479, "tokens": 35268853760, "loss": 4.374503135681152, "l_mix": 2.7126938700675964, "l_a": 2.792725831270218, "l_c": 2.746638000011444, "z_loss": 45.945785999298096, "lr": 0.001074938410854275, "wd": 0.01, "grad_norm": 0.143138587474823, "tokens_per_second": 1384628.1677061028, "eta_seconds": 1972.3823736190795, "data_wait_seconds": 1.7970800399780273e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 67280, "total_steps": 72479, "tokens": 35274096640, "loss": 4.295692265033722, "l_mix": 2.662671595811844, "l_a": 2.7448913753032684, "l_c": 2.698510706424713, "z_loss": 47.63315296173096, "lr": 0.00107162214435513, "wd": 0.01, "grad_norm": 0.1081676036119461, "tokens_per_second": 1344712.897691729, "eta_seconds": 2027.029945707321, "data_wait_seconds": 3.0040740966796875e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 67290, "total_steps": 72479, "tokens": 35279339520, "loss": 4.3785834312438965, "l_mix": 2.714579552412033, "l_a": 2.795579880475998, "l_c": 2.7510990500450134, "z_loss": 46.34814405441284, "lr": 0.0010683065325430075, "wd": 0.01, "grad_norm": 0.1507672816514969, "tokens_per_second": 1431356.7273295573, "eta_seconds": 1900.6655574083327, "data_wait_seconds": 2.3037195205688477e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 67300, "total_steps": 72479, "tokens": 35284582400, "loss": 4.627293765544891, "l_mix": 2.871713787317276, "l_a": 2.9461250603199005, "l_c": 2.9058081209659576, "z_loss": 44.38434410095215, "lr": 0.0010649916031130772, "wd": 0.01, "grad_norm": 0.09072576463222504, "tokens_per_second": 1443803.4538459852, "eta_seconds": 1880.6490210056304, "data_wait_seconds": 2.104043960571289e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.16453879, "gate_a2b/L1_absmax": 0.67365259, "gate_a2b/L1_rms": 0.17396411, "gate_a2b/L3_absmean": 0.15482414, "gate_a2b/L3_absmax": 0.50381696, "gate_a2b/L3_rms": 0.16032293, "gate_a2b/L5_absmean": 0.22224537, "gate_a2b/L5_absmax": 0.7332949, "gate_a2b/L5_rms": 0.22818059, "gate_c2b/L1_absmean": 0.04417145, "gate_c2b/L1_absmax": 0.6071316, "gate_c2b/L1_rms": 0.07161354, "gate_c2b/L3_absmean": 0.16406937, "gate_c2b/L3_absmax": 0.38913292, "gate_c2b/L3_rms": 0.16759758, "gate_c2b/L5_absmean": 0.46897385, "gate_c2b/L5_absmax": 0.68060094, "gate_c2b/L5_rms": 0.4778358, "gate_b2a/L7_absmean": 0.05193011, "gate_b2a/L7_absmax": 0.33394337, "gate_b2a/L7_rms": 0.05761971, "gate_b2a/L11_absmean": 0.08507411, "gate_b2a/L11_absmax": 0.61378646, "gate_b2a/L11_rms": 0.09403141, "gate_b2a/L14_absmean": 0.22102219, "gate_b2a/L14_absmax": 0.80003619, "gate_b2a/L14_rms": 0.22978413, "gate_b2c/L3_absmean": 0.03813123, "gate_b2c/L3_absmax": 0.55027169, "gate_b2c/L3_rms": 0.04794129, "gate_b2c/L5_absmean": 0.03904205, "gate_b2c/L5_absmax": 0.53290975, "gate_b2c/L5_rms": 0.04786834, "gate_b2c/L6_absmean": 0.05648986, "gate_b2c/L6_absmax": 0.33478156, "gate_b2c/L6_rms": 0.06198505}, "streams": {"cos_a2b/round0": 0.353455, "cos_c2b/round0": 0.237885, "cos_b2a/round0": 0.177165, "cos_b2c/round0": 0.153517, "rms_a/round0": 7.127018, "rms_b/round0": 12.435903, "rms_c/round0": 4.814947, "cos_a2b/round1": 0.410705, "cos_c2b/round1": 0.307338, "cos_b2a/round1": 0.260131, "cos_b2c/round1": 0.111221, "rms_a/round1": 14.220753, "rms_b/round1": 29.16968, "rms_c/round1": 10.619239, "cos_a2b/round2": 0.547633, "cos_c2b/round2": 0.515029, "cos_b2a/round2": 0.104969, "cos_b2c/round2": 0.187509, "rms_a/round2": 42.492935, "rms_b/round2": 81.007088, "rms_c/round2": 21.140368}, "alpha_mean": 0.45016507897838665, "alpha_hist": [0.00714, 0.05943, 0.15276, 0.18576, 0.19086, 0.19298, 0.13353, 0.06866, 0.00861, 0.00027], "alpha_by_type": {"special": {"share": 0.00079, "mean": 0.53391, "std": 0.13165}, "whitespace": {"share": 0.07426, "mean": 0.52422, "std": 0.11329}, "digit": {"share": 0.02861, "mean": 0.46413, "std": 0.13782}, "word_start": {"share": 0.54903, "mean": 0.46714, "std": 0.16081}, "word_cont": {"share": 0.23985, "mean": 0.33068, "std": 0.13472}, "punct": {"share": 0.10745, "mean": 0.57462, "std": 0.16047}}, "alpha_p05": 0.18447, "alpha_p50": 0.43975, "alpha_p95": 0.71483}
{"step": 67310, "total_steps": 72479, "tokens": 35289825280, "loss": 4.553095281124115, "l_mix": 2.825575351715088, "l_a": 2.8971428275108337, "l_c": 2.861256629228592, "z_loss": 45.093891620635986, "lr": 0.0010616773837548092, "wd": 0.01, "grad_norm": 0.12094517797231674, "tokens_per_second": 1419182.4818018717, "eta_seconds": 1909.5815420150755, "data_wait_seconds": 2.0891427993774414e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 67320, "total_steps": 72479, "tokens": 35295068160, "loss": 4.515763580799103, "l_mix": 2.8011404871940613, "l_a": 2.8774681091308594, "l_c": 2.837941884994507, "z_loss": 45.82087516784668, "lr": 0.0010583639021517425, "wd": 0.01, "grad_norm": 0.1205928772687912, "tokens_per_second": 1356129.9233668274, "eta_seconds": 1994.500486564636, "data_wait_seconds": 1.9818544387817383e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 67330, "total_steps": 72479, "tokens": 35300311040, "loss": 4.543319344520569, "l_mix": 2.8187623023986816, "l_a": 2.893833100795746, "l_c": 2.8546900153160095, "z_loss": 45.32379674911499, "lr": 0.0010550511859812534, "wd": 0.01, "grad_norm": 0.13075119256973267, "tokens_per_second": 1327517.2246719757, "eta_seconds": 2033.5396496772767, "data_wait_seconds": 2.467632293701172e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 67340, "total_steps": 72479, "tokens": 35305553920, "loss": 4.664270401000977, "l_mix": 2.890781909227371, "l_a": 2.9856683015823364, "l_c": 2.925959676504135, "z_loss": 40.37746858596802, "lr": 0.0010517392629143243, "wd": 0.01, "grad_norm": 1.097113847732544, "tokens_per_second": 1465088.5909861028, "eta_seconds": 1839.0123631954193, "data_wait_seconds": 1.6629695892333984e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 67350, "total_steps": 72479, "tokens": 35310796800, "loss": 2.9486116468906403, "l_mix": 1.8257943987846375, "l_a": 1.8921753615140915, "l_c": 1.8505486994981766, "z_loss": 56.039010524749756, "lr": 0.0010484281606153133, "wd": 0.01, "grad_norm": 0.1386212557554245, "tokens_per_second": 1401128.8528896528, "eta_seconds": 1919.2190257549285, "data_wait_seconds": 1.6927719116210938e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 67360, "total_steps": 72479, "tokens": 35316039680, "loss": 4.410833954811096, "l_mix": 2.735002726316452, "l_a": 2.81665101647377, "l_c": 2.769452929496765, "z_loss": 45.863933086395264, "lr": 0.001045117906741722, "wd": 0.01, "grad_norm": 0.12734553217887878, "tokens_per_second": 1419367.7919692257, "eta_seconds": 1890.8631625890732, "data_wait_seconds": 1.7255544662475586e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 67370, "total_steps": 72479, "tokens": 35321282560, "loss": 4.362094581127167, "l_mix": 2.704385370016098, "l_a": 2.786418229341507, "l_c": 2.7392789125442505, "z_loss": 47.12548542022705, "lr": 0.0010418085289439665, "wd": 0.01, "grad_norm": 0.11778685450553894, "tokens_per_second": 1280171.1019082847, "eta_seconds": 2092.366706299782, "data_wait_seconds": 2.4318695068359375e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 67380, "total_steps": 72479, "tokens": 35326525440, "loss": 4.32977357506752, "l_mix": 2.6835970878601074, "l_a": 2.7671110033988953, "l_c": 2.7201434671878815, "z_loss": 46.379685401916504, "lr": 0.0010385000548651436, "wd": 0.01, "grad_norm": 0.11864462494850159, "tokens_per_second": 1322739.7708466304, "eta_seconds": 2021.0661015272142, "data_wait_seconds": 1.901388168334961e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 67390, "total_steps": 72479, "tokens": 35331768320, "loss": 4.340523600578308, "l_mix": 2.6909604966640472, "l_a": 2.772027462720871, "l_c": 2.7265158891677856, "z_loss": 45.753647327423096, "lr": 0.0010351925121408023, "wd": 0.01, "grad_norm": 0.10133183747529984, "tokens_per_second": 1390832.9176703296, "eta_seconds": 1918.3480618715287, "data_wait_seconds": 1.71661376953125e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 67400, "total_steps": 72479, "tokens": 35337011200, "loss": 4.388834655284882, "l_mix": 2.720563620328903, "l_a": 2.805613398551941, "l_c": 2.755290001630783, "z_loss": 46.15392875671387, "lr": 0.0010318859283987107, "wd": 0.01, "grad_norm": 0.09693633019924164, "tokens_per_second": 1407888.3009957806, "eta_seconds": 1891.3849558353425, "data_wait_seconds": 1.8864870071411133e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16482155, "gate_a2b/L1_absmax": 0.67451251, "gate_a2b/L1_rms": 0.17426178, "gate_a2b/L3_absmean": 0.15470497, "gate_a2b/L3_absmax": 0.50695014, "gate_a2b/L3_rms": 0.16023172, "gate_a2b/L5_absmean": 0.22227643, "gate_a2b/L5_absmax": 0.73267561, "gate_a2b/L5_rms": 0.22823228, "gate_c2b/L1_absmean": 0.044529, "gate_c2b/L1_absmax": 0.60686821, "gate_c2b/L1_rms": 0.07176195, "gate_c2b/L3_absmean": 0.16409723, "gate_c2b/L3_absmax": 0.39001644, "gate_c2b/L3_rms": 0.16759375, "gate_c2b/L5_absmean": 0.46950313, "gate_c2b/L5_absmax": 0.67950344, "gate_c2b/L5_rms": 0.47832793, "gate_b2a/L7_absmean": 0.05222797, "gate_b2a/L7_absmax": 0.33496967, "gate_b2a/L7_rms": 0.057934, "gate_b2a/L11_absmean": 0.08522627, "gate_b2a/L11_absmax": 0.61390734, "gate_b2a/L11_rms": 0.09415398, "gate_b2a/L14_absmean": 0.22127603, "gate_b2a/L14_absmax": 0.79917192, "gate_b2a/L14_rms": 0.22998509, "gate_b2c/L3_absmean": 0.03816345, "gate_b2c/L3_absmax": 0.55080831, "gate_b2c/L3_rms": 0.04796563, "gate_b2c/L5_absmean": 0.03910904, "gate_b2c/L5_absmax": 0.53455925, "gate_b2c/L5_rms": 0.04796672, "gate_b2c/L6_absmean": 0.05650856, "gate_b2c/L6_absmax": 0.33395043, "gate_b2c/L6_rms": 0.06199264}, "streams": {"cos_a2b/round0": 0.35412, "cos_c2b/round0": 0.231481, "cos_b2a/round0": 0.166553, "cos_b2c/round0": 0.150926, "rms_a/round0": 7.200563, "rms_b/round0": 12.514054, "rms_c/round0": 4.93253, "cos_a2b/round1": 0.409912, "cos_c2b/round1": 0.307847, "cos_b2a/round1": 0.259242, "cos_b2c/round1": 0.105141, "rms_a/round1": 14.187451, "rms_b/round1": 29.653124, "rms_c/round1": 10.651586, "cos_a2b/round2": 0.534096, "cos_c2b/round2": 0.521284, "cos_b2a/round2": 0.093365, "cos_b2c/round2": 0.183107, "rms_a/round2": 42.808655, "rms_b/round2": 82.867401, "rms_c/round2": 21.030256}, "alpha_mean": 0.420788729325281, "alpha_hist": [0.01103, 0.09927, 0.19557, 0.18851, 0.16173, 0.14619, 0.11861, 0.07101, 0.0079, 0.00017], "alpha_by_type": {"special": {"share": 0.00092, "mean": 0.55835, "std": 0.10741}, "whitespace": {"share": 0.02324, "mean": 0.51759, "std": 0.12772}, "digit": {"share": 0.03922, "mean": 0.44651, "std": 0.10483}, "word_start": {"share": 0.5705, "mean": 0.44457, "std": 0.1751}, "word_cont": {"share": 0.26448, "mean": 0.29819, "std": 0.1303}, "punct": {"share": 0.10165, "mean": 0.57298, "std": 0.16687}}, "alpha_p05": 0.15585, "alpha_p50": 0.40051, "alpha_p95": 0.71483}
{"step": 67410, "total_steps": 72479, "tokens": 35342254080, "loss": 2.9733533561229706, "l_mix": 1.8417276293039322, "l_a": 1.9048969894647598, "l_c": 1.8671887665987015, "z_loss": 58.35269737243652, "lr": 0.001028580331258629, "wd": 0.01, "grad_norm": 0.1653658002614975, "tokens_per_second": 1457254.8759881926, "eta_seconds": 1823.7138305664062, "data_wait_seconds": 2.422928810119629e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 67420, "total_steps": 72479, "tokens": 35347496960, "loss": 4.506382763385773, "l_mix": 2.7960805892944336, "l_a": 2.8690800070762634, "l_c": 2.8319270610809326, "z_loss": 46.2740535736084, "lr": 0.0010252757483320744, "wd": 0.01, "grad_norm": 0.11549066752195358, "tokens_per_second": 1443228.3709523664, "eta_seconds": 1837.8054682016373, "data_wait_seconds": 2.5838613510131836e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 67430, "total_steps": 72479, "tokens": 35352739840, "loss": 4.511407911777496, "l_mix": 2.7992414832115173, "l_a": 2.874116450548172, "l_c": 2.833104968070984, "z_loss": 45.80308437347412, "lr": 0.0010219722072220922, "wd": 0.01, "grad_norm": 0.16154487431049347, "tokens_per_second": 1473700.8536531345, "eta_seconds": 1796.2465757131574, "data_wait_seconds": 2.014636993408203e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 67440, "total_steps": 72479, "tokens": 35357982720, "loss": 4.58417284488678, "l_mix": 2.844468802213669, "l_a": 2.9188599586486816, "l_c": 2.880153179168701, "z_loss": 45.202595233917236, "lr": 0.0010186697355230273, "wd": 0.01, "grad_norm": 0.11134596914052963, "tokens_per_second": 1424127.1763541694, "eta_seconds": 1855.0922107696533, "data_wait_seconds": 1.7076730728149414e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 67450, "total_steps": 72479, "tokens": 35363225600, "loss": 4.505904018878937, "l_mix": 2.79574853181839, "l_a": 2.8676134049892426, "l_c": 2.8329047858715057, "z_loss": 44.50229597091675, "lr": 0.0010153683608202897, "wd": 0.01, "grad_norm": 0.19372345507144928, "tokens_per_second": 1353002.6222697815, "eta_seconds": 1948.7355815887452, "data_wait_seconds": 1.7017126083374023e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 67460, "total_steps": 72479, "tokens": 35368468480, "loss": 4.631724417209625, "l_mix": 2.8741027414798737, "l_a": 2.9484477043151855, "l_c": 2.9102911949157715, "z_loss": 44.40198802947998, "lr": 0.001012068110690128, "wd": 0.01, "grad_norm": 0.11089631915092468, "tokens_per_second": 1470657.683701861, "eta_seconds": 1789.2685029029847, "data_wait_seconds": 1.963973045349121e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 67470, "total_steps": 72479, "tokens": 35373711360, "loss": 4.453693747520447, "l_mix": 2.761966049671173, "l_a": 2.842264413833618, "l_c": 2.796827733516693, "z_loss": 45.85057067871094, "lr": 0.0010087690126993965, "wd": 0.01, "grad_norm": 0.10822144895792007, "tokens_per_second": 1422525.0254386056, "eta_seconds": 1846.1247043371202, "data_wait_seconds": 2.03549861907959e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 67480, "total_steps": 72479, "tokens": 35378954240, "loss": 4.332957923412323, "l_mix": 2.6864088773727417, "l_a": 2.7675461769104004, "l_c": 2.7209505140781403, "z_loss": 46.384263038635254, "lr": 0.0010054710944053255, "wd": 0.01, "grad_norm": 0.09895993769168854, "tokens_per_second": 1410789.852757183, "eta_seconds": 1857.764788198471, "data_wait_seconds": 2.0295381546020508e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 67490, "total_steps": 72479, "tokens": 35384197120, "loss": 4.308672726154327, "l_mix": 2.670696049928665, "l_a": 2.7543104588985443, "l_c": 2.705611526966095, "z_loss": 46.543705463409424, "lr": 0.0010021743833552917, "wd": 0.01, "grad_norm": 0.08668430149555206, "tokens_per_second": 1426655.8752019533, "eta_seconds": 1833.4294047117235, "data_wait_seconds": 1.71661376953125e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 67500, "total_steps": 72479, "tokens": 35389440000, "loss": 4.330540776252747, "l_mix": 2.6847912967205048, "l_a": 2.766040116548538, "l_c": 2.7197912335395813, "z_loss": 46.90358829498291, "lr": 0.0009988789070865874, "wd": 0.01, "grad_norm": 0.12866471707820892, "tokens_per_second": 1362173.89973069, "eta_seconds": 1916.370554828644, "data_wait_seconds": 3.892183303833008e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.1651767, "gate_a2b/L1_absmax": 0.67431766, "gate_a2b/L1_rms": 0.17461491, "gate_a2b/L3_absmean": 0.15489474, "gate_a2b/L3_absmax": 0.50379455, "gate_a2b/L3_rms": 0.16038337, "gate_a2b/L5_absmean": 0.22271414, "gate_a2b/L5_absmax": 0.73363763, "gate_a2b/L5_rms": 0.22863148, "gate_c2b/L1_absmean": 0.04472392, "gate_c2b/L1_absmax": 0.60719728, "gate_c2b/L1_rms": 0.07156806, "gate_c2b/L3_absmean": 0.16422249, "gate_c2b/L3_absmax": 0.38883901, "gate_c2b/L3_rms": 0.16774736, "gate_c2b/L5_absmean": 0.46937355, "gate_c2b/L5_absmax": 0.68044436, "gate_c2b/L5_rms": 0.47821334, "gate_b2a/L7_absmean": 0.0523154, "gate_b2a/L7_absmax": 0.33518547, "gate_b2a/L7_rms": 0.05797896, "gate_b2a/L11_absmean": 0.08536142, "gate_b2a/L11_absmax": 0.61498207, "gate_b2a/L11_rms": 0.09429024, "gate_b2a/L14_absmean": 0.2212932, "gate_b2a/L14_absmax": 0.79833674, "gate_b2a/L14_rms": 0.22998732, "gate_b2c/L3_absmean": 0.0383351, "gate_b2c/L3_absmax": 0.55013299, "gate_b2c/L3_rms": 0.04811136, "gate_b2c/L5_absmean": 0.03915211, "gate_b2c/L5_absmax": 0.53440398, "gate_b2c/L5_rms": 0.0480254, "gate_b2c/L6_absmean": 0.05668806, "gate_b2c/L6_absmax": 0.33401662, "gate_b2c/L6_rms": 0.06218635}, "streams": {"cos_a2b/round0": 0.355112, "cos_c2b/round0": 0.236811, "cos_b2a/round0": 0.173845, "cos_b2c/round0": 0.152176, "rms_a/round0": 7.188398, "rms_b/round0": 12.534718, "rms_c/round0": 4.894259, "cos_a2b/round1": 0.413696, "cos_c2b/round1": 0.307098, "cos_b2a/round1": 0.260983, "cos_b2c/round1": 0.109693, "rms_a/round1": 14.314642, "rms_b/round1": 29.635201, "rms_c/round1": 10.65885, "cos_a2b/round2": 0.533613, "cos_c2b/round2": 0.526167, "cos_b2a/round2": 0.099558, "cos_b2c/round2": 0.185149, "rms_a/round2": 42.728363, "rms_b/round2": 83.154007, "rms_c/round2": 21.187054}, "alpha_mean": 0.41918916615399837, "alpha_hist": [0.0166, 0.10048, 0.18291, 0.19016, 0.1718, 0.14417, 0.11923, 0.06453, 0.00986, 0.00027], "alpha_by_type": {"special": {"share": 0.00105, "mean": 0.53681, "std": 0.13036}, "whitespace": {"share": 0.02895, "mean": 0.49094, "std": 0.14796}, "digit": {"share": 0.0239, "mean": 0.48128, "std": 0.12575}, "word_start": {"share": 0.57626, "mean": 0.43439, "std": 0.17429}, "word_cont": {"share": 0.27368, "mean": 0.3126, "std": 0.13855}, "punct": {"share": 0.09616, "mean": 0.59317, "std": 0.16206}}, "alpha_p05": 0.14904, "alpha_p50": 0.39817, "alpha_p95": 0.73942}
{"step": 67510, "total_steps": 72479, "tokens": 35394682880, "loss": 4.357881546020508, "l_mix": 2.701641321182251, "l_a": 2.7840813994407654, "l_c": 2.7367190420627594, "z_loss": 46.73906898498535, "lr": 0.0009955846931261917, "wd": 0.01, "grad_norm": 0.11205697804689407, "tokens_per_second": 1285321.5698829954, "eta_seconds": 2026.8757119178772, "data_wait_seconds": 2.2202730178833008e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 67520, "total_steps": 72479, "tokens": 35399925760, "loss": 2.9704078137874603, "l_mix": 1.839941293001175, "l_a": 1.9038461595773697, "l_c": 1.8643756061792374, "z_loss": 57.4061541557312, "lr": 0.000992291768990539, "wd": 0.01, "grad_norm": 0.10869146883487701, "tokens_per_second": 1451059.7213651668, "eta_seconds": 1791.755469274521, "data_wait_seconds": 2.7507543563842773e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 67530, "total_steps": 72479, "tokens": 35405168640, "loss": 2.8456073105335236, "l_mix": 1.7615120708942413, "l_a": 1.8277641832828522, "l_c": 1.785886436700821, "z_loss": 58.88201332092285, "lr": 0.0009890001621852891, "wd": 0.01, "grad_norm": 0.1090511605143547, "tokens_per_second": 1365933.212940338, "eta_seconds": 1899.5813905239104, "data_wait_seconds": 1.8596649169921875e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 67540, "total_steps": 72479, "tokens": 35410411520, "loss": 4.388054192066193, "l_mix": 2.7204956710338593, "l_a": 2.8028887808322906, "l_c": 2.755639463663101, "z_loss": 46.856940269470215, "lr": 0.0009857099002051001, "wd": 0.01, "grad_norm": 0.15340185165405273, "tokens_per_second": 1419953.536274554, "eta_seconds": 1823.6219466686248, "data_wait_seconds": 1.9222497940063477e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 67550, "total_steps": 72479, "tokens": 35415654400, "loss": 4.371558964252472, "l_mix": 2.7099528312683105, "l_a": 2.7931253612041473, "l_c": 2.745561331510544, "z_loss": 46.581111431121826, "lr": 0.0009824210105333947, "wd": 0.01, "grad_norm": 0.1272972822189331, "tokens_per_second": 1477631.9565824044, "eta_seconds": 1748.8898642778397, "data_wait_seconds": 1.8715858459472656e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 67560, "total_steps": 72479, "tokens": 35420897280, "loss": 4.329390227794647, "l_mix": 2.6841340959072113, "l_a": 2.765612691640854, "l_c": 2.718574047088623, "z_loss": 46.2024188041687, "lr": 0.0009791335206421347, "wd": 0.01, "grad_norm": 0.10254321992397308, "tokens_per_second": 1455273.1623082282, "eta_seconds": 1772.1571034193037, "data_wait_seconds": 2.6851892471313477e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 67570, "total_steps": 72479, "tokens": 35426140160, "loss": 4.462893724441528, "l_mix": 2.7674639225006104, "l_a": 2.8485954105854034, "l_c": 2.802836835384369, "z_loss": 45.50326585769653, "lr": 0.0009758474579915886, "wd": 0.01, "grad_norm": 0.10814465582370758, "tokens_per_second": 1474781.1110316014, "eta_seconds": 1745.1605345010757, "data_wait_seconds": 1.609325408935547e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 67580, "total_steps": 72479, "tokens": 35431383040, "loss": 4.428115606307983, "l_mix": 2.7450875639915466, "l_a": 2.82856422662735, "l_c": 2.781529039144516, "z_loss": 46.94483041763306, "lr": 0.000972562850030103, "wd": 0.01, "grad_norm": 0.10319165885448456, "tokens_per_second": 1437390.4478878265, "eta_seconds": 1786.9096846818923, "data_wait_seconds": 1.6450881958007812e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 67590, "total_steps": 72479, "tokens": 35436625920, "loss": 4.374684512615204, "l_mix": 2.711916834115982, "l_a": 2.794709712266922, "l_c": 2.7478488087654114, "z_loss": 46.93273448944092, "lr": 0.0009692797241938739, "wd": 0.01, "grad_norm": 0.12618550658226013, "tokens_per_second": 1420597.3032012263, "eta_seconds": 1804.342459487915, "data_wait_seconds": 1.7642974853515625e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 67600, "total_steps": 72479, "tokens": 35441868800, "loss": 4.396120190620422, "l_mix": 2.725983142852783, "l_a": 2.8056481778621674, "l_c": 2.7614746689796448, "z_loss": 46.14002513885498, "lr": 0.0009659981079067168, "wd": 0.01, "grad_norm": 0.10613778233528137, "tokens_per_second": 1318837.3840036613, "eta_seconds": 1939.5879909276962, "data_wait_seconds": 1.6182661056518555e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16533741, "gate_a2b/L1_absmax": 0.673895, "gate_a2b/L1_rms": 0.17476864, "gate_a2b/L3_absmean": 0.15501027, "gate_a2b/L3_absmax": 0.50431371, "gate_a2b/L3_rms": 0.16051258, "gate_a2b/L5_absmean": 0.22276817, "gate_a2b/L5_absmax": 0.73346305, "gate_a2b/L5_rms": 0.22867851, "gate_c2b/L1_absmean": 0.04468074, "gate_c2b/L1_absmax": 0.60495883, "gate_c2b/L1_rms": 0.07163838, "gate_c2b/L3_absmean": 0.1639964, "gate_c2b/L3_absmax": 0.38846633, "gate_c2b/L3_rms": 0.16749403, "gate_c2b/L5_absmean": 0.4693439, "gate_c2b/L5_absmax": 0.68066424, "gate_c2b/L5_rms": 0.47819, "gate_b2a/L7_absmean": 0.05248479, "gate_b2a/L7_absmax": 0.33472592, "gate_b2a/L7_rms": 0.05810873, "gate_b2a/L11_absmean": 0.08545474, "gate_b2a/L11_absmax": 0.6137358, "gate_b2a/L11_rms": 0.09434161, "gate_b2a/L14_absmean": 0.22162375, "gate_b2a/L14_absmax": 0.79799342, "gate_b2a/L14_rms": 0.23030534, "gate_b2c/L3_absmean": 0.03838106, "gate_b2c/L3_absmax": 0.55107981, "gate_b2c/L3_rms": 0.04816887, "gate_b2c/L5_absmean": 0.0392235, "gate_b2c/L5_absmax": 0.53373975, "gate_b2c/L5_rms": 0.04804692, "gate_b2c/L6_absmean": 0.05684008, "gate_b2c/L6_absmax": 0.33340371, "gate_b2c/L6_rms": 0.06224007}, "streams": {"cos_a2b/round0": 0.354337, "cos_c2b/round0": 0.231883, "cos_b2a/round0": 0.173896, "cos_b2c/round0": 0.150592, "rms_a/round0": 7.203425, "rms_b/round0": 12.601341, "rms_c/round0": 4.90688, "cos_a2b/round1": 0.411271, "cos_c2b/round1": 0.30757, "cos_b2a/round1": 0.2598, "cos_b2c/round1": 0.110169, "rms_a/round1": 14.482837, "rms_b/round1": 29.825449, "rms_c/round1": 10.805045, "cos_a2b/round2": 0.537825, "cos_c2b/round2": 0.520469, "cos_b2a/round2": 0.095176, "cos_b2c/round2": 0.182763, "rms_a/round2": 43.187225, "rms_b/round2": 82.939529, "rms_c/round2": 21.642309}, "alpha_mean": 0.43275766362111057, "alpha_hist": [0.01477, 0.08073, 0.17346, 0.19067, 0.1743, 0.15898, 0.12582, 0.07129, 0.00952, 0.00044], "alpha_by_type": {"special": {"share": 0.00169, "mean": 0.54003, "std": 0.13724}, "whitespace": {"share": 0.02771, "mean": 0.49429, "std": 0.15009}, "digit": {"share": 0.02371, "mean": 0.45769, "std": 0.14272}, "word_start": {"share": 0.57698, "mean": 0.44923, "std": 0.17258}, "word_cont": {"share": 0.26491, "mean": 0.32799, "std": 0.14019}, "punct": {"share": 0.105, "mean": 0.58296, "std": 0.16658}}, "alpha_p05": 0.17385, "alpha_p50": 0.43831, "alpha_p95": 0.72429}
{"step": 67610, "total_steps": 72479, "tokens": 35447111680, "loss": 4.313190281391144, "l_mix": 2.6734029948711395, "l_a": 2.7583052217960358, "l_c": 2.7076520919799805, "z_loss": 46.72178030014038, "lr": 0.0009627180285798391, "wd": 0.01, "grad_norm": 0.10379897803068161, "tokens_per_second": 1341780.9861650357, "eta_seconds": 1902.5148651838304, "data_wait_seconds": 1.868605613708496e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 67620, "total_steps": 72479, "tokens": 35452354560, "loss": 4.363137423992157, "l_mix": 2.7052339911460876, "l_a": 2.7858994603157043, "l_c": 2.7404448986053467, "z_loss": 47.47154235839844, "lr": 0.000959439513611609, "wd": 0.01, "grad_norm": 0.1379360556602478, "tokens_per_second": 1391817.0853658405, "eta_seconds": 1830.352148127556, "data_wait_seconds": 2.8640031814575195e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 67630, "total_steps": 72479, "tokens": 35457597440, "loss": 4.28920978307724, "l_mix": 2.658964902162552, "l_a": 2.741410255432129, "l_c": 2.6927393674850464, "z_loss": 46.721224784851074, "lr": 0.0009561625903873274, "wd": 0.01, "grad_norm": 0.1298709660768509, "tokens_per_second": 1351915.710423578, "eta_seconds": 1880.4963152647017, "data_wait_seconds": 3.501772880554199e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 67640, "total_steps": 72479, "tokens": 35462840320, "loss": 2.9434695541858673, "l_mix": 1.8223361521959305, "l_a": 1.8908181488513947, "l_c": 1.8462928533554077, "z_loss": 56.923319816589355, "lr": 0.0009528872862790007, "wd": 0.01, "grad_norm": 0.11248033493757248, "tokens_per_second": 1417087.3508486226, "eta_seconds": 1790.312806391716, "data_wait_seconds": 2.300739288330078e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 67650, "total_steps": 72479, "tokens": 35468083200, "loss": 4.357069194316864, "l_mix": 2.70115265250206, "l_a": 2.7838775515556335, "l_c": 2.7358440160751343, "z_loss": 45.40114641189575, "lr": 0.0009496136286451097, "wd": 0.01, "grad_norm": 0.1241104006767273, "tokens_per_second": 1391218.848274037, "eta_seconds": 1819.8335618734359, "data_wait_seconds": 1.615285873413086e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 67660, "total_steps": 72479, "tokens": 35473326080, "loss": 4.3151368498802185, "l_mix": 2.675000101327896, "l_a": 2.7561724483966827, "l_c": 2.710949808359146, "z_loss": 46.77419567108154, "lr": 0.0009463416448303827, "wd": 0.01, "grad_norm": 0.09508518874645233, "tokens_per_second": 1414574.4109424504, "eta_seconds": 1786.080571269989, "data_wait_seconds": 1.710653305053711e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 67670, "total_steps": 72479, "tokens": 35478568960, "loss": 4.320033669471741, "l_mix": 2.677786409854889, "l_a": 2.761345773935318, "l_c": 2.71281161904335, "z_loss": 47.636425495147705, "lr": 0.0009430713621655671, "wd": 0.01, "grad_norm": 0.1361347734928131, "tokens_per_second": 1424960.6766541163, "eta_seconds": 1769.3828561782836, "data_wait_seconds": 2.232193946838379e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 67680, "total_steps": 72479, "tokens": 35483811840, "loss": 4.369440019130707, "l_mix": 2.7092626988887787, "l_a": 2.789205312728882, "l_c": 2.7447187900543213, "z_loss": 46.61370277404785, "lr": 0.0009398028079672002, "wd": 0.01, "grad_norm": 0.10439508408308029, "tokens_per_second": 1464409.2420174058, "eta_seconds": 1718.1386458158493, "data_wait_seconds": 1.9997358322143555e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 67690, "total_steps": 72479, "tokens": 35489054720, "loss": 4.387514591217041, "l_mix": 2.72012060880661, "l_a": 2.8010209798812866, "l_c": 2.756958693265915, "z_loss": 46.302133083343506, "lr": 0.000936536009537381, "wd": 0.01, "grad_norm": 0.13227355480194092, "tokens_per_second": 1449799.2095823132, "eta_seconds": 1731.8365297794342, "data_wait_seconds": 2.7388334274291992e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 67700, "total_steps": 72479, "tokens": 35494297600, "loss": 4.6596673130989075, "l_mix": 2.891542285680771, "l_a": 2.964709520339966, "l_c": 2.9290400445461273, "z_loss": 44.96917104721069, "lr": 0.0009332709941635433, "wd": 0.01, "grad_norm": 0.11132992058992386, "tokens_per_second": 1408033.6180011258, "eta_seconds": 1779.4833304882047, "data_wait_seconds": 2.2172927856445312e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.16554841, "gate_a2b/L1_absmax": 0.6735909, "gate_a2b/L1_rms": 0.17485404, "gate_a2b/L3_absmean": 0.15508945, "gate_a2b/L3_absmax": 0.50512773, "gate_a2b/L3_rms": 0.16059279, "gate_a2b/L5_absmean": 0.22313447, "gate_a2b/L5_absmax": 0.73306274, "gate_a2b/L5_rms": 0.22906005, "gate_c2b/L1_absmean": 0.0451031, "gate_c2b/L1_absmax": 0.60546178, "gate_c2b/L1_rms": 0.07170493, "gate_c2b/L3_absmean": 0.16397166, "gate_c2b/L3_absmax": 0.3875564, "gate_c2b/L3_rms": 0.1674456, "gate_c2b/L5_absmean": 0.46923429, "gate_c2b/L5_absmax": 0.68119735, "gate_c2b/L5_rms": 0.47807863, "gate_b2a/L7_absmean": 0.05211939, "gate_b2a/L7_absmax": 0.33448014, "gate_b2a/L7_rms": 0.05779039, "gate_b2a/L11_absmean": 0.08555743, "gate_b2a/L11_absmax": 0.61416149, "gate_b2a/L11_rms": 0.09445061, "gate_b2a/L14_absmean": 0.22166868, "gate_b2a/L14_absmax": 0.79810178, "gate_b2a/L14_rms": 0.23029374, "gate_b2c/L3_absmean": 0.03849992, "gate_b2c/L3_absmax": 0.5508939, "gate_b2c/L3_rms": 0.04826542, "gate_b2c/L5_absmean": 0.03939252, "gate_b2c/L5_absmax": 0.53440356, "gate_b2c/L5_rms": 0.04821039, "gate_b2c/L6_absmean": 0.05699251, "gate_b2c/L6_absmax": 0.3332729, "gate_b2c/L6_rms": 0.06242702}, "streams": {"cos_a2b/round0": 0.352647, "cos_c2b/round0": 0.232107, "cos_b2a/round0": 0.174597, "cos_b2c/round0": 0.152053, "rms_a/round0": 7.077246, "rms_b/round0": 12.387526, "rms_c/round0": 4.84912, "cos_a2b/round1": 0.411913, "cos_c2b/round1": 0.314431, "cos_b2a/round1": 0.2631, "cos_b2c/round1": 0.110152, "rms_a/round1": 13.975072, "rms_b/round1": 28.978607, "rms_c/round1": 10.564158, "cos_a2b/round2": 0.550972, "cos_c2b/round2": 0.516159, "cos_b2a/round2": 0.102668, "cos_b2c/round2": 0.189799, "rms_a/round2": 42.336216, "rms_b/round2": 81.265656, "rms_c/round2": 20.895847}, "alpha_mean": 0.4521725025204937, "alpha_hist": [0.00645, 0.05655, 0.15369, 0.18735, 0.19044, 0.18919, 0.13593, 0.07076, 0.00917, 0.00047], "alpha_by_type": {"special": {"share": 0.00056, "mean": 0.50377, "std": 0.17589}, "whitespace": {"share": 0.04224, "mean": 0.55568, "std": 0.12546}, "digit": {"share": 0.02533, "mean": 0.46969, "std": 0.11631}, "word_start": {"share": 0.57721, "mean": 0.46855, "std": 0.16043}, "word_cont": {"share": 0.25569, "mean": 0.34149, "std": 0.13353}, "punct": {"share": 0.09897, "mean": 0.59366, "std": 0.15377}}, "alpha_p05": 0.1833, "alpha_p50": 0.4552, "alpha_p95": 0.72739}
{"step": 67710, "total_steps": 72479, "tokens": 35499540480, "loss": 4.560203194618225, "l_mix": 2.829818457365036, "l_a": 2.9018226265907288, "l_c": 2.866126447916031, "z_loss": 45.94607591629028, "lr": 0.0009300077891182275, "wd": 0.01, "grad_norm": 0.11142182350158691, "tokens_per_second": 1413592.6102299697, "eta_seconds": 1768.7765583276748, "data_wait_seconds": 1.5527009963989258e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 67720, "total_steps": 72479, "tokens": 35504783360, "loss": 4.640563070774078, "l_mix": 2.879710614681244, "l_a": 2.953405022621155, "l_c": 2.916102796792984, "z_loss": 46.084372997283936, "lr": 0.0009267464216588516, "wd": 0.01, "grad_norm": 0.12495005130767822, "tokens_per_second": 1402650.6663394063, "eta_seconds": 1778.8367780208587, "data_wait_seconds": 1.6182661056518555e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 67730, "total_steps": 72479, "tokens": 35510026240, "loss": 4.653775691986084, "l_mix": 2.8882730305194855, "l_a": 2.9616303145885468, "l_c": 2.923378497362137, "z_loss": 44.57849359512329, "lr": 0.0009234869190274847, "wd": 0.01, "grad_norm": 0.10166435688734055, "tokens_per_second": 1367766.7227629616, "eta_seconds": 1820.3716105699539, "data_wait_seconds": 1.823902130126953e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 67740, "total_steps": 72479, "tokens": 35515269120, "loss": 4.6380526423454285, "l_mix": 2.8782066702842712, "l_a": 2.952965945005417, "l_c": 2.9131873548030853, "z_loss": 45.481224060058594, "lr": 0.0009202293084506183, "wd": 0.01, "grad_norm": 0.11285221576690674, "tokens_per_second": 1427301.7516046104, "eta_seconds": 1740.7677312850951, "data_wait_seconds": 1.7464160919189453e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 67750, "total_steps": 72479, "tokens": 35520512000, "loss": 2.8595517575740814, "l_mix": 1.7708306312561035, "l_a": 1.8329570293426514, "l_c": 1.796113207936287, "z_loss": 59.028635025024414, "lr": 0.000916973617138941, "wd": 0.01, "grad_norm": 0.1454477161169052, "tokens_per_second": 1334315.373936867, "eta_seconds": 1858.149880027771, "data_wait_seconds": 2.321600914001465e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 67760, "total_steps": 72479, "tokens": 35525754880, "loss": 4.282591283321381, "l_mix": 2.654508799314499, "l_a": 2.73914298415184, "l_c": 2.6877982020378113, "z_loss": 45.85509538650513, "lr": 0.000913719872287109, "wd": 0.01, "grad_norm": 0.12813349068164825, "tokens_per_second": 1340524.451969346, "eta_seconds": 1845.6321840047838, "data_wait_seconds": 2.2202730178833008e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 67770, "total_steps": 72479, "tokens": 35530997760, "loss": 4.38252329826355, "l_mix": 2.7170311510562897, "l_a": 2.799269378185272, "l_c": 2.752370595932007, "z_loss": 45.98960590362549, "lr": 0.0009104681010735201, "wd": 0.01, "grad_norm": 0.11528743803501129, "tokens_per_second": 1409336.4092423548, "eta_seconds": 1751.7976373910903, "data_wait_seconds": 3.361701965332031e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 67780, "total_steps": 72479, "tokens": 35536240640, "loss": 4.385509192943573, "l_mix": 2.71878319978714, "l_a": 2.8010251820087433, "l_c": 2.754728317260742, "z_loss": 46.896310806274414, "lr": 0.0009072183306600863, "wd": 0.01, "grad_norm": 0.11334118247032166, "tokens_per_second": 1432923.0066173105, "eta_seconds": 1719.3033405303956, "data_wait_seconds": 1.9252300262451172e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 67790, "total_steps": 72479, "tokens": 35541483520, "loss": 4.356981337070465, "l_mix": 2.701171398162842, "l_a": 2.7831994891166687, "l_c": 2.736166924238205, "z_loss": 47.60661792755127, "lr": 0.0009039705881920062, "wd": 0.01, "grad_norm": 0.13162824511528015, "tokens_per_second": 1473443.4267040337, "eta_seconds": 1668.46340174675, "data_wait_seconds": 1.9431114196777344e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 67800, "total_steps": 72479, "tokens": 35546726400, "loss": 4.423009514808655, "l_mix": 2.741979330778122, "l_a": 2.8253622949123383, "l_c": 2.778071314096451, "z_loss": 46.06713819503784, "lr": 0.0009007249007975409, "wd": 0.01, "grad_norm": 0.12140634655952454, "tokens_per_second": 1439085.8894520176, "eta_seconds": 1704.6540237665179, "data_wait_seconds": 1.5616416931152344e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.16589952, "gate_a2b/L1_absmax": 0.67328948, "gate_a2b/L1_rms": 0.17523071, "gate_a2b/L3_absmean": 0.15541983, "gate_a2b/L3_absmax": 0.50406927, "gate_a2b/L3_rms": 0.16089077, "gate_a2b/L5_absmean": 0.2232441, "gate_a2b/L5_absmax": 0.73389882, "gate_a2b/L5_rms": 0.22910503, "gate_c2b/L1_absmean": 0.04535428, "gate_c2b/L1_absmax": 0.60330987, "gate_c2b/L1_rms": 0.07176937, "gate_c2b/L3_absmean": 0.16360974, "gate_c2b/L3_absmax": 0.38650134, "gate_c2b/L3_rms": 0.16705686, "gate_c2b/L5_absmean": 0.46917707, "gate_c2b/L5_absmax": 0.6820547, "gate_c2b/L5_rms": 0.47799253, "gate_b2a/L7_absmean": 0.05224708, "gate_b2a/L7_absmax": 0.33439291, "gate_b2a/L7_rms": 0.05789343, "gate_b2a/L11_absmean": 0.08558813, "gate_b2a/L11_absmax": 0.61294389, "gate_b2a/L11_rms": 0.09443329, "gate_b2a/L14_absmean": 0.22179221, "gate_b2a/L14_absmax": 0.79740667, "gate_b2a/L14_rms": 0.23041467, "gate_b2c/L3_absmean": 0.03866781, "gate_b2c/L3_absmax": 0.551247, "gate_b2c/L3_rms": 0.04839811, "gate_b2c/L5_absmean": 0.03935672, "gate_b2c/L5_absmax": 0.53510588, "gate_b2c/L5_rms": 0.0481866, "gate_b2c/L6_absmean": 0.05708558, "gate_b2c/L6_absmax": 0.33334711, "gate_b2c/L6_rms": 0.06248432}, "streams": {"cos_a2b/round0": 0.355766, "cos_c2b/round0": 0.236577, "cos_b2a/round0": 0.174345, "cos_b2c/round0": 0.152644, "rms_a/round0": 7.08334, "rms_b/round0": 12.344076, "rms_c/round0": 4.904536, "cos_a2b/round1": 0.415652, "cos_c2b/round1": 0.308634, "cos_b2a/round1": 0.264043, "cos_b2c/round1": 0.105954, "rms_a/round1": 14.148875, "rms_b/round1": 29.393908, "rms_c/round1": 10.603659, "cos_a2b/round2": 0.537741, "cos_c2b/round2": 0.526704, "cos_b2a/round2": 0.095955, "cos_b2c/round2": 0.18323, "rms_a/round2": 42.471836, "rms_b/round2": 83.002541, "rms_c/round2": 21.184109}, "alpha_mean": 0.41382799030459694, "alpha_hist": [0.0153, 0.10011, 0.19872, 0.19405, 0.16525, 0.14241, 0.11182, 0.06367, 0.00847, 0.0002], "alpha_by_type": {"special": {"share": 0.0009, "mean": 0.57709, "std": 0.10958}, "whitespace": {"share": 0.02184, "mean": 0.51265, "std": 0.12415}, "digit": {"share": 0.02116, "mean": 0.4562, "std": 0.11075}, "word_start": {"share": 0.60252, "mean": 0.43383, "std": 0.17309}, "word_cont": {"share": 0.26059, "mean": 0.29781, "std": 0.13173}, "punct": {"share": 0.09299, "mean": 0.57489, "std": 0.16625}}, "alpha_p05": 0.16212, "alpha_p50": 0.4104, "alpha_p95": 0.7299}
{"step": 67810, "total_steps": 72479, "tokens": 35551969280, "loss": 3.2791803181171417, "l_mix": 2.0336380004882812, "l_a": 2.0938110947608948, "l_c": 2.05799663066864, "z_loss": 55.08231830596924, "lr": 0.0008974812955877838, "wd": 0.01, "grad_norm": 0.12901127338409424, "tokens_per_second": 1418074.470984175, "eta_seconds": 1726.2144704580307, "data_wait_seconds": 3.454089164733887e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 67820, "total_steps": 72479, "tokens": 35557212160, "loss": 4.613259136676788, "l_mix": 2.863466799259186, "l_a": 2.93387570977211, "l_c": 2.89876526594162, "z_loss": 45.645822525024414, "lr": 0.0008942397996564369, "wd": 0.01, "grad_norm": 0.10405932366847992, "tokens_per_second": 1389101.157807045, "eta_seconds": 1758.4448607444763, "data_wait_seconds": 1.5974044799804688e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 67830, "total_steps": 72479, "tokens": 35562455040, "loss": 4.656801283359528, "l_mix": 2.8901273608207703, "l_a": 2.965109884738922, "l_c": 2.9238028526306152, "z_loss": 44.3313946723938, "lr": 0.0008910004400795828, "wd": 0.01, "grad_norm": 0.13233087956905365, "tokens_per_second": 1443941.5836608214, "eta_seconds": 1688.0287537813185, "data_wait_seconds": 1.6182661056518555e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 67840, "total_steps": 72479, "tokens": 35567697920, "loss": 4.602442264556885, "l_mix": 2.8560956716537476, "l_a": 2.929302394390106, "l_c": 2.891852468252182, "z_loss": 45.191330432891846, "lr": 0.0008877632439154592, "wd": 0.01, "grad_norm": 0.10410066694021225, "tokens_per_second": 1453177.263846158, "eta_seconds": 1673.6925993204118, "data_wait_seconds": 2.1904706954956055e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 67850, "total_steps": 72479, "tokens": 35572940800, "loss": 4.370977014303207, "l_mix": 2.7124095261096954, "l_a": 2.781638741493225, "l_c": 2.7469191551208496, "z_loss": 46.703654289245605, "lr": 0.0008845282382042342, "wd": 0.01, "grad_norm": 0.1011575236916542, "tokens_per_second": 1441268.3496308322, "eta_seconds": 1683.8843041419982, "data_wait_seconds": 2.059340476989746e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 67860, "total_steps": 72479, "tokens": 35578183680, "loss": 4.642114877700806, "l_mix": 2.8807722628116608, "l_a": 2.9551979303359985, "l_c": 2.915944129228592, "z_loss": 45.809523582458496, "lr": 0.0008812954499677776, "wd": 0.01, "grad_norm": 0.10116288810968399, "tokens_per_second": 1398243.0365829668, "eta_seconds": 1731.9494598865508, "data_wait_seconds": 2.059340476989746e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 67870, "total_steps": 72479, "tokens": 35583426560, "loss": 4.4447484612464905, "l_mix": 2.7562998235225677, "l_a": 2.8358074128627777, "l_c": 2.7923543751239777, "z_loss": 47.10161876678467, "lr": 0.0008780649062094376, "wd": 0.01, "grad_norm": 0.09981575608253479, "tokens_per_second": 1374441.2480719588, "eta_seconds": 1758.1278176784515, "data_wait_seconds": 1.7702579498291016e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 67880, "total_steps": 72479, "tokens": 35588669440, "loss": 4.34721964597702, "l_mix": 2.694876939058304, "l_a": 2.7776187360286713, "l_c": 2.7301900386810303, "z_loss": 46.57820510864258, "lr": 0.000874836633913814, "wd": 0.01, "grad_norm": 0.08356059342622757, "tokens_per_second": 1454088.5844355978, "eta_seconds": 1658.2211962938309, "data_wait_seconds": 1.7702579498291016e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 67890, "total_steps": 72479, "tokens": 35593912320, "loss": 4.468391299247742, "l_mix": 2.77163827419281, "l_a": 2.8496852815151215, "l_c": 2.806158035993576, "z_loss": 45.72683572769165, "lr": 0.0008716106600465326, "wd": 0.01, "grad_norm": 0.10592499375343323, "tokens_per_second": 1343413.8537505134, "eta_seconds": 1790.9281084775926, "data_wait_seconds": 1.6242265701293945e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 67900, "total_steps": 72479, "tokens": 35599155200, "loss": 4.3308839201927185, "l_mix": 2.684723377227783, "l_a": 2.7667962312698364, "l_c": 2.7204052209854126, "z_loss": 47.20203638076782, "lr": 0.0008683870115540223, "wd": 0.01, "grad_norm": 0.09587696194648743, "tokens_per_second": 1454540.825071124, "eta_seconds": 1650.4966451406478, "data_wait_seconds": 2.250075340270996e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16600835, "gate_a2b/L1_absmax": 0.67342716, "gate_a2b/L1_rms": 0.17535418, "gate_a2b/L3_absmean": 0.15533857, "gate_a2b/L3_absmax": 0.50456691, "gate_a2b/L3_rms": 0.16083676, "gate_a2b/L5_absmean": 0.22323148, "gate_a2b/L5_absmax": 0.73392516, "gate_a2b/L5_rms": 0.22913548, "gate_c2b/L1_absmean": 0.04563348, "gate_c2b/L1_absmax": 0.60223621, "gate_c2b/L1_rms": 0.07179756, "gate_c2b/L3_absmean": 0.16377872, "gate_c2b/L3_absmax": 0.38685483, "gate_c2b/L3_rms": 0.16720533, "gate_c2b/L5_absmean": 0.46901613, "gate_c2b/L5_absmax": 0.68225479, "gate_c2b/L5_rms": 0.47783622, "gate_b2a/L7_absmean": 0.05229476, "gate_b2a/L7_absmax": 0.33508071, "gate_b2a/L7_rms": 0.05794358, "gate_b2a/L11_absmean": 0.08566529, "gate_b2a/L11_absmax": 0.61401182, "gate_b2a/L11_rms": 0.09455079, "gate_b2a/L14_absmean": 0.22154292, "gate_b2a/L14_absmax": 0.79918122, "gate_b2a/L14_rms": 0.23020841, "gate_b2c/L3_absmean": 0.03874583, "gate_b2c/L3_absmax": 0.55133063, "gate_b2c/L3_rms": 0.04846597, "gate_b2c/L5_absmean": 0.03946096, "gate_b2c/L5_absmax": 0.53552586, "gate_b2c/L5_rms": 0.04828796, "gate_b2c/L6_absmean": 0.05711794, "gate_b2c/L6_absmax": 0.33302432, "gate_b2c/L6_rms": 0.06253676}, "streams": {"cos_a2b/round0": 0.353503, "cos_c2b/round0": 0.233156, "cos_b2a/round0": 0.173172, "cos_b2c/round0": 0.153432, "rms_a/round0": 7.109619, "rms_b/round0": 12.432331, "rms_c/round0": 4.851305, "cos_a2b/round1": 0.411991, "cos_c2b/round1": 0.309774, "cos_b2a/round1": 0.259154, "cos_b2c/round1": 0.109139, "rms_a/round1": 14.173352, "rms_b/round1": 29.290905, "rms_c/round1": 10.576379, "cos_a2b/round2": 0.547548, "cos_c2b/round2": 0.513258, "cos_b2a/round2": 0.107429, "cos_b2c/round2": 0.18624, "rms_a/round2": 42.984295, "rms_b/round2": 82.174911, "rms_c/round2": 20.979063}, "alpha_mean": 0.4502893765282945, "alpha_hist": [0.01109, 0.07346, 0.15721, 0.17545, 0.17137, 0.17715, 0.13768, 0.08183, 0.01456, 0.0002], "alpha_by_type": {"special": {"share": 0.00107, "mean": 0.56588, "std": 0.12634}, "whitespace": {"share": 0.02309, "mean": 0.52797, "std": 0.12121}, "digit": {"share": 0.02719, "mean": 0.46014, "std": 0.11869}, "word_start": {"share": 0.59319, "mean": 0.47343, "std": 0.16895}, "word_cont": {"share": 0.24559, "mean": 0.31858, "std": 0.1398}, "punct": {"share": 0.10988, "mean": 0.59986, "std": 0.16375}}, "alpha_p05": 0.16886, "alpha_p50": 0.45, "alpha_p95": 0.73182}
{"step": 67910, "total_steps": 72479, "tokens": 35604398080, "loss": 4.299370765686035, "l_mix": 2.665175825357437, "l_a": 2.747537761926651, "l_c": 2.6997784674167633, "z_loss": 46.76211500167847, "lr": 0.0008651657153632866, "wd": 0.01, "grad_norm": 0.09721935540437698, "tokens_per_second": 1475884.6459364004, "eta_seconds": 1623.0752712249755, "data_wait_seconds": 1.4573335647583008e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 67920, "total_steps": 72479, "tokens": 35609640960, "loss": 2.942106455564499, "l_mix": 1.8224976658821106, "l_a": 1.8859635442495346, "l_c": 1.8460655957460403, "z_loss": 58.062254905700684, "lr": 0.0008619467983816808, "wd": 0.01, "grad_norm": 0.13337984681129456, "tokens_per_second": 1466881.0613195826, "eta_seconds": 1629.4633934736253, "data_wait_seconds": 2.136826515197754e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 67930, "total_steps": 72479, "tokens": 35614883840, "loss": 2.8379657864570618, "l_mix": 1.75748710334301, "l_a": 1.8199432641267776, "l_c": 1.7816522270441055, "z_loss": 58.4347448348999, "lr": 0.0008587302874966868, "wd": 0.01, "grad_norm": 0.11141312122344971, "tokens_per_second": 1452398.301360544, "eta_seconds": 1642.101970076561, "data_wait_seconds": 1.9371509552001953e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 67940, "total_steps": 72479, "tokens": 35620126720, "loss": 4.723585069179535, "l_mix": 2.931925058364868, "l_a": 3.005182534456253, "l_c": 2.967017263174057, "z_loss": 44.45353364944458, "lr": 0.0008555162095756879, "wd": 0.01, "grad_norm": 0.11638869345188141, "tokens_per_second": 1378940.9015331229, "eta_seconds": 1725.776086091995, "data_wait_seconds": 1.6897916793823242e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 67950, "total_steps": 72479, "tokens": 35625369600, "loss": 4.589203655719757, "l_mix": 2.848024159669876, "l_a": 2.921134203672409, "l_c": 2.882797032594681, "z_loss": 44.82003307342529, "lr": 0.0008523045914657466, "wd": 0.01, "grad_norm": 0.10596590489149094, "tokens_per_second": 1384486.7697929463, "eta_seconds": 1715.0762317180631, "data_wait_seconds": 2.1785497665405273e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 67960, "total_steps": 72479, "tokens": 35630612480, "loss": 4.638144373893738, "l_mix": 2.8788313269615173, "l_a": 2.9491343200206757, "l_c": 2.915242463350296, "z_loss": 44.88467454910278, "lr": 0.000849095459993377, "wd": 0.01, "grad_norm": 0.11583005636930466, "tokens_per_second": 1424905.5535751372, "eta_seconds": 1662.747026324272, "data_wait_seconds": 1.7762184143066406e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 67970, "total_steps": 72479, "tokens": 35635855360, "loss": 4.629978716373444, "l_mix": 2.872990310192108, "l_a": 2.9485364854335785, "l_c": 2.9080914855003357, "z_loss": 46.09273290634155, "lr": 0.0008458888419643232, "wd": 0.01, "grad_norm": 0.09988868981599808, "tokens_per_second": 1431549.7039034478, "eta_seconds": 1651.3674555301666, "data_wait_seconds": 2.3365020751953125e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 67980, "total_steps": 72479, "tokens": 35641098240, "loss": 4.271940529346466, "l_mix": 2.6483405232429504, "l_a": 2.729650765657425, "l_c": 2.682349354028702, "z_loss": 46.91256237030029, "lr": 0.0008426847641633345, "wd": 0.01, "grad_norm": 0.11089310795068741, "tokens_per_second": 1442976.9339735121, "eta_seconds": 1634.6565606594086, "data_wait_seconds": 1.7851591110229492e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 67990, "total_steps": 72479, "tokens": 35646341120, "loss": 4.3200642466545105, "l_mix": 2.678095191717148, "l_a": 2.760372281074524, "l_c": 2.7128575444221497, "z_loss": 47.14190626144409, "lr": 0.000839483253353941, "wd": 0.01, "grad_norm": 0.09182901680469513, "tokens_per_second": 1474243.2553092141, "eta_seconds": 1596.4318124055862, "data_wait_seconds": 1.9371509552001953e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 68000, "total_steps": 72479, "tokens": 35651584000, "loss": 4.399939775466919, "l_mix": 2.727855682373047, "l_a": 2.8108878135681152, "l_c": 2.762725740671158, "z_loss": 46.65603733062744, "lr": 0.0008362843362782329, "wd": 0.01, "grad_norm": 0.10432681441307068, "tokens_per_second": 1386571.1690651237, "eta_seconds": 1693.5920812368392, "data_wait_seconds": 2.5898218154907227e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16651689, "gate_a2b/L1_absmax": 0.67330885, "gate_a2b/L1_rms": 0.17585388, "gate_a2b/L3_absmean": 0.15561913, "gate_a2b/L3_absmax": 0.50644732, "gate_a2b/L3_rms": 0.16109455, "gate_a2b/L5_absmean": 0.22330235, "gate_a2b/L5_absmax": 0.73412001, "gate_a2b/L5_rms": 0.22922549, "gate_c2b/L1_absmean": 0.04559934, "gate_c2b/L1_absmax": 0.60202056, "gate_c2b/L1_rms": 0.07172763, "gate_c2b/L3_absmean": 0.16359378, "gate_c2b/L3_absmax": 0.3869752, "gate_c2b/L3_rms": 0.16708143, "gate_c2b/L5_absmean": 0.46925232, "gate_c2b/L5_absmax": 0.68231875, "gate_c2b/L5_rms": 0.47807848, "gate_b2a/L7_absmean": 0.05229, "gate_b2a/L7_absmax": 0.33487496, "gate_b2a/L7_rms": 0.0579514, "gate_b2a/L11_absmean": 0.08576109, "gate_b2a/L11_absmax": 0.61309868, "gate_b2a/L11_rms": 0.09460294, "gate_b2a/L14_absmean": 0.22192797, "gate_b2a/L14_absmax": 0.79759687, "gate_b2a/L14_rms": 0.23054993, "gate_b2c/L3_absmean": 0.03883431, "gate_b2c/L3_absmax": 0.5519706, "gate_b2c/L3_rms": 0.0485603, "gate_b2c/L5_absmean": 0.03956979, "gate_b2c/L5_absmax": 0.53522336, "gate_b2c/L5_rms": 0.04835568, "gate_b2c/L6_absmean": 0.05720162, "gate_b2c/L6_absmax": 0.3322061, "gate_b2c/L6_rms": 0.06259381}, "streams": {"cos_a2b/round0": 0.353848, "cos_c2b/round0": 0.237709, "cos_b2a/round0": 0.171734, "cos_b2c/round0": 0.152831, "rms_a/round0": 7.088381, "rms_b/round0": 12.308918, "rms_c/round0": 4.83733, "cos_a2b/round1": 0.41455, "cos_c2b/round1": 0.306337, "cos_b2a/round1": 0.258011, "cos_b2c/round1": 0.106205, "rms_a/round1": 14.070786, "rms_b/round1": 29.317408, "rms_c/round1": 10.54255, "cos_a2b/round2": 0.538013, "cos_c2b/round2": 0.520754, "cos_b2a/round2": 0.108718, "cos_b2c/round2": 0.182933, "rms_a/round2": 42.331203, "rms_b/round2": 82.713882, "rms_c/round2": 21.046892}, "alpha_mean": 0.4322847198340547, "alpha_hist": [0.0106, 0.07558, 0.17104, 0.20917, 0.18724, 0.14877, 0.11357, 0.07127, 0.0126, 0.00015], "alpha_by_type": {"special": {"share": 0.00095, "mean": 0.55224, "std": 0.11957}, "whitespace": {"share": 0.02643, "mean": 0.48102, "std": 0.12246}, "digit": {"share": 0.02602, "mean": 0.44044, "std": 0.11381}, "word_start": {"share": 0.57022, "mean": 0.45286, "std": 0.17202}, "word_cont": {"share": 0.2621, "mean": 0.32411, "std": 0.13147}, "punct": {"share": 0.11429, "mean": 0.5636, "std": 0.16629}}, "alpha_p05": 0.17781, "alpha_p50": 0.41632, "alpha_p95": 0.75128}
{"step": 68010, "total_steps": 72479, "tokens": 35656826880, "loss": 4.343719005584717, "l_mix": 2.6930454075336456, "l_a": 2.7757956981658936, "l_c": 2.7264495491981506, "z_loss": 46.457321643829346, "lr": 0.0008330880396566325, "wd": 0.01, "grad_norm": 0.09737341105937958, "tokens_per_second": 1366919.7454734084, "eta_seconds": 1714.1043428182602, "data_wait_seconds": 1.671910285949707e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 68020, "total_steps": 72479, "tokens": 35662069760, "loss": 4.3770472407341, "l_mix": 2.7135717570781708, "l_a": 2.7972271144390106, "l_c": 2.747691124677658, "z_loss": 47.228248596191406, "lr": 0.0008298943901876756, "wd": 0.01, "grad_norm": 0.11347949504852295, "tokens_per_second": 1394925.5413570115, "eta_seconds": 1675.9318850278855, "data_wait_seconds": 2.130866050720215e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 68030, "total_steps": 72479, "tokens": 35667312640, "loss": 2.8456895649433136, "l_mix": 1.7617666274309158, "l_a": 1.8244222551584244, "l_c": 1.7886541932821274, "z_loss": 59.08873891830444, "lr": 0.0008267034145477852, "wd": 0.01, "grad_norm": 0.3263345956802368, "tokens_per_second": 1436835.2007650496, "eta_seconds": 1623.3993367910384, "data_wait_seconds": 3.606081008911133e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 68040, "total_steps": 72479, "tokens": 35672555520, "loss": 2.7387455701828003, "l_mix": 1.6952969431877136, "l_a": 1.7593254148960114, "l_c": 1.7188364416360855, "z_loss": 58.0337610244751, "lr": 0.0008235151393910496, "wd": 0.01, "grad_norm": 0.16379006206989288, "tokens_per_second": 1417658.784519046, "eta_seconds": 1641.660502099991, "data_wait_seconds": 2.613663673400879e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 68050, "total_steps": 72479, "tokens": 35677798400, "loss": 4.540255963802338, "l_mix": 2.8172968327999115, "l_a": 2.8886967301368713, "l_c": 2.8545002341270447, "z_loss": 45.70774173736572, "lr": 0.0008203295913490023, "wd": 0.01, "grad_norm": 0.09616722911596298, "tokens_per_second": 1380080.1225955072, "eta_seconds": 1682.562855577469, "data_wait_seconds": 2.3126602172851562e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 68060, "total_steps": 72479, "tokens": 35683041280, "loss": 4.643041491508484, "l_mix": 2.8816899359226227, "l_a": 2.9533770978450775, "l_c": 2.9177945256233215, "z_loss": 45.87203025817871, "lr": 0.0008171467970303946, "wd": 0.01, "grad_norm": 0.10740864276885986, "tokens_per_second": 1322407.831766005, "eta_seconds": 1751.9774281024934, "data_wait_seconds": 2.7239322662353516e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 68070, "total_steps": 72479, "tokens": 35688284160, "loss": 4.359310507774353, "l_mix": 2.702140837907791, "l_a": 2.7842688858509064, "l_c": 2.7396296560764313, "z_loss": 47.16802930831909, "lr": 0.0008139667830209778, "wd": 0.01, "grad_norm": 0.10541220009326935, "tokens_per_second": 1476788.776276858, "eta_seconds": 1565.2785483837129, "data_wait_seconds": 2.1219253540039062e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 68080, "total_steps": 72479, "tokens": 35693527040, "loss": 4.3146693110466, "l_mix": 2.67448890209198, "l_a": 2.7572480142116547, "l_c": 2.71001997590065, "z_loss": 46.876097202301025, "lr": 0.0008107895758832787, "wd": 0.01, "grad_norm": 0.09417261928319931, "tokens_per_second": 1446427.214928918, "eta_seconds": 1594.5101752758026, "data_wait_seconds": 2.2470951080322266e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 68090, "total_steps": 72479, "tokens": 35698769920, "loss": 4.392454087734222, "l_mix": 2.723340183496475, "l_a": 2.8030849993228912, "l_c": 2.7606275975704193, "z_loss": 46.90427875518799, "lr": 0.0008076152021563781, "wd": 0.01, "grad_norm": 0.09383874386548996, "tokens_per_second": 1476803.6528388984, "eta_seconds": 1558.1624731063844, "data_wait_seconds": 3.1441450119018555e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 68100, "total_steps": 72479, "tokens": 35704012800, "loss": 4.344129681587219, "l_mix": 2.6933261156082153, "l_a": 2.774738162755966, "l_c": 2.7279399931430817, "z_loss": 46.703675746917725, "lr": 0.0008044436883556912, "wd": 0.01, "grad_norm": 0.09492433071136475, "tokens_per_second": 1374390.909110909, "eta_seconds": 1670.454262161255, "data_wait_seconds": 1.6570091247558594e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.16603427, "gate_a2b/L1_absmax": 0.67578, "gate_a2b/L1_rms": 0.17542258, "gate_a2b/L3_absmean": 0.15545066, "gate_a2b/L3_absmax": 0.50648588, "gate_a2b/L3_rms": 0.16100602, "gate_a2b/L5_absmean": 0.22338, "gate_a2b/L5_absmax": 0.73754013, "gate_a2b/L5_rms": 0.22933359, "gate_c2b/L1_absmean": 0.04584635, "gate_c2b/L1_absmax": 0.60225552, "gate_c2b/L1_rms": 0.07175685, "gate_c2b/L3_absmean": 0.16398697, "gate_c2b/L3_absmax": 0.38713706, "gate_c2b/L3_rms": 0.16750024, "gate_c2b/L5_absmean": 0.46900901, "gate_c2b/L5_absmax": 0.68681622, "gate_c2b/L5_rms": 0.47779262, "gate_b2a/L7_absmean": 0.05246409, "gate_b2a/L7_absmax": 0.33686718, "gate_b2a/L7_rms": 0.05814989, "gate_b2a/L11_absmean": 0.0863039, "gate_b2a/L11_absmax": 0.61134946, "gate_b2a/L11_rms": 0.0950434, "gate_b2a/L14_absmean": 0.22214973, "gate_b2a/L14_absmax": 0.79660207, "gate_b2a/L14_rms": 0.23076779, "gate_b2c/L3_absmean": 0.03907194, "gate_b2c/L3_absmax": 0.55272281, "gate_b2c/L3_rms": 0.04875868, "gate_b2c/L5_absmean": 0.0394159, "gate_b2c/L5_absmax": 0.53649426, "gate_b2c/L5_rms": 0.04828868, "gate_b2c/L6_absmean": 0.05711875, "gate_b2c/L6_absmax": 0.33311674, "gate_b2c/L6_rms": 0.06257317}, "streams": {"cos_a2b/round0": 0.354731, "cos_c2b/round0": 0.236621, "cos_b2a/round0": 0.174329, "cos_b2c/round0": 0.153488, "rms_a/round0": 7.044934, "rms_b/round0": 12.221639, "rms_c/round0": 4.831391, "cos_a2b/round1": 0.411414, "cos_c2b/round1": 0.307257, "cos_b2a/round1": 0.257352, "cos_b2c/round1": 0.105911, "rms_a/round1": 14.016481, "rms_b/round1": 29.059597, "rms_c/round1": 10.474815, "cos_a2b/round2": 0.544513, "cos_c2b/round2": 0.514825, "cos_b2a/round2": 0.099788, "cos_b2c/round2": 0.179667, "rms_a/round2": 42.299038, "rms_b/round2": 81.960243, "rms_c/round2": 20.763504}, "alpha_mean": 0.4447083752503982, "alpha_hist": [0.01105, 0.06879, 0.16423, 0.19099, 0.17075, 0.17198, 0.13609, 0.07362, 0.01187, 0.00063], "alpha_by_type": {"special": {"share": 0.00101, "mean": 0.57736, "std": 0.14871}, "whitespace": {"share": 0.02863, "mean": 0.53159, "std": 0.15369}, "digit": {"share": 0.03261, "mean": 0.4994, "std": 0.13237}, "word_start": {"share": 0.5782, "mean": 0.46069, "std": 0.16779}, "word_cont": {"share": 0.25327, "mean": 0.32843, "std": 0.13833}, "punct": {"share": 0.10629, "mean": 0.5934, "std": 0.16038}}, "alpha_p05": 0.1538, "alpha_p50": 0.40592, "alpha_p95": 0.7237}
{"step": 68110, "total_steps": 72479, "tokens": 35709255680, "loss": 4.535133481025696, "l_mix": 2.8124872744083405, "l_a": 2.894293636083603, "l_c": 2.8478603065013885, "z_loss": 46.25044822692871, "lr": 0.0008012750609727421, "wd": 0.01, "grad_norm": 0.09688801318407059, "tokens_per_second": 1374029.0238548662, "eta_seconds": 1667.0785203456878, "data_wait_seconds": 1.582503318786621e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 68120, "total_steps": 72479, "tokens": 35714498560, "loss": 4.358065247535706, "l_mix": 2.7013794481754303, "l_a": 2.7860856652259827, "l_c": 2.7362000346183777, "z_loss": 48.24898147583008, "lr": 0.0007981093464749459, "wd": 0.01, "grad_norm": 0.11732252687215805, "tokens_per_second": 1441584.5853115031, "eta_seconds": 1585.3189714193345, "data_wait_seconds": 1.5914440155029297e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 68130, "total_steps": 72479, "tokens": 35719741440, "loss": 4.380832850933075, "l_mix": 2.715983361005783, "l_a": 2.7977904677391052, "l_c": 2.751707524061203, "z_loss": 46.94015884399414, "lr": 0.0007949465713053864, "wd": 0.01, "grad_norm": 0.10038904845714569, "tokens_per_second": 1421833.7268402448, "eta_seconds": 1603.65341527462, "data_wait_seconds": 1.7017126083374023e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 68140, "total_steps": 72479, "tokens": 35724984320, "loss": 2.747419089078903, "l_mix": 1.7011379450559616, "l_a": 1.7617509812116623, "l_c": 1.7258527427911758, "z_loss": 58.21788215637207, "lr": 0.0007917867618825947, "wd": 0.01, "grad_norm": 0.10649244487285614, "tokens_per_second": 1438922.9821523745, "eta_seconds": 1580.9641379117966, "data_wait_seconds": 2.2143125534057617e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 68150, "total_steps": 72479, "tokens": 35730227200, "loss": 2.8435210287570953, "l_mix": 1.7605004161596298, "l_a": 1.8241254687309265, "l_c": 1.7859431505203247, "z_loss": 58.027971267700195, "lr": 0.0007886299446003305, "wd": 0.01, "grad_norm": 0.11976294219493866, "tokens_per_second": 1347557.145287675, "eta_seconds": 1684.2645671367643, "data_wait_seconds": 1.7315149307250977e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 68160, "total_steps": 72479, "tokens": 35735470080, "loss": 3.0269347727298737, "l_mix": 1.8621642738580704, "l_a": 1.9739819318056107, "l_c": 1.9085862338542938, "z_loss": 51.87649154663086, "lr": 0.0007854761458273592, "wd": 0.01, "grad_norm": 0.10884436964988708, "tokens_per_second": 1376943.4205292265, "eta_seconds": 1644.5119227409364, "data_wait_seconds": 1.7940998077392578e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 68170, "total_steps": 72479, "tokens": 35740712960, "loss": 4.312911927700043, "l_mix": 2.673539251089096, "l_a": 2.756528317928314, "l_c": 2.70804700255394, "z_loss": 46.48653841018677, "lr": 0.0007823253919072325, "wd": 0.01, "grad_norm": 0.10619229078292847, "tokens_per_second": 1466722.8552436556, "eta_seconds": 1540.2753041744231, "data_wait_seconds": 1.4096498489379883e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 68180, "total_steps": 72479, "tokens": 35745955840, "loss": 4.423872411251068, "l_mix": 2.7433061003684998, "l_a": 2.823530614376068, "l_c": 2.778356909751892, "z_loss": 46.292606353759766, "lr": 0.0007791777091580694, "wd": 0.01, "grad_norm": 0.0897018313407898, "tokens_per_second": 1149016.9062814454, "eta_seconds": 1961.6022181034089, "data_wait_seconds": 2.524256706237793e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 68190, "total_steps": 72479, "tokens": 35751198720, "loss": 4.3614426255226135, "l_mix": 2.7042351961135864, "l_a": 2.7869236171245575, "l_c": 2.7371009588241577, "z_loss": 46.76038885116577, "lr": 0.0007760331238723352, "wd": 0.01, "grad_norm": 0.09611202776432037, "tokens_per_second": 1439102.3705685409, "eta_seconds": 1562.551266670227, "data_wait_seconds": 3.263354301452637e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 68200, "total_steps": 72479, "tokens": 35756441600, "loss": 4.473561763763428, "l_mix": 2.7743603885173798, "l_a": 2.8540008664131165, "l_c": 2.810003101825714, "z_loss": 46.50074100494385, "lr": 0.0007728916623166224, "wd": 0.01, "grad_norm": 0.0991431325674057, "tokens_per_second": 1320516.9018683166, "eta_seconds": 1698.9016564846038, "data_wait_seconds": 1.5020370483398438e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.16650164, "gate_a2b/L1_absmax": 0.67427993, "gate_a2b/L1_rms": 0.17578653, "gate_a2b/L3_absmean": 0.15581135, "gate_a2b/L3_absmax": 0.50593221, "gate_a2b/L3_rms": 0.1613043, "gate_a2b/L5_absmean": 0.22362852, "gate_a2b/L5_absmax": 0.73633289, "gate_a2b/L5_rms": 0.22953793, "gate_c2b/L1_absmean": 0.04588822, "gate_c2b/L1_absmax": 0.6023224, "gate_c2b/L1_rms": 0.0719322, "gate_c2b/L3_absmean": 0.16369377, "gate_c2b/L3_absmax": 0.38344905, "gate_c2b/L3_rms": 0.16713372, "gate_c2b/L5_absmean": 0.46913901, "gate_c2b/L5_absmax": 0.68573064, "gate_c2b/L5_rms": 0.47792786, "gate_b2a/L7_absmean": 0.05240134, "gate_b2a/L7_absmax": 0.33731732, "gate_b2a/L7_rms": 0.05810636, "gate_b2a/L11_absmean": 0.08603961, "gate_b2a/L11_absmax": 0.61025792, "gate_b2a/L11_rms": 0.0947834, "gate_b2a/L14_absmean": 0.22229639, "gate_b2a/L14_absmax": 0.79639459, "gate_b2a/L14_rms": 0.230887, "gate_b2c/L3_absmean": 0.03901875, "gate_b2c/L3_absmax": 0.55140537, "gate_b2c/L3_rms": 0.04866974, "gate_b2c/L5_absmean": 0.03972478, "gate_b2c/L5_absmax": 0.53489232, "gate_b2c/L5_rms": 0.04848295, "gate_b2c/L6_absmean": 0.05734393, "gate_b2c/L6_absmax": 0.33158997, "gate_b2c/L6_rms": 0.06273548}, "streams": {"cos_a2b/round0": 0.355532, "cos_c2b/round0": 0.233361, "cos_b2a/round0": 0.172545, "cos_b2c/round0": 0.153773, "rms_a/round0": 7.061142, "rms_b/round0": 12.401944, "rms_c/round0": 4.865207, "cos_a2b/round1": 0.409484, "cos_c2b/round1": 0.307331, "cos_b2a/round1": 0.26103, "cos_b2c/round1": 0.107783, "rms_a/round1": 14.153081, "rms_b/round1": 29.298077, "rms_c/round1": 10.709236, "cos_a2b/round2": 0.539212, "cos_c2b/round2": 0.51629, "cos_b2a/round2": 0.099131, "cos_b2c/round2": 0.182918, "rms_a/round2": 42.693157, "rms_b/round2": 82.504318, "rms_c/round2": 21.424086}, "alpha_mean": 0.43304805126593315, "alpha_hist": [0.01508, 0.08154, 0.17834, 0.18542, 0.17226, 0.15749, 0.12395, 0.07303, 0.01233, 0.00056], "alpha_by_type": {"special": {"share": 0.00154, "mean": 0.54268, "std": 0.13545}, "whitespace": {"share": 0.02911, "mean": 0.52903, "std": 0.1426}, "digit": {"share": 0.03993, "mean": 0.48121, "std": 0.13913}, "word_start": {"share": 0.56685, "mean": 0.45052, "std": 0.17267}, "word_cont": {"share": 0.26009, "mean": 0.31302, "std": 0.13756}, "punct": {"share": 0.10248, "mean": 0.59334, "std": 0.16268}}, "alpha_p05": 0.13987, "alpha_p50": 0.42322, "alpha_p95": 0.73867}
{"step": 68210, "total_steps": 72479, "tokens": 35761684480, "loss": 4.2721622586250305, "l_mix": 2.648045688867569, "l_a": 2.7293263375759125, "l_c": 2.6843952536582947, "z_loss": 47.50704050064087, "lr": 0.0007697533507314305, "wd": 0.01, "grad_norm": 0.1025095283985138, "tokens_per_second": 1468830.075892759, "eta_seconds": 1523.7878831148148, "data_wait_seconds": 2.065300941467285e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 68220, "total_steps": 72479, "tokens": 35766927360, "loss": 4.31294447183609, "l_mix": 2.674217998981476, "l_a": 2.754040002822876, "l_c": 2.7083812057971954, "z_loss": 46.97465372085571, "lr": 0.000766618215330948, "wd": 0.01, "grad_norm": 0.09815964847803116, "tokens_per_second": 1467443.3260575873, "eta_seconds": 1521.6550801992416, "data_wait_seconds": 1.5795230865478516e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 68230, "total_steps": 72479, "tokens": 35772170240, "loss": 4.626958072185516, "l_mix": 2.8722270727157593, "l_a": 2.94345960021019, "l_c": 2.9056437015533447, "z_loss": 44.443363189697266, "lr": 0.0007634862823028321, "wd": 0.01, "grad_norm": 0.0979878306388855, "tokens_per_second": 1443038.1041711562, "eta_seconds": 1543.7566794395448, "data_wait_seconds": 2.3633241653442383e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 68240, "total_steps": 72479, "tokens": 35777413120, "loss": 4.634568810462952, "l_mix": 2.8758276104927063, "l_a": 2.9506216943264008, "l_c": 2.9118484258651733, "z_loss": 45.208229064941406, "lr": 0.0007603575778079921, "wd": 0.01, "grad_norm": 0.11567866802215576, "tokens_per_second": 1361211.223486916, "eta_seconds": 1632.7053389310836, "data_wait_seconds": 3.045797348022461e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 68250, "total_steps": 72479, "tokens": 35782656000, "loss": 4.573005437850952, "l_mix": 2.8377083837985992, "l_a": 2.910884350538254, "l_c": 2.8734388947486877, "z_loss": 45.92214870452881, "lr": 0.0007572321279803686, "wd": 0.01, "grad_norm": 0.09895963966846466, "tokens_per_second": 1451517.1696044563, "eta_seconds": 1527.5147951602935, "data_wait_seconds": 1.609325408935547e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 68260, "total_steps": 72479, "tokens": 35787898880, "loss": 2.850999742746353, "l_mix": 1.7652819454669952, "l_a": 1.8293454051017761, "l_c": 1.7897140234708786, "z_loss": 58.37035369873047, "lr": 0.0007541099589267159, "wd": 0.01, "grad_norm": 0.10398567467927933, "tokens_per_second": 1475502.8889101665, "eta_seconds": 1499.1302888154983, "data_wait_seconds": 2.294778823852539e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 68270, "total_steps": 72479, "tokens": 35793141760, "loss": 3.0100724697113037, "l_mix": 1.8523739874362946, "l_a": 1.9611827433109283, "l_c": 1.8978119641542435, "z_loss": 51.457679748535156, "lr": 0.0007509910967263843, "wd": 0.01, "grad_norm": 0.09992362558841705, "tokens_per_second": 1455525.5311979894, "eta_seconds": 1516.1040769815445, "data_wait_seconds": 2.7477741241455078e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 68280, "total_steps": 72479, "tokens": 35798384640, "loss": 4.3346094489097595, "l_mix": 2.6869271993637085, "l_a": 2.769869416952133, "l_c": 2.722404658794403, "z_loss": 46.785752296447754, "lr": 0.0007478755674311022, "wd": 0.01, "grad_norm": 0.09825916588306427, "tokens_per_second": 1350691.7326882186, "eta_seconds": 1629.8947115182875, "data_wait_seconds": 1.6927719116210938e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 68290, "total_steps": 72479, "tokens": 35803627520, "loss": 4.338381409645081, "l_mix": 2.6896065771579742, "l_a": 2.771714597940445, "l_c": 2.7242011725902557, "z_loss": 46.38632678985596, "lr": 0.0007447633970647587, "wd": 0.01, "grad_norm": 0.09882551431655884, "tokens_per_second": 1462151.2329461544, "eta_seconds": 1502.0624286413192, "data_wait_seconds": 2.8699636459350586e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 68300, "total_steps": 72479, "tokens": 35808870400, "loss": 4.358072459697723, "l_mix": 2.7018821835517883, "l_a": 2.784122049808502, "l_c": 2.7365121245384216, "z_loss": 47.07070779800415, "lr": 0.0007416546116231855, "wd": 0.01, "grad_norm": 0.10035032778978348, "tokens_per_second": 1459378.3482653126, "eta_seconds": 1501.3238716363905, "data_wait_seconds": 1.4543533325195312e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16708079, "gate_a2b/L1_absmax": 0.6739586, "gate_a2b/L1_rms": 0.17630431, "gate_a2b/L3_absmean": 0.15602855, "gate_a2b/L3_absmax": 0.50582248, "gate_a2b/L3_rms": 0.16154079, "gate_a2b/L5_absmean": 0.22385001, "gate_a2b/L5_absmax": 0.73588389, "gate_a2b/L5_rms": 0.22974515, "gate_c2b/L1_absmean": 0.04584568, "gate_c2b/L1_absmax": 0.60161448, "gate_c2b/L1_rms": 0.07181165, "gate_c2b/L3_absmean": 0.16352184, "gate_c2b/L3_absmax": 0.38502428, "gate_c2b/L3_rms": 0.16700479, "gate_c2b/L5_absmean": 0.46912009, "gate_c2b/L5_absmax": 0.68579793, "gate_c2b/L5_rms": 0.47786781, "gate_b2a/L7_absmean": 0.05259093, "gate_b2a/L7_absmax": 0.33757204, "gate_b2a/L7_rms": 0.05827736, "gate_b2a/L11_absmean": 0.08599801, "gate_b2a/L11_absmax": 0.61057329, "gate_b2a/L11_rms": 0.09477814, "gate_b2a/L14_absmean": 0.2223715, "gate_b2a/L14_absmax": 0.79711807, "gate_b2a/L14_rms": 0.23097715, "gate_b2c/L3_absmean": 0.03927506, "gate_b2c/L3_absmax": 0.55142385, "gate_b2c/L3_rms": 0.04888536, "gate_b2c/L5_absmean": 0.03976114, "gate_b2c/L5_absmax": 0.53523064, "gate_b2c/L5_rms": 0.04850556, "gate_b2c/L6_absmean": 0.05744654, "gate_b2c/L6_absmax": 0.33223182, "gate_b2c/L6_rms": 0.06283281}, "streams": {"cos_a2b/round0": 0.35444, "cos_c2b/round0": 0.233633, "cos_b2a/round0": 0.172561, "cos_b2c/round0": 0.152798, "rms_a/round0": 7.002773, "rms_b/round0": 12.252284, "rms_c/round0": 4.839139, "cos_a2b/round1": 0.408024, "cos_c2b/round1": 0.30652, "cos_b2a/round1": 0.258229, "cos_b2c/round1": 0.106101, "rms_a/round1": 13.961354, "rms_b/round1": 29.231308, "rms_c/round1": 10.632565, "cos_a2b/round2": 0.534578, "cos_c2b/round2": 0.519938, "cos_b2a/round2": 0.100366, "cos_b2c/round2": 0.180545, "rms_a/round2": 42.348003, "rms_b/round2": 82.599915, "rms_c/round2": 21.165052}, "alpha_mean": 0.4283979156578397, "alpha_hist": [0.01634, 0.08301, 0.1778, 0.19083, 0.17963, 0.1524, 0.11984, 0.06958, 0.01044, 0.00014], "alpha_by_type": {"special": {"share": 0.00105, "mean": 0.55805, "std": 0.11773}, "whitespace": {"share": 0.02492, "mean": 0.51018, "std": 0.12393}, "digit": {"share": 0.0385, "mean": 0.45073, "std": 0.12743}, "word_start": {"share": 0.56972, "mean": 0.45163, "std": 0.16891}, "word_cont": {"share": 0.25975, "mean": 0.30107, "std": 0.13226}, "punct": {"share": 0.10606, "mean": 0.58684, "std": 0.16438}}, "alpha_p05": 0.16532, "alpha_p50": 0.41371, "alpha_p95": 0.7235}
{"step": 68310, "total_steps": 72479, "tokens": 35814113280, "loss": 4.337126791477203, "l_mix": 2.688164383172989, "l_a": 2.7713291347026825, "l_c": 2.7252119183540344, "z_loss": 48.096970081329346, "lr": 0.0007385492370739401, "wd": 0.01, "grad_norm": 0.12124668061733246, "tokens_per_second": 1222180.0691560972, "eta_seconds": 1788.4080481767653, "data_wait_seconds": 1.7523765563964844e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 68320, "total_steps": 72479, "tokens": 35819356160, "loss": 4.404762089252472, "l_mix": 2.7314237654209137, "l_a": 2.812265396118164, "l_c": 2.7655290067195892, "z_loss": 46.7471284866333, "lr": 0.0007354472993560893, "wd": 0.01, "grad_norm": 0.09547089040279388, "tokens_per_second": 1457925.9585327546, "eta_seconds": 1495.627249956131, "data_wait_seconds": 2.0623207092285156e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 68330, "total_steps": 72479, "tokens": 35824599040, "loss": 4.507254004478455, "l_mix": 2.7962206304073334, "l_a": 2.8723791241645813, "l_c": 2.831065148115158, "z_loss": 46.35220384597778, "lr": 0.0007323488243799914, "wd": 0.01, "grad_norm": 0.0907105877995491, "tokens_per_second": 1466453.6800609354, "eta_seconds": 1483.3546681880953, "data_wait_seconds": 1.6450881958007812e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 68340, "total_steps": 72479, "tokens": 35829841920, "loss": 4.553586483001709, "l_mix": 2.82563453912735, "l_a": 2.8983050286769867, "l_c": 2.861534297466278, "z_loss": 46.52228832244873, "lr": 0.000729253838027082, "wd": 0.01, "grad_norm": 0.0865062028169632, "tokens_per_second": 1444182.2600156001, "eta_seconds": 1502.599839425087, "data_wait_seconds": 2.0295381546020508e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 68350, "total_steps": 72479, "tokens": 35835084800, "loss": 4.499208033084869, "l_mix": 2.7918991148471832, "l_a": 2.8629716634750366, "l_c": 2.828057587146759, "z_loss": 46.53911113739014, "lr": 0.0007261623661496551, "wd": 0.01, "grad_norm": 0.097107894718647, "tokens_per_second": 1445060.014075879, "eta_seconds": 1498.0589947223664, "data_wait_seconds": 2.3484230041503906e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 68360, "total_steps": 72479, "tokens": 35840327680, "loss": 4.535170197486877, "l_mix": 2.814325273036957, "l_a": 2.8869617581367493, "l_c": 2.849187821149826, "z_loss": 45.94838523864746, "lr": 0.000723074434570649, "wd": 0.01, "grad_norm": 0.10933104902505875, "tokens_per_second": 1419103.627447355, "eta_seconds": 1521.7650284528734, "data_wait_seconds": 2.3573637008666992e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 68370, "total_steps": 72479, "tokens": 35845570560, "loss": 4.58298271894455, "l_mix": 2.84362930059433, "l_a": 2.9196481108665466, "l_c": 2.8781962990760803, "z_loss": 46.27436828613281, "lr": 0.0007199900690834297, "wd": 0.01, "grad_norm": 0.10337655246257782, "tokens_per_second": 1234876.7138587867, "eta_seconds": 1744.5461298465727, "data_wait_seconds": 2.16066837310791e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 68380, "total_steps": 72479, "tokens": 35850813440, "loss": 2.8672090768814087, "l_mix": 1.7758835554122925, "l_a": 1.83798848092556, "l_c": 1.7997632026672363, "z_loss": 59.30888891220093, "lr": 0.0007169092954515755, "wd": 0.01, "grad_norm": 0.125991553068161, "tokens_per_second": 1426941.099915319, "eta_seconds": 1506.0583174228668, "data_wait_seconds": 1.800060272216797e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 68390, "total_steps": 72479, "tokens": 35856056320, "loss": 4.327820658683777, "l_mix": 2.682992696762085, "l_a": 2.7653248608112335, "l_c": 2.717434674501419, "z_loss": 47.27526140213013, "lr": 0.0007138321394086635, "wd": 0.01, "grad_norm": 0.08955685794353485, "tokens_per_second": 1249183.1999258792, "eta_seconds": 1716.1723213434218, "data_wait_seconds": 2.41696834564209e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 68400, "total_steps": 72479, "tokens": 35861299200, "loss": 4.196863800287247, "l_mix": 2.6010630428791046, "l_a": 2.6857587695121765, "l_c": 2.6335769593715668, "z_loss": 47.9911789894104, "lr": 0.0007107586266580516, "wd": 0.01, "grad_norm": 0.09620614349842072, "tokens_per_second": 1207791.858828585, "eta_seconds": 1770.645112705231, "data_wait_seconds": 1.8417835235595703e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16698991, "gate_a2b/L1_absmax": 0.67288268, "gate_a2b/L1_rms": 0.17624895, "gate_a2b/L3_absmean": 0.15623844, "gate_a2b/L3_absmax": 0.50661111, "gate_a2b/L3_rms": 0.16173449, "gate_a2b/L5_absmean": 0.2239743, "gate_a2b/L5_absmax": 0.73611599, "gate_a2b/L5_rms": 0.22987929, "gate_c2b/L1_absmean": 0.04623481, "gate_c2b/L1_absmax": 0.60081172, "gate_c2b/L1_rms": 0.07193378, "gate_c2b/L3_absmean": 0.16341066, "gate_c2b/L3_absmax": 0.38518995, "gate_c2b/L3_rms": 0.16687581, "gate_c2b/L5_absmean": 0.46912786, "gate_c2b/L5_absmax": 0.6852656, "gate_c2b/L5_rms": 0.47791424, "gate_b2a/L7_absmean": 0.0526246, "gate_b2a/L7_absmax": 0.33831719, "gate_b2a/L7_rms": 0.05830404, "gate_b2a/L11_absmean": 0.08614767, "gate_b2a/L11_absmax": 0.6101771, "gate_b2a/L11_rms": 0.09487423, "gate_b2a/L14_absmean": 0.22266796, "gate_b2a/L14_absmax": 0.79523677, "gate_b2a/L14_rms": 0.23123395, "gate_b2c/L3_absmean": 0.03926441, "gate_b2c/L3_absmax": 0.55181044, "gate_b2c/L3_rms": 0.04888503, "gate_b2c/L5_absmean": 0.03986229, "gate_b2c/L5_absmax": 0.53678554, "gate_b2c/L5_rms": 0.04862401, "gate_b2c/L6_absmean": 0.05753576, "gate_b2c/L6_absmax": 0.33253217, "gate_b2c/L6_rms": 0.06290638}, "streams": {"cos_a2b/round0": 0.357751, "cos_c2b/round0": 0.240938, "cos_b2a/round0": 0.170108, "cos_b2c/round0": 0.154209, "rms_a/round0": 6.990511, "rms_b/round0": 12.18055, "rms_c/round0": 4.841293, "cos_a2b/round1": 0.40667, "cos_c2b/round1": 0.295694, "cos_b2a/round1": 0.25187, "cos_b2c/round1": 0.098981, "rms_a/round1": 14.075233, "rms_b/round1": 29.061102, "rms_c/round1": 10.560538, "cos_a2b/round2": 0.530671, "cos_c2b/round2": 0.52089, "cos_b2a/round2": 0.097783, "cos_b2c/round2": 0.173013, "rms_a/round2": 42.273846, "rms_b/round2": 82.694344, "rms_c/round2": 21.166182}, "alpha_mean": 0.43392653066407405, "alpha_hist": [0.01048, 0.0782, 0.18668, 0.18774, 0.17651, 0.15039, 0.1179, 0.07773, 0.01413, 0.00023], "alpha_by_type": {"special": {"share": 0.00095, "mean": 0.55661, "std": 0.12901}, "whitespace": {"share": 0.03304, "mean": 0.55345, "std": 0.12621}, "digit": {"share": 0.06593, "mean": 0.46687, "std": 0.1234}, "word_start": {"share": 0.51233, "mean": 0.44486, "std": 0.17394}, "word_cont": {"share": 0.25826, "mean": 0.30974, "std": 0.12754}, "punct": {"share": 0.1295, "mean": 0.59015, "std": 0.16458}}, "alpha_p05": 0.17696, "alpha_p50": 0.41206, "alpha_p95": 0.74055}
{"step": 68410, "total_steps": 72479, "tokens": 35866542080, "loss": 4.368965327739716, "l_mix": 2.708492189645767, "l_a": 2.792059749364853, "l_c": 2.7428502440452576, "z_loss": 46.822327613830566, "lr": 0.0007076887828726668, "wd": 0.01, "grad_norm": 0.11285870522260666, "tokens_per_second": 1374219.4750599614, "eta_seconds": 1552.3924021720884, "data_wait_seconds": 2.3663043975830078e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 68420, "total_steps": 72479, "tokens": 35871784960, "loss": 4.313167333602905, "l_mix": 2.6734988689422607, "l_a": 2.7581834495067596, "l_c": 2.7073777318000793, "z_loss": 47.29018688201904, "lr": 0.0007046226336947879, "wd": 0.01, "grad_norm": 0.0975574404001236, "tokens_per_second": 1052194.4436452722, "eta_seconds": 2022.5206518173218, "data_wait_seconds": 0.14116108417510986, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 68430, "total_steps": 72479, "tokens": 35877027840, "loss": 4.324116826057434, "l_mix": 2.681041121482849, "l_a": 2.7622412741184235, "l_c": 2.7146776914596558, "z_loss": 46.67236948013306, "lr": 0.0007015602047358336, "wd": 0.01, "grad_norm": 0.102692611515522, "tokens_per_second": 1481098.0185232956, "eta_seconds": 1433.2894146442413, "data_wait_seconds": 1.6182661056518555e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 68440, "total_steps": 72479, "tokens": 35882270720, "loss": 2.9986666440963745, "l_mix": 1.8449644148349762, "l_a": 1.9577346742153168, "l_c": 1.8879392594099045, "z_loss": 51.46911811828613, "lr": 0.0006985015215761486, "wd": 0.01, "grad_norm": 0.10221933573484421, "tokens_per_second": 1055288.2437535538, "eta_seconds": 2006.6548116445542, "data_wait_seconds": 4.3839216232299805e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 68450, "total_steps": 72479, "tokens": 35887513600, "loss": 2.9372538328170776, "l_mix": 1.8065754920244217, "l_a": 1.9188980162143707, "l_c": 1.8500296622514725, "z_loss": 52.49976110458374, "lr": 0.0006954466097647874, "wd": 0.01, "grad_norm": 0.09985030442476273, "tokens_per_second": 1403523.077276372, "eta_seconds": 1505.0385606050493, "data_wait_seconds": 1.7911195755004883e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 68460, "total_steps": 72479, "tokens": 35892756480, "loss": 2.9721318781375885, "l_mix": 1.8282280713319778, "l_a": 1.9403009116649628, "l_c": 1.8727115988731384, "z_loss": 52.505950927734375, "lr": 0.000692395494819304, "wd": 0.01, "grad_norm": 0.11211839318275452, "tokens_per_second": 1088823.6800910423, "eta_seconds": 1935.2201008558275, "data_wait_seconds": 1.5676021575927734e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 68470, "total_steps": 72479, "tokens": 35897999360, "loss": 4.2585868537425995, "l_mix": 2.641894042491913, "l_a": 2.714873343706131, "l_c": 2.6741023659706116, "z_loss": 47.421730518341064, "lr": 0.0006893482022255362, "wd": 0.01, "grad_norm": 0.1588485687971115, "tokens_per_second": 1471392.1668646792, "eta_seconds": 1428.4910843849182, "data_wait_seconds": 1.749396324157715e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 68480, "total_steps": 72479, "tokens": 35903242240, "loss": 4.335842549800873, "l_mix": 2.689665138721466, "l_a": 2.763662874698639, "l_c": 2.7235948741436005, "z_loss": 48.753135681152344, "lr": 0.0006863047574373944, "wd": 0.01, "grad_norm": 0.14303313195705414, "tokens_per_second": 1378354.6302554228, "eta_seconds": 1521.1090571165087, "data_wait_seconds": 2.8401613235473633e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 68490, "total_steps": 72479, "tokens": 35908485120, "loss": 2.741157650947571, "l_mix": 1.697661653161049, "l_a": 1.7562628984451294, "l_c": 1.7220569103956223, "z_loss": 58.46739149093628, "lr": 0.000683265185876649, "wd": 0.01, "grad_norm": 0.1345399022102356, "tokens_per_second": 1437607.5166086366, "eta_seconds": 1454.7675967454911, "data_wait_seconds": 2.9981136322021484e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 68500, "total_steps": 72479, "tokens": 35913728000, "loss": 4.329707682132721, "l_mix": 2.684188812971115, "l_a": 2.7675044238567352, "l_c": 2.7175582349300385, "z_loss": 46.86208152770996, "lr": 0.0006802295129327163, "wd": 0.01, "grad_norm": 0.10607445985078812, "tokens_per_second": 1455804.6837567547, "eta_seconds": 1432.9820306777954, "data_wait_seconds": 1.558661460876465e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16699886, "gate_a2b/L1_absmax": 0.67210352, "gate_a2b/L1_rms": 0.176245, "gate_a2b/L3_absmean": 0.15639015, "gate_a2b/L3_absmax": 0.50598174, "gate_a2b/L3_rms": 0.16186728, "gate_a2b/L5_absmean": 0.22401395, "gate_a2b/L5_absmax": 0.73741603, "gate_a2b/L5_rms": 0.22993372, "gate_c2b/L1_absmean": 0.04659518, "gate_c2b/L1_absmax": 0.59887666, "gate_c2b/L1_rms": 0.07211055, "gate_c2b/L3_absmean": 0.16341345, "gate_c2b/L3_absmax": 0.38464922, "gate_c2b/L3_rms": 0.16687502, "gate_c2b/L5_absmean": 0.46937567, "gate_c2b/L5_absmax": 0.68695152, "gate_c2b/L5_rms": 0.4781152, "gate_b2a/L7_absmean": 0.05265488, "gate_b2a/L7_absmax": 0.33808637, "gate_b2a/L7_rms": 0.05834414, "gate_b2a/L11_absmean": 0.08593936, "gate_b2a/L11_absmax": 0.61114877, "gate_b2a/L11_rms": 0.09472554, "gate_b2a/L14_absmean": 0.22280495, "gate_b2a/L14_absmax": 0.79619682, "gate_b2a/L14_rms": 0.2313496, "gate_b2c/L3_absmean": 0.03940342, "gate_b2c/L3_absmax": 0.55168331, "gate_b2c/L3_rms": 0.04902239, "gate_b2c/L5_absmean": 0.03992371, "gate_b2c/L5_absmax": 0.53649265, "gate_b2c/L5_rms": 0.04866471, "gate_b2c/L6_absmean": 0.05758226, "gate_b2c/L6_absmax": 0.33228159, "gate_b2c/L6_rms": 0.06294825}, "streams": {"cos_a2b/round0": 0.357954, "cos_c2b/round0": 0.24156, "cos_b2a/round0": 0.172504, "cos_b2c/round0": 0.154172, "rms_a/round0": 6.987328, "rms_b/round0": 12.122422, "rms_c/round0": 4.902865, "cos_a2b/round1": 0.410856, "cos_c2b/round1": 0.309951, "cos_b2a/round1": 0.263725, "cos_b2c/round1": 0.099441, "rms_a/round1": 13.832585, "rms_b/round1": 28.942579, "rms_c/round1": 10.56512, "cos_a2b/round2": 0.530153, "cos_c2b/round2": 0.532513, "cos_b2a/round2": 0.085404, "cos_b2c/round2": 0.183135, "rms_a/round2": 42.258518, "rms_b/round2": 82.949043, "rms_c/round2": 21.116892}, "alpha_mean": 0.41823995478495135, "alpha_hist": [0.00787, 0.07817, 0.19272, 0.21591, 0.19221, 0.1447, 0.10759, 0.05428, 0.00639, 0.00015], "alpha_by_type": {"special": {"share": 0.00075, "mean": 0.539, "std": 0.09733}, "whitespace": {"share": 0.02434, "mean": 0.51663, "std": 0.12505}, "digit": {"share": 0.03432, "mean": 0.47639, "std": 0.12685}, "word_start": {"share": 0.57335, "mean": 0.42854, "std": 0.16188}, "word_cont": {"share": 0.25664, "mean": 0.31563, "std": 0.12146}, "punct": {"share": 0.11061, "mean": 0.56242, "std": 0.15792}}, "alpha_p05": 0.1781, "alpha_p50": 0.39911, "alpha_p95": 0.71163}
{"step": 68510, "total_steps": 72479, "tokens": 35918970880, "loss": 4.30521434545517, "l_mix": 2.6688668727874756, "l_a": 2.7523564398288727, "l_c": 2.7021347284317017, "z_loss": 47.23529577255249, "lr": 0.0006771977639624489, "wd": 0.01, "grad_norm": 0.08989623934030533, "tokens_per_second": 1333250.9112585695, "eta_seconds": 1560.7707854747773, "data_wait_seconds": 1.7970800399780273e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 68520, "total_steps": 72479, "tokens": 35924213760, "loss": 4.31468391418457, "l_mix": 2.6747798919677734, "l_a": 2.75845006108284, "l_c": 2.707896500825882, "z_loss": 47.04037570953369, "lr": 0.0006741699642899214, "wd": 0.01, "grad_norm": 0.08718577772378922, "tokens_per_second": 1467913.8098402186, "eta_seconds": 1414.0177564144135, "data_wait_seconds": 1.52587890625e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 68530, "total_steps": 72479, "tokens": 35929456640, "loss": 4.3276063203811646, "l_mix": 2.683026909828186, "l_a": 2.766109585762024, "l_c": 2.71582168340683, "z_loss": 47.185344219207764, "lr": 0.0006711461392062208, "wd": 0.01, "grad_norm": 0.09482142329216003, "tokens_per_second": 1459295.8355243, "eta_seconds": 1418.7755913496017, "data_wait_seconds": 1.901388168334961e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 68540, "total_steps": 72479, "tokens": 35934699520, "loss": 4.273675978183746, "l_mix": 2.6497195959091187, "l_a": 2.730759918689728, "l_c": 2.6824277341365814, "z_loss": 46.73590803146362, "lr": 0.0006681263139692344, "wd": 0.01, "grad_norm": 0.10939635336399078, "tokens_per_second": 1364920.6536082155, "eta_seconds": 1513.0333228826523, "data_wait_seconds": 2.130866050720215e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 68550, "total_steps": 72479, "tokens": 35939942400, "loss": 3.0056311786174774, "l_mix": 1.8491058200597763, "l_a": 1.9624646753072739, "l_c": 1.8926198035478592, "z_loss": 51.43081188201904, "lr": 0.0006651105138034388, "wd": 0.01, "grad_norm": 0.09708236157894135, "tokens_per_second": 1307380.638897498, "eta_seconds": 1575.614240193367, "data_wait_seconds": 3.2454729080200195e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 68560, "total_steps": 72479, "tokens": 35945185280, "loss": 3.021570175886154, "l_mix": 1.8589957505464554, "l_a": 1.9728239178657532, "l_c": 1.9024239629507065, "z_loss": 51.375760555267334, "lr": 0.00066209876389969, "wd": 0.01, "grad_norm": 0.11181369423866272, "tokens_per_second": 1455523.508045935, "eta_seconds": 1411.6465042591094, "data_wait_seconds": 1.9103288650512695e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 68570, "total_steps": 72479, "tokens": 35950428160, "loss": 2.9550956189632416, "l_mix": 1.8174354434013367, "l_a": 1.9299525618553162, "l_c": 1.8622477352619171, "z_loss": 51.82856464385986, "lr": 0.0006590910894150108, "wd": 0.01, "grad_norm": 0.1100386530160904, "tokens_per_second": 1409143.6858002944, "eta_seconds": 1454.3880887746811, "data_wait_seconds": 1.8417835235595703e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 68580, "total_steps": 72479, "tokens": 35955671040, "loss": 2.9931259751319885, "l_mix": 1.8415888249874115, "l_a": 1.9529491662979126, "l_c": 1.8855078518390656, "z_loss": 52.10096597671509, "lr": 0.0006560875154723837, "wd": 0.01, "grad_norm": 0.10973123461008072, "tokens_per_second": 1335543.9047076725, "eta_seconds": 1530.6115394592287, "data_wait_seconds": 2.0623207092285156e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 68590, "total_steps": 72479, "tokens": 35960913920, "loss": 3.0071192383766174, "l_mix": 1.8499353379011154, "l_a": 1.9643041640520096, "l_c": 1.8929751515388489, "z_loss": 51.75935697555542, "lr": 0.0006530880671605388, "wd": 0.01, "grad_norm": 0.10084380209445953, "tokens_per_second": 1405928.2845752884, "eta_seconds": 1450.2560723543168, "data_wait_seconds": 0.0060551464557647705, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 68600, "total_steps": 72479, "tokens": 35966156800, "loss": 4.19029825925827, "l_mix": 2.5970200300216675, "l_a": 2.6797827780246735, "l_c": 2.6311442852020264, "z_loss": 47.8464937210083, "lr": 0.0006500927695337459, "wd": 0.01, "grad_norm": 0.09168339520692825, "tokens_per_second": 1359463.6435119712, "eta_seconds": 1495.967296886444, "data_wait_seconds": 1.9431114196777344e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.16712616, "gate_a2b/L1_absmax": 0.67358899, "gate_a2b/L1_rms": 0.17636938, "gate_a2b/L3_absmean": 0.15666482, "gate_a2b/L3_absmax": 0.50777024, "gate_a2b/L3_rms": 0.16210157, "gate_a2b/L5_absmean": 0.22391191, "gate_a2b/L5_absmax": 0.73677236, "gate_a2b/L5_rms": 0.22981349, "gate_c2b/L1_absmean": 0.0470711, "gate_c2b/L1_absmax": 0.59922433, "gate_c2b/L1_rms": 0.07236387, "gate_c2b/L3_absmean": 0.16334903, "gate_c2b/L3_absmax": 0.38424239, "gate_c2b/L3_rms": 0.16683437, "gate_c2b/L5_absmean": 0.46954867, "gate_c2b/L5_absmax": 0.68650615, "gate_c2b/L5_rms": 0.47831067, "gate_b2a/L7_absmean": 0.05262446, "gate_b2a/L7_absmax": 0.33742177, "gate_b2a/L7_rms": 0.05832794, "gate_b2a/L11_absmean": 0.08616368, "gate_b2a/L11_absmax": 0.61116832, "gate_b2a/L11_rms": 0.09490954, "gate_b2a/L14_absmean": 0.22311284, "gate_b2a/L14_absmax": 0.79506546, "gate_b2a/L14_rms": 0.23159419, "gate_b2c/L3_absmean": 0.03938898, "gate_b2c/L3_absmax": 0.55167007, "gate_b2c/L3_rms": 0.04899998, "gate_b2c/L5_absmean": 0.04005482, "gate_b2c/L5_absmax": 0.53645581, "gate_b2c/L5_rms": 0.04879508, "gate_b2c/L6_absmean": 0.05772622, "gate_b2c/L6_absmax": 0.33180213, "gate_b2c/L6_rms": 0.06306022}, "streams": {"cos_a2b/round0": 0.355865, "cos_c2b/round0": 0.240614, "cos_b2a/round0": 0.176395, "cos_b2c/round0": 0.154081, "rms_a/round0": 6.87923, "rms_b/round0": 12.099746, "rms_c/round0": 4.789461, "cos_a2b/round1": 0.409612, "cos_c2b/round1": 0.307207, "cos_b2a/round1": 0.259243, "cos_b2c/round1": 0.10281, "rms_a/round1": 13.839224, "rms_b/round1": 28.734324, "rms_c/round1": 10.447982, "cos_a2b/round2": 0.533421, "cos_c2b/round2": 0.52803, "cos_b2a/round2": 0.099975, "cos_b2c/round2": 0.180309, "rms_a/round2": 41.737129, "rms_b/round2": 81.58992, "rms_c/round2": 20.758589}, "alpha_mean": 0.4259215954154101, "alpha_hist": [0.01291, 0.07671, 0.18175, 0.20084, 0.18005, 0.1636, 0.11923, 0.05844, 0.00642, 5e-05], "alpha_by_type": {"special": {"share": 0.00082, "mean": 0.54894, "std": 0.13195}, "whitespace": {"share": 0.04735, "mean": 0.51074, "std": 0.1211}, "digit": {"share": 0.0399, "mean": 0.40126, "std": 0.14043}, "word_start": {"share": 0.51729, "mean": 0.44631, "std": 0.16534}, "word_cont": {"share": 0.27335, "mean": 0.32218, "std": 0.13535}, "punct": {"share": 0.12129, "mean": 0.54692, "std": 0.17148}}, "alpha_p05": 0.16532, "alpha_p50": 0.41087, "alpha_p95": 0.70497}
{"step": 68610, "total_steps": 72479, "tokens": 35971399680, "loss": 4.424732804298401, "l_mix": 2.7435312271118164, "l_a": 2.8277263045310974, "l_c": 2.7762786746025085, "z_loss": 46.412678241729736, "lr": 0.0006471016476116024, "wd": 0.01, "grad_norm": 0.09118170291185379, "tokens_per_second": 1448831.9701798088, "eta_seconds": 1400.0728267669676, "data_wait_seconds": 1.9609928131103516e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 68620, "total_steps": 72479, "tokens": 35976642560, "loss": 4.305187821388245, "l_mix": 2.6690550446510315, "l_a": 2.7528288066387177, "l_c": 2.700947016477585, "z_loss": 46.50000238418579, "lr": 0.0006441147263788281, "wd": 0.01, "grad_norm": 0.09759223461151123, "tokens_per_second": 1421128.6806145622, "eta_seconds": 1423.6764197349548, "data_wait_seconds": 3.325939178466797e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 68630, "total_steps": 72479, "tokens": 35981885440, "loss": 4.2624176144599915, "l_mix": 2.6424688398838043, "l_a": 2.724391460418701, "l_c": 2.6754377484321594, "z_loss": 47.215277671813965, "lr": 0.0006411320307850536, "wd": 0.01, "grad_norm": 0.09160111844539642, "tokens_per_second": 1430601.5349443988, "eta_seconds": 1410.5846126317977, "data_wait_seconds": 2.1904706954956055e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 68640, "total_steps": 72479, "tokens": 35987128320, "loss": 4.2476125955581665, "l_mix": 2.632955640554428, "l_a": 2.7173520624637604, "l_c": 2.664837658405304, "z_loss": 47.566808223724365, "lr": 0.0006381535857446138, "wd": 0.01, "grad_norm": 0.09181849658489227, "tokens_per_second": 1359714.308156357, "eta_seconds": 1480.2680386066438, "data_wait_seconds": 1.704692840576172e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 68650, "total_steps": 72479, "tokens": 35992371200, "loss": 4.228081166744232, "l_mix": 2.6210042536258698, "l_a": 2.7032863199710846, "l_c": 2.653636485338211, "z_loss": 46.94979429244995, "lr": 0.0006351794161363376, "wd": 0.01, "grad_norm": 0.09288627654314041, "tokens_per_second": 1374274.868660646, "eta_seconds": 1460.7694557905197, "data_wait_seconds": 1.919269561767578e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 68660, "total_steps": 72479, "tokens": 35997614080, "loss": 3.0111462771892548, "l_mix": 1.8527113795280457, "l_a": 1.9640928506851196, "l_c": 1.8973567634820938, "z_loss": 51.186437129974365, "lr": 0.000632209546803342, "wd": 0.01, "grad_norm": 0.10363879054784775, "tokens_per_second": 1421035.284373682, "eta_seconds": 1409.0120731115342, "data_wait_seconds": 1.71661376953125e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 68670, "total_steps": 72479, "tokens": 36002856960, "loss": 2.9834208488464355, "l_mix": 1.8356816321611404, "l_a": 1.9459094852209091, "l_c": 1.8798875957727432, "z_loss": 52.575562953948975, "lr": 0.0006292440025528248, "wd": 0.01, "grad_norm": 0.08994171023368835, "tokens_per_second": 1406990.7413131236, "eta_seconds": 1419.3504856586458, "data_wait_seconds": 3.606081008911133e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 68680, "total_steps": 72479, "tokens": 36008099840, "loss": 3.028161972761154, "l_mix": 1.8629825115203857, "l_a": 1.9755128473043442, "l_c": 1.9084183722734451, "z_loss": 51.73040723800659, "lr": 0.0006262828081558552, "wd": 0.01, "grad_norm": 0.10834960639476776, "tokens_per_second": 1390195.888194283, "eta_seconds": 1432.7262286663056, "data_wait_seconds": 1.817941665649414e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 68690, "total_steps": 72479, "tokens": 36013342720, "loss": 2.975765585899353, "l_mix": 1.831117957830429, "l_a": 1.941556453704834, "l_c": 1.873935505747795, "z_loss": 51.76208782196045, "lr": 0.0006233259883471697, "wd": 0.01, "grad_norm": 0.10838024318218231, "tokens_per_second": 1325832.973673239, "eta_seconds": 1498.3239001035693, "data_wait_seconds": 1.8149614334106445e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 68700, "total_steps": 72479, "tokens": 36018585600, "loss": 3.0112409591674805, "l_mix": 1.8532008677721024, "l_a": 1.9631186425685883, "l_c": 1.8970148414373398, "z_loss": 51.3366003036499, "lr": 0.0006203735678249622, "wd": 0.01, "grad_norm": 0.11066056787967682, "tokens_per_second": 1429739.951031762, "eta_seconds": 1385.7655376911164, "data_wait_seconds": 1.659989356994629e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16727248, "gate_a2b/L1_absmax": 0.67281908, "gate_a2b/L1_rms": 0.17651789, "gate_a2b/L3_absmean": 0.15665947, "gate_a2b/L3_absmax": 0.50879037, "gate_a2b/L3_rms": 0.16212736, "gate_a2b/L5_absmean": 0.22423016, "gate_a2b/L5_absmax": 0.73719347, "gate_a2b/L5_rms": 0.23012532, "gate_c2b/L1_absmean": 0.04708013, "gate_c2b/L1_absmax": 0.59875119, "gate_c2b/L1_rms": 0.07222068, "gate_c2b/L3_absmean": 0.16325639, "gate_c2b/L3_absmax": 0.38383168, "gate_c2b/L3_rms": 0.16671465, "gate_c2b/L5_absmean": 0.46952096, "gate_c2b/L5_absmax": 0.68667859, "gate_c2b/L5_rms": 0.4782939, "gate_b2a/L7_absmean": 0.05279114, "gate_b2a/L7_absmax": 0.33795369, "gate_b2a/L7_rms": 0.05845676, "gate_b2a/L11_absmean": 0.08626416, "gate_b2a/L11_absmax": 0.61133152, "gate_b2a/L11_rms": 0.09498861, "gate_b2a/L14_absmean": 0.22320646, "gate_b2a/L14_absmax": 0.79466099, "gate_b2a/L14_rms": 0.23167974, "gate_b2c/L3_absmean": 0.03951559, "gate_b2c/L3_absmax": 0.55196095, "gate_b2c/L3_rms": 0.04909052, "gate_b2c/L5_absmean": 0.0400548, "gate_b2c/L5_absmax": 0.53682756, "gate_b2c/L5_rms": 0.04880162, "gate_b2c/L6_absmean": 0.05783811, "gate_b2c/L6_absmax": 0.3309333, "gate_b2c/L6_rms": 0.06313836}, "streams": {"cos_a2b/round0": 0.353642, "cos_c2b/round0": 0.20758, "cos_b2a/round0": 0.16161, "cos_b2c/round0": 0.13154, "rms_a/round0": 7.214203, "rms_b/round0": 12.39081, "rms_c/round0": 4.938292, "cos_a2b/round1": 0.407554, "cos_c2b/round1": 0.306776, "cos_b2a/round1": 0.250428, "cos_b2c/round1": 0.114563, "rms_a/round1": 14.698895, "rms_b/round1": 30.36672, "rms_c/round1": 11.31128, "cos_a2b/round2": 0.519135, "cos_c2b/round2": 0.513547, "cos_b2a/round2": 0.088134, "cos_b2c/round2": 0.172323, "rms_a/round2": 43.187275, "rms_b/round2": 82.905968, "rms_c/round2": 22.782263}, "alpha_mean": 0.4071007273616871, "alpha_hist": [0.04529, 0.17413, 0.18292, 0.13844, 0.11188, 0.1091, 0.10335, 0.08249, 0.04677, 0.00563], "alpha_by_type": {"special": {"share": 0.00111, "mean": 0.65562, "std": 0.11081}, "whitespace": {"share": 0.03694, "mean": 0.61713, "std": 0.16876}, "digit": {"share": 0.01266, "mean": 0.38032, "std": 0.1687}, "word_start": {"share": 0.56151, "mean": 0.40993, "std": 0.21538}, "word_cont": {"share": 0.25996, "mean": 0.26498, "std": 0.14692}, "punct": {"share": 0.12781, "mean": 0.62353, "std": 0.18658}}, "alpha_p05": 0.10669, "alpha_p50": 0.38145, "alpha_p95": 0.81405}
{"step": 68710, "total_steps": 72479, "tokens": 36023828480, "loss": 4.297685205936432, "l_mix": 2.664319157600403, "l_a": 2.7479249835014343, "l_c": 2.6966281831264496, "z_loss": 47.04981517791748, "lr": 0.0006174255712506814, "wd": 0.01, "grad_norm": 0.08803652971982956, "tokens_per_second": 1469066.4609273872, "eta_seconds": 1345.1001194000246, "data_wait_seconds": 1.9043684005737305e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 68720, "total_steps": 72479, "tokens": 36029071360, "loss": 4.2883588671684265, "l_mix": 2.657771557569504, "l_a": 2.7437292635440826, "l_c": 2.6915616393089294, "z_loss": 47.75759983062744, "lr": 0.000614482023248821, "wd": 0.01, "grad_norm": 0.09981221705675125, "tokens_per_second": 1435456.2666351073, "eta_seconds": 1372.9422747373583, "data_wait_seconds": 1.5944242477416992e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 68730, "total_steps": 72479, "tokens": 36034314240, "loss": 4.259460151195526, "l_mix": 2.640587240457535, "l_a": 2.721935510635376, "l_c": 2.674307256937027, "z_loss": 47.25425481796265, "lr": 0.0006115429484067187, "wd": 0.01, "grad_norm": 0.09591438621282578, "tokens_per_second": 1456662.1787408267, "eta_seconds": 1349.35590467453, "data_wait_seconds": 1.6421079635620117e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 68740, "total_steps": 72479, "tokens": 36039557120, "loss": 4.354123950004578, "l_mix": 2.6996930837631226, "l_a": 2.781695604324341, "l_c": 2.7330740094184875, "z_loss": 46.58602571487427, "lr": 0.000608608371274345, "wd": 0.01, "grad_norm": 0.09513264149427414, "tokens_per_second": 1415849.137076441, "eta_seconds": 1384.5492296218872, "data_wait_seconds": 1.8268823623657227e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 68750, "total_steps": 72479, "tokens": 36044800000, "loss": 4.528331935405731, "l_mix": 2.809878349304199, "l_a": 2.884071558713913, "l_c": 2.844106376171112, "z_loss": 46.78675985336304, "lr": 0.0006056783163641038, "wd": 0.01, "grad_norm": 0.09147781133651733, "tokens_per_second": 1366044.1999469989, "eta_seconds": 1431.1908443927764, "data_wait_seconds": 5.0574541091918945e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 68760, "total_steps": 72479, "tokens": 36050042880, "loss": 4.5502108335494995, "l_mix": 2.823410928249359, "l_a": 2.8974860310554504, "l_c": 2.858513295650482, "z_loss": 46.004786014556885, "lr": 0.0006027528081506227, "wd": 0.01, "grad_norm": 0.0953204557299614, "tokens_per_second": 1376922.555876855, "eta_seconds": 1416.0760630130767, "data_wait_seconds": 3.507733345031738e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 68770, "total_steps": 72479, "tokens": 36055285760, "loss": 2.9806855618953705, "l_mix": 1.8337648510932922, "l_a": 1.945237785577774, "l_c": 1.8778311163187027, "z_loss": 51.76487064361572, "lr": 0.0005998318710705531, "wd": 0.01, "grad_norm": 0.11264955997467041, "tokens_per_second": 1458940.6143488178, "eta_seconds": 1332.874123096466, "data_wait_seconds": 1.6957521438598633e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 68780, "total_steps": 72479, "tokens": 36060528640, "loss": 2.9745030999183655, "l_mix": 1.829691767692566, "l_a": 1.9429394453763962, "l_c": 1.8730982691049576, "z_loss": 51.81365966796875, "lr": 0.0005969155295223625, "wd": 0.01, "grad_norm": 0.10871483385562897, "tokens_per_second": 1401706.6956583376, "eta_seconds": 1383.5571435928343, "data_wait_seconds": 3.269314765930176e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 68790, "total_steps": 72479, "tokens": 36065771520, "loss": 2.980959862470627, "l_mix": 1.8339138180017471, "l_a": 1.9454306960105896, "l_c": 1.8780561536550522, "z_loss": 51.666892528533936, "lr": 0.0005940038078661328, "wd": 0.01, "grad_norm": 0.09350337088108063, "tokens_per_second": 1409926.0149641642, "eta_seconds": 1371.7729948043823, "data_wait_seconds": 2.3692846298217773e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 68800, "total_steps": 72479, "tokens": 36071014400, "loss": 2.991418480873108, "l_mix": 1.8409197479486465, "l_a": 1.9511094093322754, "l_c": 1.8838862031698227, "z_loss": 51.92652606964111, "lr": 0.0005910967304233564, "wd": 0.01, "grad_norm": 0.10266592353582382, "tokens_per_second": 1365678.808504729, "eta_seconds": 1412.3786207914354, "data_wait_seconds": 2.8640031814575195e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16729987, "gate_a2b/L1_absmax": 0.67250836, "gate_a2b/L1_rms": 0.17653844, "gate_a2b/L3_absmean": 0.15686856, "gate_a2b/L3_absmax": 0.50699157, "gate_a2b/L3_rms": 0.16232578, "gate_a2b/L5_absmean": 0.22442034, "gate_a2b/L5_absmax": 0.73687327, "gate_a2b/L5_rms": 0.23029555, "gate_c2b/L1_absmean": 0.04727873, "gate_c2b/L1_absmax": 0.59747833, "gate_c2b/L1_rms": 0.07223382, "gate_c2b/L3_absmean": 0.16324143, "gate_c2b/L3_absmax": 0.38318852, "gate_c2b/L3_rms": 0.16664346, "gate_c2b/L5_absmean": 0.4695386, "gate_c2b/L5_absmax": 0.68661487, "gate_c2b/L5_rms": 0.47829151, "gate_b2a/L7_absmean": 0.05273267, "gate_b2a/L7_absmax": 0.3379969, "gate_b2a/L7_rms": 0.05840828, "gate_b2a/L11_absmean": 0.08641374, "gate_b2a/L11_absmax": 0.61109138, "gate_b2a/L11_rms": 0.09513637, "gate_b2a/L14_absmean": 0.22330472, "gate_b2a/L14_absmax": 0.79377592, "gate_b2a/L14_rms": 0.23177548, "gate_b2c/L3_absmean": 0.03960676, "gate_b2c/L3_absmax": 0.55256689, "gate_b2c/L3_rms": 0.0491845, "gate_b2c/L5_absmean": 0.04012317, "gate_b2c/L5_absmax": 0.53654665, "gate_b2c/L5_rms": 0.0488368, "gate_b2c/L6_absmean": 0.05788421, "gate_b2c/L6_absmax": 0.33044639, "gate_b2c/L6_rms": 0.06319215}, "streams": {"cos_a2b/round0": 0.352928, "cos_c2b/round0": 0.208173, "cos_b2a/round0": 0.161888, "cos_b2c/round0": 0.132777, "rms_a/round0": 7.147785, "rms_b/round0": 12.273988, "rms_c/round0": 4.931393, "cos_a2b/round1": 0.407809, "cos_c2b/round1": 0.305255, "cos_b2a/round1": 0.250155, "cos_b2c/round1": 0.11277, "rms_a/round1": 14.544783, "rms_b/round1": 30.2269, "rms_c/round1": 11.300409, "cos_a2b/round2": 0.518561, "cos_c2b/round2": 0.511419, "cos_b2a/round2": 0.089327, "cos_b2c/round2": 0.173025, "rms_a/round2": 42.696404, "rms_b/round2": 82.533279, "rms_c/round2": 22.609623}, "alpha_mean": 0.3995321553890321, "alpha_hist": [0.04808, 0.18204, 0.18823, 0.1396, 0.10927, 0.10393, 0.09859, 0.08017, 0.04306, 0.00703], "alpha_by_type": {"special": {"share": 0.00104, "mean": 0.64263, "std": 0.11377}, "whitespace": {"share": 0.03929, "mean": 0.60886, "std": 0.17734}, "digit": {"share": 0.0118, "mean": 0.43144, "std": 0.16848}, "word_start": {"share": 0.56764, "mean": 0.40167, "std": 0.21492}, "word_cont": {"share": 0.25974, "mean": 0.25977, "std": 0.14303}, "punct": {"share": 0.1205, "mean": 0.61727, "std": 0.19645}}, "alpha_p05": 0.10088, "alpha_p50": 0.37114, "alpha_p95": 0.801}
{"step": 68810, "total_steps": 72479, "tokens": 36076257280, "loss": 2.962909549474716, "l_mix": 1.8226732313632965, "l_a": 1.9359824061393738, "l_c": 1.8648051917552948, "z_loss": 51.575538635253906, "lr": 0.0005881943214767311, "wd": 0.01, "grad_norm": 0.11450795084238052, "tokens_per_second": 1379978.360832725, "eta_seconds": 1393.9440839052202, "data_wait_seconds": 2.434849739074707e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 68820, "total_steps": 72479, "tokens": 36081500160, "loss": 4.175324976444244, "l_mix": 2.5882084667682648, "l_a": 2.6714687645435333, "l_c": 2.618919402360916, "z_loss": 46.42544889450073, "lr": 0.0005852966052699609, "wd": 0.01, "grad_norm": 0.12554582953453064, "tokens_per_second": 1441822.302199349, "eta_seconds": 1330.5174910068513, "data_wait_seconds": 1.5437602996826172e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 68830, "total_steps": 72479, "tokens": 36086743040, "loss": 4.172662079334259, "l_mix": 2.586268723011017, "l_a": 2.6693736910820007, "l_c": 2.6186039447784424, "z_loss": 47.49246168136597, "lr": 0.0005824036060075505, "wd": 0.01, "grad_norm": 0.0971541628241539, "tokens_per_second": 1471536.0201320595, "eta_seconds": 1300.088401389122, "data_wait_seconds": 1.5914440155029297e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 68840, "total_steps": 72479, "tokens": 36091985920, "loss": 4.256284117698669, "l_mix": 2.6384476721286774, "l_a": 2.720388561487198, "l_c": 2.672399491071701, "z_loss": 47.16594839096069, "lr": 0.0005795153478546051, "wd": 0.01, "grad_norm": 0.09924008697271347, "tokens_per_second": 1399293.2819946394, "eta_seconds": 1363.462582540512, "data_wait_seconds": 1.7255544662475586e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 68850, "total_steps": 72479, "tokens": 36097228800, "loss": 4.2230567038059235, "l_mix": 2.617461681365967, "l_a": 2.7007876336574554, "l_c": 2.6511956453323364, "z_loss": 47.533658027648926, "lr": 0.0005766318549366258, "wd": 0.01, "grad_norm": 0.1023649275302887, "tokens_per_second": 1461228.0335872902, "eta_seconds": 1302.083664059639, "data_wait_seconds": 1.5676021575927734e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 68860, "total_steps": 72479, "tokens": 36102471680, "loss": 2.951046109199524, "l_mix": 1.814997836947441, "l_a": 1.9280469119548798, "l_c": 1.8587806671857834, "z_loss": 52.507694244384766, "lr": 0.0005737531513393116, "wd": 0.01, "grad_norm": 0.09232012927532196, "tokens_per_second": 1442595.9206600315, "eta_seconds": 1315.266627907753, "data_wait_seconds": 2.899765968322754e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 68870, "total_steps": 72479, "tokens": 36107714560, "loss": 3.0010433197021484, "l_mix": 1.8466345965862274, "l_a": 1.958441823720932, "l_c": 1.8895870447158813, "z_loss": 52.740768909454346, "lr": 0.0005708792611083559, "wd": 0.01, "grad_norm": 0.10559911280870438, "tokens_per_second": 1383465.587775402, "eta_seconds": 1367.6924158573152, "data_wait_seconds": 2.282857894897461e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 68880, "total_steps": 72479, "tokens": 36112957440, "loss": 4.501299500465393, "l_mix": 2.792802095413208, "l_a": 2.8675136864185333, "l_c": 2.827477753162384, "z_loss": 46.71750736236572, "lr": 0.0005680102082492463, "wd": 0.01, "grad_norm": 0.08857855945825577, "tokens_per_second": 1419384.2826133978, "eta_seconds": 1329.388055872917, "data_wait_seconds": 1.9252300262451172e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 68890, "total_steps": 72479, "tokens": 36118200320, "loss": 2.9694249629974365, "l_mix": 1.8262614011764526, "l_a": 1.9400565326213837, "l_c": 1.8704884350299835, "z_loss": 53.110126972198486, "lr": 0.0005651460167270642, "wd": 0.01, "grad_norm": 0.0986868143081665, "tokens_per_second": 1353769.4264794006, "eta_seconds": 1389.9483879566194, "data_wait_seconds": 2.35140323638916e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 68900, "total_steps": 72479, "tokens": 36123443200, "loss": 3.0106508135795593, "l_mix": 1.8522156029939651, "l_a": 1.9631522744894028, "l_c": 1.8982983827590942, "z_loss": 52.65891456604004, "lr": 0.0005622867104662837, "wd": 0.01, "grad_norm": 0.09881623834371567, "tokens_per_second": 1418063.9546864035, "eta_seconds": 1323.2313999652863, "data_wait_seconds": 1.996755599975586e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16758576, "gate_a2b/L1_absmax": 0.67161369, "gate_a2b/L1_rms": 0.17678662, "gate_a2b/L3_absmean": 0.15693827, "gate_a2b/L3_absmax": 0.50559646, "gate_a2b/L3_rms": 0.16236812, "gate_a2b/L5_absmean": 0.22466336, "gate_a2b/L5_absmax": 0.73651725, "gate_a2b/L5_rms": 0.23052, "gate_c2b/L1_absmean": 0.04779444, "gate_c2b/L1_absmax": 0.59725511, "gate_c2b/L1_rms": 0.07259344, "gate_c2b/L3_absmean": 0.16327523, "gate_c2b/L3_absmax": 0.38173619, "gate_c2b/L3_rms": 0.16671872, "gate_c2b/L5_absmean": 0.46962616, "gate_c2b/L5_absmax": 0.68656588, "gate_c2b/L5_rms": 0.47836798, "gate_b2a/L7_absmean": 0.05283311, "gate_b2a/L7_absmax": 0.33795732, "gate_b2a/L7_rms": 0.05848714, "gate_b2a/L11_absmean": 0.08634845, "gate_b2a/L11_absmax": 0.61090934, "gate_b2a/L11_rms": 0.09505058, "gate_b2a/L14_absmean": 0.22342105, "gate_b2a/L14_absmax": 0.79365081, "gate_b2a/L14_rms": 0.23185696, "gate_b2c/L3_absmean": 0.03979856, "gate_b2c/L3_absmax": 0.55239475, "gate_b2c/L3_rms": 0.04932748, "gate_b2c/L5_absmean": 0.04025568, "gate_b2c/L5_absmax": 0.53682655, "gate_b2c/L5_rms": 0.04896094, "gate_b2c/L6_absmean": 0.05801612, "gate_b2c/L6_absmax": 0.33130309, "gate_b2c/L6_rms": 0.06332032}, "streams": {"cos_a2b/round0": 0.355007, "cos_c2b/round0": 0.209618, "cos_b2a/round0": 0.162442, "cos_b2c/round0": 0.133427, "rms_a/round0": 7.163093, "rms_b/round0": 12.291913, "rms_c/round0": 4.910124, "cos_a2b/round1": 0.407563, "cos_c2b/round1": 0.307681, "cos_b2a/round1": 0.250647, "cos_b2c/round1": 0.114602, "rms_a/round1": 14.638107, "rms_b/round1": 30.187128, "rms_c/round1": 11.315317, "cos_a2b/round2": 0.521121, "cos_c2b/round2": 0.511975, "cos_b2a/round2": 0.090529, "cos_b2c/round2": 0.170466, "rms_a/round2": 43.025139, "rms_b/round2": 82.714943, "rms_c/round2": 22.725435}, "alpha_mean": 0.40590884225181156, "alpha_hist": [0.0517, 0.17839, 0.18216, 0.13518, 0.10559, 0.10489, 0.09746, 0.08644, 0.052, 0.0062], "alpha_by_type": {"special": {"share": 0.00113, "mean": 0.66319, "std": 0.10675}, "whitespace": {"share": 0.03947, "mean": 0.61962, "std": 0.17983}, "digit": {"share": 0.01445, "mean": 0.3589, "std": 0.17486}, "word_start": {"share": 0.57211, "mean": 0.40643, "std": 0.2193}, "word_cont": {"share": 0.24939, "mean": 0.26003, "std": 0.1462}, "punct": {"share": 0.12344, "mean": 0.63301, "std": 0.19221}}, "alpha_p05": 0.09436, "alpha_p50": 0.34865, "alpha_p95": 0.80131}
{"step": 68910, "total_steps": 72479, "tokens": 36128686080, "loss": 3.013648211956024, "l_mix": 1.8539596945047379, "l_a": 1.9675790071487427, "l_c": 1.8980493545532227, "z_loss": 52.223416805267334, "lr": 0.0005594323133505737, "wd": 0.01, "grad_norm": 0.10550344735383987, "tokens_per_second": 1349620.373756808, "eta_seconds": 1386.4520041227343, "data_wait_seconds": 2.944469451904297e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 68920, "total_steps": 72479, "tokens": 36133928960, "loss": 2.969377636909485, "l_mix": 1.826942816376686, "l_a": 1.9376853108406067, "l_c": 1.8704306036233902, "z_loss": 52.341684341430664, "lr": 0.0005565828492225953, "wd": 0.01, "grad_norm": 0.1045389175415039, "tokens_per_second": 1468459.9020542928, "eta_seconds": 1270.6788856744768, "data_wait_seconds": 2.3812055587768555e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 68930, "total_steps": 72479, "tokens": 36139171840, "loss": 4.266637861728668, "l_mix": 2.644901782274246, "l_a": 2.7272115647792816, "l_c": 2.678575575351715, "z_loss": 47.28239297866821, "lr": 0.0005537383418838075, "wd": 0.01, "grad_norm": 0.0859949067234993, "tokens_per_second": 1403797.0652793203, "eta_seconds": 1325.4751402616503, "data_wait_seconds": 1.8477439880371094e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 68940, "total_steps": 72479, "tokens": 36144414720, "loss": 4.273503065109253, "l_mix": 2.6492651104927063, "l_a": 2.7319739162921906, "l_c": 2.6821526288986206, "z_loss": 47.27658271789551, "lr": 0.0005508988150942626, "wd": 0.01, "grad_norm": 0.09016608446836472, "tokens_per_second": 1405700.727723379, "eta_seconds": 1319.950395846367, "data_wait_seconds": 1.761317253112793e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 68950, "total_steps": 72479, "tokens": 36149657600, "loss": 4.2892120480537415, "l_mix": 2.6586905121803284, "l_a": 2.742719292640686, "l_c": 2.6923521161079407, "z_loss": 47.20958375930786, "lr": 0.0005480642925724125, "wd": 0.01, "grad_norm": 0.08684645593166351, "tokens_per_second": 1447227.4045715318, "eta_seconds": 1278.4530932426453, "data_wait_seconds": 1.704692840576172e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 68960, "total_steps": 72479, "tokens": 36154900480, "loss": 4.360791325569153, "l_mix": 2.704134911298752, "l_a": 2.785541832447052, "l_c": 2.7366460263729095, "z_loss": 47.046974182128906, "lr": 0.0005452347979949065, "wd": 0.01, "grad_norm": 0.08966883271932602, "tokens_per_second": 1429196.5423766267, "eta_seconds": 1290.9137527942657, "data_wait_seconds": 2.047419548034668e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 68970, "total_steps": 72479, "tokens": 36160143360, "loss": 2.9808100759983063, "l_mix": 1.8336663395166397, "l_a": 1.9447555541992188, "l_c": 1.8790568113327026, "z_loss": 52.5188627243042, "lr": 0.000542410354996399, "wd": 0.01, "grad_norm": 0.10468664020299911, "tokens_per_second": 1393029.2234371805, "eta_seconds": 1320.6661863565446, "data_wait_seconds": 3.0338764190673828e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 68980, "total_steps": 72479, "tokens": 36165386240, "loss": 2.9585998356342316, "l_mix": 1.820285677909851, "l_a": 1.9308603554964066, "l_c": 1.8635200262069702, "z_loss": 52.814754486083984, "lr": 0.0005395909871693456, "wd": 0.01, "grad_norm": 0.0960567370057106, "tokens_per_second": 1337513.4528782389, "eta_seconds": 1371.5628116130829, "data_wait_seconds": 3.0100345611572266e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 68990, "total_steps": 72479, "tokens": 36170629120, "loss": 2.979251593351364, "l_mix": 1.8332861810922623, "l_a": 1.9447822719812393, "l_c": 1.8751022070646286, "z_loss": 52.304758071899414, "lr": 0.0005367767180638107, "wd": 0.01, "grad_norm": 0.10962671786546707, "tokens_per_second": 1465117.1915139763, "eta_seconds": 1248.5286792039872, "data_wait_seconds": 2.709031105041504e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 69000, "total_steps": 72479, "tokens": 36175872000, "loss": 4.567173063755035, "l_mix": 2.8337842226028442, "l_a": 2.9092130064964294, "l_c": 2.8687495589256287, "z_loss": 46.157423973083496, "lr": 0.0005339675711872696, "wd": 0.01, "grad_norm": 0.09538238495588303, "tokens_per_second": 1404316.663833583, "eta_seconds": 1298.8508923768998, "data_wait_seconds": 2.4706125259399414e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.16780655, "gate_a2b/L1_absmax": 0.6717419, "gate_a2b/L1_rms": 0.17699531, "gate_a2b/L3_absmean": 0.15708666, "gate_a2b/L3_absmax": 0.50687319, "gate_a2b/L3_rms": 0.16252662, "gate_a2b/L5_absmean": 0.22490267, "gate_a2b/L5_absmax": 0.73542994, "gate_a2b/L5_rms": 0.23075977, "gate_c2b/L1_absmean": 0.04760695, "gate_c2b/L1_absmax": 0.59586632, "gate_c2b/L1_rms": 0.07234495, "gate_c2b/L3_absmean": 0.16313919, "gate_c2b/L3_absmax": 0.38231722, "gate_c2b/L3_rms": 0.1665588, "gate_c2b/L5_absmean": 0.46948242, "gate_c2b/L5_absmax": 0.6858542, "gate_c2b/L5_rms": 0.47822222, "gate_b2a/L7_absmean": 0.0527712, "gate_b2a/L7_absmax": 0.33765739, "gate_b2a/L7_rms": 0.05843572, "gate_b2a/L11_absmean": 0.08643766, "gate_b2a/L11_absmax": 0.61101139, "gate_b2a/L11_rms": 0.09513301, "gate_b2a/L14_absmean": 0.22337902, "gate_b2a/L14_absmax": 0.79284191, "gate_b2a/L14_rms": 0.23181158, "gate_b2c/L3_absmean": 0.03974876, "gate_b2c/L3_absmax": 0.5523293, "gate_b2c/L3_rms": 0.0492788, "gate_b2c/L5_absmean": 0.0402744, "gate_b2c/L5_absmax": 0.53699529, "gate_b2c/L5_rms": 0.04896386, "gate_b2c/L6_absmean": 0.05814986, "gate_b2c/L6_absmax": 0.33057237, "gate_b2c/L6_rms": 0.06340522}, "streams": {"cos_a2b/round0": 0.349575, "cos_c2b/round0": 0.230245, "cos_b2a/round0": 0.17584, "cos_b2c/round0": 0.15197, "rms_a/round0": 6.810187, "rms_b/round0": 12.00529, "rms_c/round0": 4.827368, "cos_a2b/round1": 0.410615, "cos_c2b/round1": 0.313726, "cos_b2a/round1": 0.258528, "cos_b2c/round1": 0.103054, "rms_a/round1": 13.571695, "rms_b/round1": 28.457453, "rms_c/round1": 10.529558, "cos_a2b/round2": 0.542895, "cos_c2b/round2": 0.523609, "cos_b2a/round2": 0.097912, "cos_b2c/round2": 0.181493, "rms_a/round2": 41.627949, "rms_b/round2": 81.118896, "rms_c/round2": 21.125998}, "alpha_mean": 0.4363174451826879, "alpha_hist": [0.00806, 0.06088, 0.1759, 0.21718, 0.17563, 0.16411, 0.12169, 0.06671, 0.00967, 0.00017], "alpha_by_type": {"special": {"share": 0.00061, "mean": 0.57219, "std": 0.12167}, "whitespace": {"share": 0.04344, "mean": 0.54099, "std": 0.1278}, "digit": {"share": 0.02347, "mean": 0.44583, "std": 0.12257}, "word_start": {"share": 0.57512, "mean": 0.45854, "std": 0.1625}, "word_cont": {"share": 0.26544, "mean": 0.31997, "std": 0.12242}, "punct": {"share": 0.09192, "mean": 0.58045, "std": 0.16541}}, "alpha_p05": 0.17441, "alpha_p50": 0.43015, "alpha_p95": 0.72448}
{"step": 69010, "total_steps": 72479, "tokens": 36181114880, "loss": 4.635709822177887, "l_mix": 2.87737900018692, "l_a": 2.947834700345993, "l_c": 2.9132677614688873, "z_loss": 46.56903123855591, "lr": 0.0005311635700044097, "wd": 0.01, "grad_norm": 0.12725169956684113, "tokens_per_second": 1468246.651312668, "eta_seconds": 1238.7258437633513, "data_wait_seconds": 2.2172927856445312e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 69020, "total_steps": 72479, "tokens": 36186357760, "loss": 3.0080106258392334, "l_mix": 1.8510477542877197, "l_a": 1.9612868577241898, "l_c": 1.8952558487653732, "z_loss": 51.897632122039795, "lr": 0.0005283647379369403, "wd": 0.01, "grad_norm": 0.10932403802871704, "tokens_per_second": 1382515.7897174857, "eta_seconds": 1311.7479058742524, "data_wait_seconds": 2.3424625396728516e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 69030, "total_steps": 72479, "tokens": 36191600640, "loss": 2.9831531047821045, "l_mix": 1.8354734182357788, "l_a": 1.9465238451957703, "l_c": 1.8790751546621323, "z_loss": 52.28789663314819, "lr": 0.0005255710983633895, "wd": 0.01, "grad_norm": 0.10328412055969238, "tokens_per_second": 1392688.1513840444, "eta_seconds": 1298.4021657705307, "data_wait_seconds": 2.4825334548950195e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 69040, "total_steps": 72479, "tokens": 36196843520, "loss": 4.33501124382019, "l_mix": 2.6871798634529114, "l_a": 2.773311287164688, "l_c": 2.7194598019123077, "z_loss": 47.0340576171875, "lr": 0.0005227826746189158, "wd": 0.01, "grad_norm": 0.1017913669347763, "tokens_per_second": 1460099.7634995386, "eta_seconds": 1234.865231180191, "data_wait_seconds": 1.564621925354004e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 69050, "total_steps": 72479, "tokens": 36202086400, "loss": 4.2348469495773315, "l_mix": 2.6247635185718536, "l_a": 2.707716017961502, "l_c": 2.6592284739017487, "z_loss": 47.759414196014404, "lr": 0.0005199994899951074, "wd": 0.01, "grad_norm": 0.08568891882896423, "tokens_per_second": 1395438.5035371676, "eta_seconds": 1288.3287564754487, "data_wait_seconds": 1.6301870346069336e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 69060, "total_steps": 72479, "tokens": 36207329280, "loss": 4.21450474858284, "l_mix": 2.612007111310959, "l_a": 2.6970717906951904, "l_c": 2.644586503505707, "z_loss": 48.407333850860596, "lr": 0.0005172215677397921, "wd": 0.01, "grad_norm": 0.08952648192644119, "tokens_per_second": 1423300.9148330076, "eta_seconds": 1259.4249419212342, "data_wait_seconds": 1.570582389831543e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 69070, "total_steps": 72479, "tokens": 36212572160, "loss": 4.295133948326111, "l_mix": 2.66257843375206, "l_a": 2.7457969784736633, "l_c": 2.6960547864437103, "z_loss": 47.14683675765991, "lr": 0.0005144489310568409, "wd": 0.01, "grad_norm": 0.08988403528928757, "tokens_per_second": 1362689.313934395, "eta_seconds": 1311.5959549427032, "data_wait_seconds": 2.2470951080322266e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 69080, "total_steps": 72479, "tokens": 36217815040, "loss": 2.983493208885193, "l_mix": 1.8351625800132751, "l_a": 1.9480684697628021, "l_c": 1.8797002136707306, "z_loss": 52.35006761550903, "lr": 0.0005116816031059753, "wd": 0.01, "grad_norm": 0.10454433411359787, "tokens_per_second": 1370570.4401884149, "eta_seconds": 1300.2286199569703, "data_wait_seconds": 2.6017427444458008e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 69090, "total_steps": 72479, "tokens": 36223057920, "loss": 2.945563852787018, "l_mix": 1.812185451388359, "l_a": 1.9234013259410858, "l_c": 1.8545266091823578, "z_loss": 52.89547395706177, "lr": 0.0005089196070025712, "wd": 0.01, "grad_norm": 0.0983421579003334, "tokens_per_second": 1325385.477600529, "eta_seconds": 1340.6001967191696, "data_wait_seconds": 2.0951032638549805e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 69100, "total_steps": 72479, "tokens": 36228300800, "loss": 2.96829891204834, "l_mix": 1.8263277560472488, "l_a": 1.9375969916582108, "l_c": 1.868973195552826, "z_loss": 52.52357578277588, "lr": 0.0005061629658174697, "wd": 0.01, "grad_norm": 0.09637392312288284, "tokens_per_second": 1376525.8167547397, "eta_seconds": 1286.9857800245286, "data_wait_seconds": 3.096461296081543e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16789567, "gate_a2b/L1_absmax": 0.67198956, "gate_a2b/L1_rms": 0.17713208, "gate_a2b/L3_absmean": 0.15716524, "gate_a2b/L3_absmax": 0.50661719, "gate_a2b/L3_rms": 0.16259404, "gate_a2b/L5_absmean": 0.22512423, "gate_a2b/L5_absmax": 0.73604667, "gate_a2b/L5_rms": 0.23099223, "gate_c2b/L1_absmean": 0.04773981, "gate_c2b/L1_absmax": 0.59462696, "gate_c2b/L1_rms": 0.0723493, "gate_c2b/L3_absmean": 0.16312908, "gate_c2b/L3_absmax": 0.38329843, "gate_c2b/L3_rms": 0.16654781, "gate_c2b/L5_absmean": 0.46958852, "gate_c2b/L5_absmax": 0.68685269, "gate_c2b/L5_rms": 0.47833288, "gate_b2a/L7_absmean": 0.05276918, "gate_b2a/L7_absmax": 0.33757973, "gate_b2a/L7_rms": 0.05844064, "gate_b2a/L11_absmean": 0.08647369, "gate_b2a/L11_absmax": 0.61057603, "gate_b2a/L11_rms": 0.0951539, "gate_b2a/L14_absmean": 0.22366323, "gate_b2a/L14_absmax": 0.7930727, "gate_b2a/L14_rms": 0.23206358, "gate_b2c/L3_absmean": 0.0398746, "gate_b2c/L3_absmax": 0.55302095, "gate_b2c/L3_rms": 0.04942097, "gate_b2c/L5_absmean": 0.04028925, "gate_b2c/L5_absmax": 0.53767788, "gate_b2c/L5_rms": 0.04901585, "gate_b2c/L6_absmean": 0.05817311, "gate_b2c/L6_absmax": 0.3299574, "gate_b2c/L6_rms": 0.06344598}, "streams": {"cos_a2b/round0": 0.352491, "cos_c2b/round0": 0.208963, "cos_b2a/round0": 0.163714, "cos_b2c/round0": 0.132395, "rms_a/round0": 7.113523, "rms_b/round0": 12.259773, "rms_c/round0": 4.913537, "cos_a2b/round1": 0.408297, "cos_c2b/round1": 0.307986, "cos_b2a/round1": 0.25093, "cos_b2c/round1": 0.112525, "rms_a/round1": 14.576834, "rms_b/round1": 30.053917, "rms_c/round1": 11.337562, "cos_a2b/round2": 0.523155, "cos_c2b/round2": 0.509637, "cos_b2a/round2": 0.089971, "cos_b2c/round2": 0.172272, "rms_a/round2": 42.9202, "rms_b/round2": 82.457825, "rms_c/round2": 22.748726}, "alpha_mean": 0.409501085805573, "alpha_hist": [0.04639, 0.16872, 0.18442, 0.13934, 0.10947, 0.10735, 0.10435, 0.08751, 0.04655, 0.00591], "alpha_by_type": {"special": {"share": 0.00111, "mean": 0.66535, "std": 0.11626}, "whitespace": {"share": 0.04193, "mean": 0.61378, "std": 0.17538}, "digit": {"share": 0.00876, "mean": 0.37488, "std": 0.19705}, "word_start": {"share": 0.56728, "mean": 0.41272, "std": 0.21432}, "word_cont": {"share": 0.25626, "mean": 0.26439, "std": 0.14821}, "punct": {"share": 0.12466, "mean": 0.62457, "std": 0.18934}}, "alpha_p05": 0.10669, "alpha_p50": 0.38122, "alpha_p95": 0.80286}
{"step": 69110, "total_steps": 72479, "tokens": 36233543680, "loss": 2.9240553975105286, "l_mix": 1.810780480504036, "l_a": 1.8760881572961807, "l_c": 1.8348280489444733, "z_loss": 61.306931018829346, "lr": 0.0005034117025767821, "wd": 0.01, "grad_norm": 0.10038170963525772, "tokens_per_second": 1455166.0764684002, "eta_seconds": 1213.8313973665238, "data_wait_seconds": 2.8878450393676758e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 69120, "total_steps": 72479, "tokens": 36238786560, "loss": 4.270336389541626, "l_mix": 2.647157847881317, "l_a": 2.7312675416469574, "l_c": 2.679327428340912, "z_loss": 47.592018127441406, "lr": 0.0005006658402616975, "wd": 0.01, "grad_norm": 0.0995623841881752, "tokens_per_second": 1276002.633882972, "eta_seconds": 1380.1565492391585, "data_wait_seconds": 2.0623207092285156e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 69130, "total_steps": 72479, "tokens": 36244029440, "loss": 4.270861327648163, "l_mix": 2.6471686959266663, "l_a": 2.7325460612773895, "l_c": 2.6797623932361603, "z_loss": 47.065279960632324, "lr": 0.000497925401808292, "wd": 0.01, "grad_norm": 0.08813884109258652, "tokens_per_second": 1473510.7617762352, "eta_seconds": 1191.6034531593323, "data_wait_seconds": 2.6285648345947266e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 69140, "total_steps": 72479, "tokens": 36249272320, "loss": 4.276170909404755, "l_mix": 2.650998502969742, "l_a": 2.73320010304451, "l_c": 2.6840407848358154, "z_loss": 46.80445146560669, "lr": 0.000495190410107335, "wd": 0.01, "grad_norm": 0.0875348448753357, "tokens_per_second": 1468124.0234237746, "eta_seconds": 1192.4044590711594, "data_wait_seconds": 2.136826515197754e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 69150, "total_steps": 72479, "tokens": 36254515200, "loss": 4.195783197879791, "l_mix": 2.600264072418213, "l_a": 2.6850839257240295, "l_c": 2.6333130300045013, "z_loss": 47.98518514633179, "lr": 0.0004924608880041011, "wd": 0.01, "grad_norm": 0.09476429969072342, "tokens_per_second": 1248455.7013552953, "eta_seconds": 1398.0109587430954, "data_wait_seconds": 1.817941665649414e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 69160, "total_steps": 72479, "tokens": 36259758080, "loss": 4.327587068080902, "l_mix": 2.682561367750168, "l_a": 2.766744315624237, "l_c": 2.716674417257309, "z_loss": 47.17037010192871, "lr": 0.0004897368582981758, "wd": 0.01, "grad_norm": 0.08932344615459442, "tokens_per_second": 1395234.2471705882, "eta_seconds": 1247.1825971364974, "data_wait_seconds": 1.868605613708496e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 69170, "total_steps": 72479, "tokens": 36265000960, "loss": 4.3229498863220215, "l_mix": 2.679534465074539, "l_a": 2.7650459110736847, "l_c": 2.7130051851272583, "z_loss": 46.95269250869751, "lr": 0.0004870183437432693, "wd": 0.01, "grad_norm": 0.0979965403676033, "tokens_per_second": 1423883.1801042305, "eta_seconds": 1218.4068301677703, "data_wait_seconds": 4.2766332626342773e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 69180, "total_steps": 72479, "tokens": 36270243840, "loss": 4.192029356956482, "l_mix": 2.598695158958435, "l_a": 2.6801677346229553, "l_c": 2.6309461891651154, "z_loss": 46.820054054260254, "lr": 0.00048430536704702134, "wd": 0.01, "grad_norm": 0.0878513976931572, "tokens_per_second": 1234272.9418138233, "eta_seconds": 1401.3319529294968, "data_wait_seconds": 1.9669532775878906e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 69190, "total_steps": 72479, "tokens": 36275486720, "loss": 2.9768683314323425, "l_mix": 1.8315253853797913, "l_a": 1.9441913217306137, "l_c": 1.8736182749271393, "z_loss": 52.27977466583252, "lr": 0.00048159795087081547, "wd": 0.01, "grad_norm": 0.10493841767311096, "tokens_per_second": 1450527.544199066, "eta_seconds": 1188.7973026752472, "data_wait_seconds": 1.633167266845703e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 69200, "total_steps": 72479, "tokens": 36280729600, "loss": 2.9869727790355682, "l_mix": 1.8379319608211517, "l_a": 1.9485002607107162, "l_c": 1.8816357403993607, "z_loss": 52.62926721572876, "lr": 0.00047889611782958894, "wd": 0.01, "grad_norm": 0.10421120375394821, "tokens_per_second": 1423131.9830302852, "eta_seconds": 1207.9978332996368, "data_wait_seconds": 1.9311904907226562e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16798612, "gate_a2b/L1_absmax": 0.67159158, "gate_a2b/L1_rms": 0.17720747, "gate_a2b/L3_absmean": 0.15725736, "gate_a2b/L3_absmax": 0.50563425, "gate_a2b/L3_rms": 0.16269904, "gate_a2b/L5_absmean": 0.22512107, "gate_a2b/L5_absmax": 0.73561639, "gate_a2b/L5_rms": 0.23098794, "gate_c2b/L1_absmean": 0.04807427, "gate_c2b/L1_absmax": 0.59395385, "gate_c2b/L1_rms": 0.07253393, "gate_c2b/L3_absmean": 0.16318595, "gate_c2b/L3_absmax": 0.38242689, "gate_c2b/L3_rms": 0.16660182, "gate_c2b/L5_absmean": 0.46967253, "gate_c2b/L5_absmax": 0.68677163, "gate_c2b/L5_rms": 0.47841609, "gate_b2a/L7_absmean": 0.05287419, "gate_b2a/L7_absmax": 0.33806914, "gate_b2a/L7_rms": 0.05853685, "gate_b2a/L11_absmean": 0.08660531, "gate_b2a/L11_absmax": 0.61025536, "gate_b2a/L11_rms": 0.09525869, "gate_b2a/L14_absmean": 0.22381595, "gate_b2a/L14_absmax": 0.79297954, "gate_b2a/L14_rms": 0.23220682, "gate_b2c/L3_absmean": 0.04008683, "gate_b2c/L3_absmax": 0.55337894, "gate_b2c/L3_rms": 0.0495915, "gate_b2c/L5_absmean": 0.04038765, "gate_b2c/L5_absmax": 0.53773522, "gate_b2c/L5_rms": 0.04909889, "gate_b2c/L6_absmean": 0.05822449, "gate_b2c/L6_absmax": 0.32987443, "gate_b2c/L6_rms": 0.06348446}, "streams": {"cos_a2b/round0": 0.352099, "cos_c2b/round0": 0.209207, "cos_b2a/round0": 0.1661, "cos_b2c/round0": 0.132773, "rms_a/round0": 7.103625, "rms_b/round0": 12.292754, "rms_c/round0": 4.946969, "cos_a2b/round1": 0.408531, "cos_c2b/round1": 0.309975, "cos_b2a/round1": 0.253171, "cos_b2c/round1": 0.114191, "rms_a/round1": 14.526332, "rms_b/round1": 30.125275, "rms_c/round1": 11.380219, "cos_a2b/round2": 0.520954, "cos_c2b/round2": 0.513262, "cos_b2a/round2": 0.090915, "cos_b2c/round2": 0.172323, "rms_a/round2": 42.87719, "rms_b/round2": 82.268196, "rms_c/round2": 22.932486}, "alpha_mean": 0.40467147642030454, "alpha_hist": [0.05272, 0.17924, 0.17885, 0.13515, 0.10835, 0.1022, 0.10339, 0.08818, 0.04602, 0.00589], "alpha_by_type": {"special": {"share": 0.00116, "mean": 0.64158, "std": 0.14418}, "whitespace": {"share": 0.03101, "mean": 0.63993, "std": 0.17637}, "digit": {"share": 0.006, "mean": 0.40951, "std": 0.21632}, "word_start": {"share": 0.59366, "mean": 0.40503, "std": 0.21652}, "word_cont": {"share": 0.24728, "mean": 0.2617, "std": 0.15047}, "punct": {"share": 0.1209, "mean": 0.63248, "std": 0.18646}}, "alpha_p05": 0.09774, "alpha_p50": 0.33895, "alpha_p95": 0.79502}
{"step": 69210, "total_steps": 72479, "tokens": 36285972480, "loss": 2.9851261377334595, "l_mix": 1.8366314768791199, "l_a": 1.948034480214119, "l_c": 1.8802806437015533, "z_loss": 51.889387130737305, "lr": 0.0004761998904916418, "wd": 0.01, "grad_norm": 0.10931457579135895, "tokens_per_second": 1393563.5759616564, "eta_seconds": 1229.8667255401613, "data_wait_seconds": 1.683831214904785e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 69220, "total_steps": 72479, "tokens": 36291215360, "loss": 2.771077036857605, "l_mix": 1.7163930088281631, "l_a": 1.7774164825677872, "l_c": 1.7381968796253204, "z_loss": 58.86192321777344, "lr": 0.00047350929137845115, "wd": 0.01, "grad_norm": 0.1817430704832077, "tokens_per_second": 1434741.2997966264, "eta_seconds": 1190.9147608995438, "data_wait_seconds": 3.579258918762207e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 69230, "total_steps": 72479, "tokens": 36296458240, "loss": 4.1880828738212585, "l_mix": 2.595400035381317, "l_a": 2.6804551780223846, "l_c": 2.6284875571727753, "z_loss": 47.97545051574707, "lr": 0.0004708243429644812, "wd": 0.01, "grad_norm": 0.09869631379842758, "tokens_per_second": 1369985.544937605, "eta_seconds": 1243.3793322086333, "data_wait_seconds": 2.6911497116088867e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 69240, "total_steps": 72479, "tokens": 36301701120, "loss": 4.3259934186935425, "l_mix": 2.6823897063732147, "l_a": 2.7632212936878204, "l_c": 2.7154572010040283, "z_loss": 46.4764347076416, "lr": 0.00046814506767699665, "wd": 0.01, "grad_norm": 0.10717058181762695, "tokens_per_second": 1401178.9375480316, "eta_seconds": 1211.9571501493454, "data_wait_seconds": 1.6689300537109375e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 69250, "total_steps": 72479, "tokens": 36306944000, "loss": 4.268054127693176, "l_mix": 2.6457512378692627, "l_a": 2.7293215692043304, "l_c": 2.6783545315265656, "z_loss": 47.25560808181763, "lr": 0.0004654714878958738, "wd": 0.01, "grad_norm": 0.11170535534620285, "tokens_per_second": 1440933.8407708278, "eta_seconds": 1174.8811111927032, "data_wait_seconds": 1.5348196029663086e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 69260, "total_steps": 72479, "tokens": 36312186880, "loss": 4.239379912614822, "l_mix": 2.6281242966651917, "l_a": 2.709862619638443, "l_c": 2.6609892547130585, "z_loss": 47.62573194503784, "lr": 0.0004628036259534153, "wd": 0.01, "grad_norm": 0.08583525568246841, "tokens_per_second": 1460420.3412897838, "eta_seconds": 1155.6146023750305, "data_wait_seconds": 1.5437602996826172e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 69270, "total_steps": 72479, "tokens": 36317429760, "loss": 4.236328542232513, "l_mix": 2.6260228753089905, "l_a": 2.7082601189613342, "l_c": 2.6594251692295074, "z_loss": 47.20242214202881, "lr": 0.00046014150413416314, "wd": 0.01, "grad_norm": 0.09495433419942856, "tokens_per_second": 1401089.8411178375, "eta_seconds": 1200.8082155942916, "data_wait_seconds": 1.7195940017700195e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 69280, "total_steps": 72479, "tokens": 36322672640, "loss": 4.689688444137573, "l_mix": 2.9106539487838745, "l_a": 2.984488546848297, "l_c": 2.9456260204315186, "z_loss": 44.96996355056763, "lr": 0.0004574851446747122, "wd": 0.01, "grad_norm": 0.0919574722647667, "tokens_per_second": 1363113.2654263396, "eta_seconds": 1230.4166898965836, "data_wait_seconds": 1.5556812286376953e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 69290, "total_steps": 72479, "tokens": 36327915520, "loss": 4.647329151630402, "l_mix": 2.884273946285248, "l_a": 2.9553185403347015, "l_c": 2.9215316474437714, "z_loss": 46.37519645690918, "lr": 0.00045483456976352373, "wd": 0.01, "grad_norm": 0.09349801391363144, "tokens_per_second": 1396837.4990508403, "eta_seconds": 1196.957006907463, "data_wait_seconds": 1.7344951629638672e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 69300, "total_steps": 72479, "tokens": 36333158400, "loss": 2.961494266986847, "l_mix": 1.8217603862285614, "l_a": 1.934365227818489, "l_c": 1.8647474497556686, "z_loss": 52.86290884017944, "lr": 0.00045218980154074174, "wd": 0.01, "grad_norm": 0.0965811088681221, "tokens_per_second": 1422907.3861131272, "eta_seconds": 1171.342259001732, "data_wait_seconds": 1.8209218978881836e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16817883, "gate_a2b/L1_absmax": 0.67271155, "gate_a2b/L1_rms": 0.17734802, "gate_a2b/L3_absmean": 0.1573367, "gate_a2b/L3_absmax": 0.50513113, "gate_a2b/L3_rms": 0.16277684, "gate_a2b/L5_absmean": 0.22533782, "gate_a2b/L5_absmax": 0.73580623, "gate_a2b/L5_rms": 0.23121187, "gate_c2b/L1_absmean": 0.04826589, "gate_c2b/L1_absmax": 0.59629649, "gate_c2b/L1_rms": 0.07267742, "gate_c2b/L3_absmean": 0.1630711, "gate_c2b/L3_absmax": 0.38338804, "gate_c2b/L3_rms": 0.16650929, "gate_c2b/L5_absmean": 0.46987495, "gate_c2b/L5_absmax": 0.68550414, "gate_c2b/L5_rms": 0.47859794, "gate_b2a/L7_absmean": 0.05303552, "gate_b2a/L7_absmax": 0.3384687, "gate_b2a/L7_rms": 0.05867882, "gate_b2a/L11_absmean": 0.08666667, "gate_b2a/L11_absmax": 0.61133105, "gate_b2a/L11_rms": 0.09534881, "gate_b2a/L14_absmean": 0.22392243, "gate_b2a/L14_absmax": 0.79465687, "gate_b2a/L14_rms": 0.23231927, "gate_b2c/L3_absmean": 0.04011461, "gate_b2c/L3_absmax": 0.5535748, "gate_b2c/L3_rms": 0.04962248, "gate_b2c/L5_absmean": 0.04045002, "gate_b2c/L5_absmax": 0.53860891, "gate_b2c/L5_rms": 0.04915646, "gate_b2c/L6_absmean": 0.05822393, "gate_b2c/L6_absmax": 0.32969138, "gate_b2c/L6_rms": 0.06347965}, "streams": {"cos_a2b/round0": 0.35238, "cos_c2b/round0": 0.208093, "cos_b2a/round0": 0.161984, "cos_b2c/round0": 0.130689, "rms_a/round0": 7.061004, "rms_b/round0": 12.195171, "rms_c/round0": 4.923802, "cos_a2b/round1": 0.406409, "cos_c2b/round1": 0.306353, "cos_b2a/round1": 0.251355, "cos_b2c/round1": 0.112561, "rms_a/round1": 14.376645, "rms_b/round1": 30.073465, "rms_c/round1": 11.335245, "cos_a2b/round2": 0.512987, "cos_c2b/round2": 0.515679, "cos_b2a/round2": 0.090697, "cos_b2c/round2": 0.171907, "rms_a/round2": 42.381367, "rms_b/round2": 82.368263, "rms_c/round2": 22.805895}, "alpha_mean": 0.3889362289902465, "alpha_hist": [0.06836, 0.19675, 0.17834, 0.12971, 0.10306, 0.09726, 0.09555, 0.08099, 0.0448, 0.00517], "alpha_by_type": {"special": {"share": 0.00104, "mean": 0.658, "std": 0.12523}, "whitespace": {"share": 0.03889, "mean": 0.60969, "std": 0.17043}, "digit": {"share": 0.00874, "mean": 0.38778, "std": 0.19605}, "word_start": {"share": 0.55708, "mean": 0.3897, "std": 0.22129}, "word_cont": {"share": 0.26709, "mean": 0.24922, "std": 0.14799}, "punct": {"share": 0.12715, "mean": 0.60941, "std": 0.19764}}, "alpha_p05": 0.08725, "alpha_p50": 0.32424, "alpha_p95": 0.7988}
{"step": 69310, "total_steps": 72479, "tokens": 36338401280, "loss": 2.986345022916794, "l_mix": 1.8375591784715652, "l_a": 1.9474925696849823, "l_c": 1.881793573498726, "z_loss": 53.004117012023926, "lr": 0.0004495508620980073, "wd": 0.01, "grad_norm": 0.08736313879489899, "tokens_per_second": 1420558.759833664, "eta_seconds": 1169.5881360054016, "data_wait_seconds": 1.8149614334106445e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 69320, "total_steps": 72479, "tokens": 36343644160, "loss": 3.00437131524086, "l_mix": 1.8483435809612274, "l_a": 1.961566299200058, "l_c": 1.89185930788517, "z_loss": 52.629857540130615, "lr": 0.00044691777347827347, "wd": 0.01, "grad_norm": 0.09661377221345901, "tokens_per_second": 1407293.1929980128, "eta_seconds": 1176.887517285347, "data_wait_seconds": 2.4884939193725586e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 69330, "total_steps": 72479, "tokens": 36348887040, "loss": 2.66399347782135, "l_mix": 1.649780511856079, "l_a": 1.7077327817678452, "l_c": 1.672976940870285, "z_loss": 61.446173667907715, "lr": 0.0004442905576756224, "wd": 0.01, "grad_norm": 0.10190774500370026, "tokens_per_second": 1407143.8868128208, "eta_seconds": 1173.2864900827408, "data_wait_seconds": 2.7507543563842773e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 69340, "total_steps": 72479, "tokens": 36354129920, "loss": 4.307093143463135, "l_mix": 2.6698904037475586, "l_a": 2.7536531388759613, "l_c": 2.7036890983581543, "z_loss": 47.25889348983765, "lr": 0.00044166923663507916, "wd": 0.01, "grad_norm": 0.11231692135334015, "tokens_per_second": 1466529.9624121475, "eta_seconds": 1122.2000737667083, "data_wait_seconds": 1.9669532775878906e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 69350, "total_steps": 72479, "tokens": 36359372800, "loss": 4.328922390937805, "l_mix": 2.6834557950496674, "l_a": 2.7661127150058746, "l_c": 2.7187761068344116, "z_loss": 48.0919451713562, "lr": 0.0004390538322524317, "wd": 0.01, "grad_norm": 0.10027610510587692, "tokens_per_second": 1430842.3463640332, "eta_seconds": 1146.5254408836365, "data_wait_seconds": 1.570582389831543e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 69360, "total_steps": 72479, "tokens": 36364615680, "loss": 4.210652947425842, "l_mix": 2.6107215583324432, "l_a": 2.6900925040245056, "l_c": 2.64301198720932, "z_loss": 47.05772924423218, "lr": 0.0004364443663740458, "wd": 0.01, "grad_norm": 0.08743616193532944, "tokens_per_second": 1411585.6095485087, "eta_seconds": 1158.4520704507827, "data_wait_seconds": 1.4901161193847656e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 69370, "total_steps": 72479, "tokens": 36369858560, "loss": 4.17496532201767, "l_mix": 2.588191658258438, "l_a": 2.669165313243866, "l_c": 2.6200799345970154, "z_loss": 47.76563882827759, "lr": 0.00043384086079668383, "wd": 0.01, "grad_norm": 0.08663062751293182, "tokens_per_second": 1480447.199087478, "eta_seconds": 1101.0263608217238, "data_wait_seconds": 1.52587890625e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 69380, "total_steps": 72479, "tokens": 36375101440, "loss": 4.172587811946869, "l_mix": 2.586158573627472, "l_a": 2.670153945684433, "l_c": 2.6179431676864624, "z_loss": 48.13660383224487, "lr": 0.0004312433372673204, "wd": 0.01, "grad_norm": 0.08738424628973007, "tokens_per_second": 1381735.5316741455, "eta_seconds": 1175.8896509170534, "data_wait_seconds": 2.35140323638916e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 69390, "total_steps": 72479, "tokens": 36380344320, "loss": 2.830491989850998, "l_mix": 1.7522003948688507, "l_a": 1.8170287013053894, "l_c": 1.7772764712572098, "z_loss": 60.28015375137329, "lr": 0.0004286518174829642, "wd": 0.01, "grad_norm": 0.10713066905736923, "tokens_per_second": 1216433.8505963976, "eta_seconds": 1331.3717233419418, "data_wait_seconds": 2.3484230041503906e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 69400, "total_steps": 72479, "tokens": 36385587200, "loss": 4.671264052391052, "l_mix": 2.8991075456142426, "l_a": 2.9723197519779205, "l_c": 2.9348683059215546, "z_loss": 45.97805881500244, "lr": 0.0004260663230904726, "wd": 0.01, "grad_norm": 0.09222643077373505, "tokens_per_second": 1403262.0012104043, "eta_seconds": 1150.3787251472475, "data_wait_seconds": 1.4334917068481445e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.16843222, "gate_a2b/L1_absmax": 0.67216069, "gate_a2b/L1_rms": 0.17760117, "gate_a2b/L3_absmean": 0.15754233, "gate_a2b/L3_absmax": 0.50566947, "gate_a2b/L3_rms": 0.16295841, "gate_a2b/L5_absmean": 0.22535422, "gate_a2b/L5_absmax": 0.73544097, "gate_a2b/L5_rms": 0.23124698, "gate_c2b/L1_absmean": 0.04814029, "gate_c2b/L1_absmax": 0.59558123, "gate_c2b/L1_rms": 0.07254591, "gate_c2b/L3_absmean": 0.16298914, "gate_c2b/L3_absmax": 0.38264698, "gate_c2b/L3_rms": 0.16641052, "gate_c2b/L5_absmean": 0.46994123, "gate_c2b/L5_absmax": 0.68532729, "gate_c2b/L5_rms": 0.47867072, "gate_b2a/L7_absmean": 0.05277825, "gate_b2a/L7_absmax": 0.3377741, "gate_b2a/L7_rms": 0.05846938, "gate_b2a/L11_absmean": 0.0867043, "gate_b2a/L11_absmax": 0.61077356, "gate_b2a/L11_rms": 0.09534621, "gate_b2a/L14_absmean": 0.22392926, "gate_b2a/L14_absmax": 0.79261619, "gate_b2a/L14_rms": 0.2323055, "gate_b2c/L3_absmean": 0.04023975, "gate_b2c/L3_absmax": 0.55335075, "gate_b2c/L3_rms": 0.04971453, "gate_b2c/L5_absmean": 0.04051905, "gate_b2c/L5_absmax": 0.53805923, "gate_b2c/L5_rms": 0.04921466, "gate_b2c/L6_absmean": 0.05834495, "gate_b2c/L6_absmax": 0.32911891, "gate_b2c/L6_rms": 0.06356446}, "streams": {"cos_a2b/round0": 0.354255, "cos_c2b/round0": 0.235454, "cos_b2a/round0": 0.182777, "cos_b2c/round0": 0.156647, "rms_a/round0": 6.714876, "rms_b/round0": 11.860946, "rms_c/round0": 4.756129, "cos_a2b/round1": 0.41426, "cos_c2b/round1": 0.316874, "cos_b2a/round1": 0.268463, "cos_b2c/round1": 0.108163, "rms_a/round1": 13.358395, "rms_b/round1": 27.963247, "rms_c/round1": 10.384524, "cos_a2b/round2": 0.550475, "cos_c2b/round2": 0.520572, "cos_b2a/round2": 0.100738, "cos_b2c/round2": 0.187141, "rms_a/round2": 41.158081, "rms_b/round2": 79.817421, "rms_c/round2": 20.708773}, "alpha_mean": 0.4471796410681108, "alpha_hist": [0.00734, 0.06299, 0.16103, 0.18369, 0.1862, 0.18932, 0.13133, 0.06911, 0.00871, 0.00029], "alpha_by_type": {"special": {"share": 0.00046, "mean": 0.57071, "std": 0.16658}, "whitespace": {"share": 0.04684, "mean": 0.54582, "std": 0.11617}, "digit": {"share": 0.02161, "mean": 0.45501, "std": 0.13618}, "word_start": {"share": 0.60446, "mean": 0.46569, "std": 0.16156}, "word_cont": {"share": 0.23425, "mean": 0.326, "std": 0.13086}, "punct": {"share": 0.09238, "mean": 0.58088, "std": 0.16626}}, "alpha_p05": 0.17555, "alpha_p50": 0.44059, "alpha_p95": 0.73867}
{"step": 69410, "total_steps": 72479, "tokens": 36390830080, "loss": 4.312788009643555, "l_mix": 2.67643404006958, "l_a": 2.7435282170772552, "l_c": 2.7109846770763397, "z_loss": 48.160621643066406, "lr": 0.0004234868756863765, "wd": 0.01, "grad_norm": 0.09445928782224655, "tokens_per_second": 1372036.8345579419, "eta_seconds": 1172.7381011009215, "data_wait_seconds": 2.3365020751953125e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 69420, "total_steps": 72479, "tokens": 36396072960, "loss": 4.699494481086731, "l_mix": 2.9168936610221863, "l_a": 2.989266127347946, "l_c": 2.952736258506775, "z_loss": 46.074164390563965, "lr": 0.000420913496816693, "wd": 0.01, "grad_norm": 0.08789406716823578, "tokens_per_second": 1351535.4088403617, "eta_seconds": 1186.6481495857238, "data_wait_seconds": 2.4110078811645508e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 69430, "total_steps": 72479, "tokens": 36401315840, "loss": 4.625065505504608, "l_mix": 2.870949447154999, "l_a": 2.941406935453415, "l_c": 2.9056462347507477, "z_loss": 46.03926134109497, "lr": 0.00041834620797675116, "wd": 0.01, "grad_norm": 0.09622932225465775, "tokens_per_second": 1470620.9984615154, "eta_seconds": 1086.9925791025162, "data_wait_seconds": 1.5795230865478516e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 69440, "total_steps": 72479, "tokens": 36406558720, "loss": 2.9757573306560516, "l_mix": 1.8304939270019531, "l_a": 1.9424240291118622, "l_c": 1.875120371580124, "z_loss": 53.20603370666504, "lr": 0.00041578503061100993, "wd": 0.01, "grad_norm": 0.09505157917737961, "tokens_per_second": 1449501.9082612759, "eta_seconds": 1099.2129247426988, "data_wait_seconds": 2.944469451904297e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 69450, "total_steps": 72479, "tokens": 36411801600, "loss": 4.152241349220276, "l_mix": 2.5735622346401215, "l_a": 2.6544536352157593, "l_c": 2.607810229063034, "z_loss": 48.12433958053589, "lr": 0.000413229986112878, "wd": 0.01, "grad_norm": 0.10787651687860489, "tokens_per_second": 1428310.3018093624, "eta_seconds": 1111.8510802507399, "data_wait_seconds": 1.5974044799804688e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 69460, "total_steps": 72479, "tokens": 36417044480, "loss": 4.340264201164246, "l_mix": 2.6903520226478577, "l_a": 2.7756510972976685, "l_c": 2.7240557968616486, "z_loss": 47.56528425216675, "lr": 0.0004106810958245392, "wd": 0.01, "grad_norm": 0.08429798483848572, "tokens_per_second": 1447710.4596218138, "eta_seconds": 1093.3301348209382, "data_wait_seconds": 1.531839370727539e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 69470, "total_steps": 72479, "tokens": 36422287360, "loss": 4.332993865013123, "l_mix": 2.686078041791916, "l_a": 2.770100384950638, "l_c": 2.7196187376976013, "z_loss": 47.68799686431885, "lr": 0.0004081383810367684, "wd": 0.01, "grad_norm": 0.0994369387626648, "tokens_per_second": 1429450.1683938194, "eta_seconds": 1103.6289525032043, "data_wait_seconds": 1.6361474990844727e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 69480, "total_steps": 72479, "tokens": 36427530240, "loss": 4.2612550258636475, "l_mix": 2.641473174095154, "l_a": 2.7252867817878723, "l_c": 2.673985540866852, "z_loss": 47.99150228500366, "lr": 0.0004056018629887594, "wd": 0.01, "grad_norm": 0.09200192242860794, "tokens_per_second": 1454357.567355752, "eta_seconds": 1081.1232033252716, "data_wait_seconds": 1.6123056411743164e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 69490, "total_steps": 72479, "tokens": 36432773120, "loss": 4.278849363327026, "l_mix": 2.6522665321826935, "l_a": 2.737197518348694, "l_c": 2.684745281934738, "z_loss": 47.43669128417969, "lr": 0.00040307156286794254, "wd": 0.01, "grad_norm": 0.08229126781225204, "tokens_per_second": 1465100.2067654934, "eta_seconds": 1069.6175079107284, "data_wait_seconds": 1.5497207641601562e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 69500, "total_steps": 72479, "tokens": 36438016000, "loss": 2.889818787574768, "l_mix": 1.7887192219495773, "l_a": 1.8567067086696625, "l_c": 1.8136251717805862, "z_loss": 60.49677038192749, "lr": 0.0004005475018098118, "wd": 0.01, "grad_norm": 0.08940742164850235, "tokens_per_second": 1451792.5821326957, "eta_seconds": 1075.810671043396, "data_wait_seconds": 1.5676021575927734e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.16861804, "gate_a2b/L1_absmax": 0.67200446, "gate_a2b/L1_rms": 0.17776848, "gate_a2b/L3_absmean": 0.15779182, "gate_a2b/L3_absmax": 0.50687492, "gate_a2b/L3_rms": 0.16320844, "gate_a2b/L5_absmean": 0.225623, "gate_a2b/L5_absmax": 0.7359764, "gate_a2b/L5_rms": 0.23148972, "gate_c2b/L1_absmean": 0.04851997, "gate_c2b/L1_absmax": 0.59547234, "gate_c2b/L1_rms": 0.07266483, "gate_c2b/L3_absmean": 0.16290247, "gate_c2b/L3_absmax": 0.38238123, "gate_c2b/L3_rms": 0.16629048, "gate_c2b/L5_absmean": 0.46981248, "gate_c2b/L5_absmax": 0.68589193, "gate_c2b/L5_rms": 0.47853655, "gate_b2a/L7_absmean": 0.05291284, "gate_b2a/L7_absmax": 0.33790493, "gate_b2a/L7_rms": 0.05854992, "gate_b2a/L11_absmean": 0.08669737, "gate_b2a/L11_absmax": 0.6104942, "gate_b2a/L11_rms": 0.09534945, "gate_b2a/L14_absmean": 0.22404519, "gate_b2a/L14_absmax": 0.79199183, "gate_b2a/L14_rms": 0.23240606, "gate_b2c/L3_absmean": 0.04035952, "gate_b2c/L3_absmax": 0.55412942, "gate_b2c/L3_rms": 0.04983522, "gate_b2c/L5_absmean": 0.04054496, "gate_b2c/L5_absmax": 0.53796142, "gate_b2c/L5_rms": 0.04922019, "gate_b2c/L6_absmean": 0.05839753, "gate_b2c/L6_absmax": 0.32887799, "gate_b2c/L6_rms": 0.06360519}, "streams": {"cos_a2b/round0": 0.364646, "cos_c2b/round0": 0.239142, "cos_b2a/round0": 0.160377, "cos_b2c/round0": 0.148637, "rms_a/round0": 6.990407, "rms_b/round0": 11.483716, "rms_c/round0": 4.66822, "cos_a2b/round1": 0.380942, "cos_c2b/round1": 0.278243, "cos_b2a/round1": 0.233075, "cos_b2c/round1": 0.088437, "rms_a/round1": 13.786234, "rms_b/round1": 27.812046, "rms_c/round1": 10.379869, "cos_a2b/round2": 0.496927, "cos_c2b/round2": 0.537629, "cos_b2a/round2": 0.077353, "cos_b2c/round2": 0.159942, "rms_a/round2": 40.293182, "rms_b/round2": 81.831047, "rms_c/round2": 20.626066}, "alpha_mean": 0.40290271777831954, "alpha_hist": [0.04909, 0.08656, 0.17633, 0.19693, 0.18025, 0.15155, 0.10359, 0.0444, 0.01019, 0.0011], "alpha_by_type": {"special": {"share": 0.00053, "mean": 0.54369, "std": 0.19278}, "whitespace": {"share": 0.09854, "mean": 0.48185, "std": 0.17609}, "digit": {"share": 0.10446, "mean": 0.37738, "std": 0.16172}, "word_start": {"share": 0.33867, "mean": 0.39115, "std": 0.17744}, "word_cont": {"share": 0.2291, "mean": 0.29918, "std": 0.13518}, "punct": {"share": 0.2287, "mean": 0.50152, "std": 0.18058}}, "alpha_p05": 0.11799, "alpha_p50": 0.38676, "alpha_p95": 0.72894}
{"step": 69510, "total_steps": 72479, "tokens": 36443258880, "loss": 2.870354562997818, "l_mix": 1.7763711661100388, "l_a": 1.846338152885437, "l_c": 1.8002731651067734, "z_loss": 60.33691930770874, "lr": 0.00039802970089774614, "wd": 0.01, "grad_norm": 0.1092567890882492, "tokens_per_second": 1395891.941429019, "eta_seconds": 1115.1372293233871, "data_wait_seconds": 4.2945146560668945e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 69520, "total_steps": 72479, "tokens": 36448501760, "loss": 4.5501933097839355, "l_mix": 2.8235916197299957, "l_a": 2.8957309424877167, "l_c": 2.8596078157424927, "z_loss": 46.18472957611084, "lr": 0.00039551818116283335, "wd": 0.01, "grad_norm": 0.09082477539777756, "tokens_per_second": 1377329.3581518903, "eta_seconds": 1126.3596341848374, "data_wait_seconds": 2.4646520614624023e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 69530, "total_steps": 72479, "tokens": 36453744640, "loss": 4.58187198638916, "l_mix": 2.8434514701366425, "l_a": 2.9157316088676453, "l_c": 2.8790033757686615, "z_loss": 45.93130159378052, "lr": 0.00039301296358369546, "wd": 0.01, "grad_norm": 0.0931420549750328, "tokens_per_second": 1181916.4373771234, "eta_seconds": 1308.1511205911638, "data_wait_seconds": 1.7017126083374023e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 69540, "total_steps": 72479, "tokens": 36458987520, "loss": 4.63491290807724, "l_mix": 2.876614898443222, "l_a": 2.9502206444740295, "l_c": 2.9107726514339447, "z_loss": 46.193878173828125, "lr": 0.0003905140690863119, "wd": 0.01, "grad_norm": 0.09142185002565384, "tokens_per_second": 1462911.4007637661, "eta_seconds": 1053.2985327720642, "data_wait_seconds": 1.850724220275879e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 69550, "total_steps": 72479, "tokens": 36464230400, "loss": 4.594795286655426, "l_mix": 2.8520168364048004, "l_a": 2.922825127840042, "l_c": 2.886436641216278, "z_loss": 45.92797613143921, "lr": 0.00038802151854384627, "wd": 0.01, "grad_norm": 0.08778952807188034, "tokens_per_second": 1439575.1154705095, "eta_seconds": 1066.731103849411, "data_wait_seconds": 1.919269561767578e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 69560, "total_steps": 72479, "tokens": 36469473280, "loss": 4.549955368041992, "l_mix": 2.823828250169754, "l_a": 2.8917833268642426, "l_c": 2.861973285675049, "z_loss": 46.754204750061035, "lr": 0.00038553533277647164, "wd": 0.01, "grad_norm": 0.10111473500728607, "tokens_per_second": 1435091.8573409785, "eta_seconds": 1066.4102539300918, "data_wait_seconds": 2.79843807220459e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 69570, "total_steps": 72479, "tokens": 36474716160, "loss": 4.264928013086319, "l_mix": 2.6436061561107635, "l_a": 2.7267518639564514, "l_c": 2.677654057741165, "z_loss": 47.824225425720215, "lr": 0.0003830555325511961, "wd": 0.01, "grad_norm": 0.08669637143611908, "tokens_per_second": 1424161.9475542204, "eta_seconds": 1070.9131743192672, "data_wait_seconds": 1.6987323760986328e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 69580, "total_steps": 72479, "tokens": 36479959040, "loss": 4.226750910282135, "l_mix": 2.6193848848342896, "l_a": 2.7053324580192566, "l_c": 2.6525541245937347, "z_loss": 47.952839851379395, "lr": 0.0003805821385816889, "wd": 0.01, "grad_norm": 0.08630958199501038, "tokens_per_second": 1393419.111166422, "eta_seconds": 1090.7780005455018, "data_wait_seconds": 1.537799835205078e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 69590, "total_steps": 72479, "tokens": 36485201920, "loss": 4.346535921096802, "l_mix": 2.6951535642147064, "l_a": 2.7741893529891968, "l_c": 2.730418384075165, "z_loss": 47.408203125, "lr": 0.0003781151715281096, "wd": 0.01, "grad_norm": 0.08447138965129852, "tokens_per_second": 991006.7451973747, "eta_seconds": 1528.413443541527, "data_wait_seconds": 1.9609928131103516e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 69600, "total_steps": 72479, "tokens": 36490444800, "loss": 4.285586893558502, "l_mix": 2.65678334236145, "l_a": 2.7401521801948547, "l_c": 2.689193069934845, "z_loss": 47.456944942474365, "lr": 0.00037565465199693166, "wd": 0.01, "grad_norm": 0.08469720184803009, "tokens_per_second": 1356273.2002152004, "eta_seconds": 1112.9211664438249, "data_wait_seconds": 2.6285648345947266e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16873312, "gate_a2b/L1_absmax": 0.67144179, "gate_a2b/L1_rms": 0.17788588, "gate_a2b/L3_absmean": 0.15782522, "gate_a2b/L3_absmax": 0.50677013, "gate_a2b/L3_rms": 0.16327028, "gate_a2b/L5_absmean": 0.22569859, "gate_a2b/L5_absmax": 0.73555309, "gate_a2b/L5_rms": 0.23157875, "gate_c2b/L1_absmean": 0.04868314, "gate_c2b/L1_absmax": 0.59405977, "gate_c2b/L1_rms": 0.07264081, "gate_c2b/L3_absmean": 0.16270196, "gate_c2b/L3_absmax": 0.38231593, "gate_c2b/L3_rms": 0.16609618, "gate_c2b/L5_absmean": 0.46996537, "gate_c2b/L5_absmax": 0.68605655, "gate_c2b/L5_rms": 0.47866055, "gate_b2a/L7_absmean": 0.05292843, "gate_b2a/L7_absmax": 0.33765239, "gate_b2a/L7_rms": 0.05855761, "gate_b2a/L11_absmean": 0.08672322, "gate_b2a/L11_absmax": 0.6104219, "gate_b2a/L11_rms": 0.09536983, "gate_b2a/L14_absmean": 0.22408646, "gate_b2a/L14_absmax": 0.7924096, "gate_b2a/L14_rms": 0.23243921, "gate_b2c/L3_absmean": 0.04028951, "gate_b2c/L3_absmax": 0.55453813, "gate_b2c/L3_rms": 0.0497882, "gate_b2c/L5_absmean": 0.04063402, "gate_b2c/L5_absmax": 0.53788334, "gate_b2c/L5_rms": 0.04930065, "gate_b2c/L6_absmean": 0.05853323, "gate_b2c/L6_absmax": 0.32923093, "gate_b2c/L6_rms": 0.06374147}, "streams": {"cos_a2b/round0": 0.352672, "cos_c2b/round0": 0.229135, "cos_b2a/round0": 0.175513, "cos_b2c/round0": 0.152057, "rms_a/round0": 6.749035, "rms_b/round0": 11.912494, "rms_c/round0": 4.815138, "cos_a2b/round1": 0.41167, "cos_c2b/round1": 0.310282, "cos_b2a/round1": 0.262309, "cos_b2c/round1": 0.105801, "rms_a/round1": 13.567084, "rms_b/round1": 28.634697, "rms_c/round1": 10.504209, "cos_a2b/round2": 0.535643, "cos_c2b/round2": 0.524635, "cos_b2a/round2": 0.093485, "cos_b2c/round2": 0.183997, "rms_a/round2": 41.587479, "rms_b/round2": 81.703575, "rms_c/round2": 21.142628}, "alpha_mean": 0.4189194396198488, "alpha_hist": [0.01295, 0.09503, 0.19983, 0.19444, 0.16499, 0.13777, 0.11417, 0.06999, 0.0106, 0.00021], "alpha_by_type": {"special": {"share": 0.00108, "mean": 0.55163, "std": 0.11313}, "whitespace": {"share": 0.02275, "mean": 0.52043, "std": 0.1324}, "digit": {"share": 0.02795, "mean": 0.45202, "std": 0.12681}, "word_start": {"share": 0.58781, "mean": 0.43881, "std": 0.1749}, "word_cont": {"share": 0.26518, "mean": 0.30486, "std": 0.13117}, "punct": {"share": 0.09521, "mean": 0.57833, "std": 0.17624}}, "alpha_p05": 0.14953, "alpha_p50": 0.41111, "alpha_p95": 0.74017}
{"step": 69610, "total_steps": 72479, "tokens": 36495687680, "loss": 2.6757186949253082, "l_mix": 1.6567702144384384, "l_a": 1.717845380306244, "l_c": 1.6786492466926575, "z_loss": 59.080410957336426, "lr": 0.00037320060054077587, "wd": 0.01, "grad_norm": 0.10342151671648026, "tokens_per_second": 1459532.4551399292, "eta_seconds": 1030.59186296463, "data_wait_seconds": 2.4527311325073242e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 69620, "total_steps": 72479, "tokens": 36500930560, "loss": 2.7808235585689545, "l_mix": 1.7219616174697876, "l_a": 1.7831848710775375, "l_c": 1.7463547736406326, "z_loss": 60.018985748291016, "lr": 0.0003707530376582324, "wd": 0.01, "grad_norm": 0.0890364721417427, "tokens_per_second": 1426543.3349324202, "eta_seconds": 1050.7492869615555, "data_wait_seconds": 2.822279930114746e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 69630, "total_steps": 72479, "tokens": 36506173440, "loss": 4.691569924354553, "l_mix": 2.9120167791843414, "l_a": 2.984705686569214, "l_c": 2.9471375942230225, "z_loss": 46.44357490539551, "lr": 0.0003683119837936943, "wd": 0.01, "grad_norm": 0.08771460503339767, "tokens_per_second": 1308603.3455450502, "eta_seconds": 1141.4432930231094, "data_wait_seconds": 3.382563591003418e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 69640, "total_steps": 72479, "tokens": 36511416320, "loss": 4.405050992965698, "l_mix": 2.7313910126686096, "l_a": 2.813822090625763, "l_c": 2.765044152736664, "z_loss": 47.155166149139404, "lr": 0.00036587745933718517, "wd": 0.01, "grad_norm": 0.09930964559316635, "tokens_per_second": 1440986.0562332473, "eta_seconds": 1032.9410375356674, "data_wait_seconds": 2.8699636459350586e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 69650, "total_steps": 72479, "tokens": 36516659200, "loss": 4.353434205055237, "l_mix": 2.6989274322986603, "l_a": 2.7824436128139496, "l_c": 2.7325787246227264, "z_loss": 47.90921354293823, "lr": 0.00036344948462418693, "wd": 0.01, "grad_norm": 0.10057477653026581, "tokens_per_second": 984992.022747297, "eta_seconds": 1505.8099129199982, "data_wait_seconds": 2.1338462829589844e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 69660, "total_steps": 72479, "tokens": 36521902080, "loss": 4.232246994972229, "l_mix": 2.6232901215553284, "l_a": 2.706051081418991, "l_c": 2.657138377428055, "z_loss": 48.35427808761597, "lr": 0.00036102807993547447, "wd": 0.01, "grad_norm": 0.08825588971376419, "tokens_per_second": 1212381.3622688968, "eta_seconds": 1219.061854624748, "data_wait_seconds": 1.704692840576172e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 69670, "total_steps": 72479, "tokens": 36527144960, "loss": 4.336380183696747, "l_mix": 2.68894624710083, "l_a": 2.768898159265518, "l_c": 2.722547799348831, "z_loss": 48.551390647888184, "lr": 0.0003586132654969413, "wd": 0.01, "grad_norm": 0.11958961188793182, "tokens_per_second": 1416138.2643480368, "eta_seconds": 1039.9584765672685, "data_wait_seconds": 1.5735626220703125e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 69680, "total_steps": 72479, "tokens": 36532387840, "loss": 4.377219915390015, "l_mix": 2.7136098444461823, "l_a": 2.797828286886215, "l_c": 2.7475383281707764, "z_loss": 48.09497785568237, "lr": 0.0003562050614794342, "wd": 0.01, "grad_norm": 0.09623544663190842, "tokens_per_second": 1371752.5119847732, "eta_seconds": 1069.7863493442535, "data_wait_seconds": 1.4662742614746094e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 69690, "total_steps": 72479, "tokens": 36537630720, "loss": 4.374251127243042, "l_mix": 2.7122325599193573, "l_a": 2.7925057411193848, "l_c": 2.7475558519363403, "z_loss": 47.93336582183838, "lr": 0.0003538034879985821, "wd": 0.01, "grad_norm": 0.08986926078796387, "tokens_per_second": 1467006.1240366912, "eta_seconds": 996.7505984067917, "data_wait_seconds": 2.193450927734375e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 69700, "total_steps": 72479, "tokens": 36542873600, "loss": 4.366901099681854, "l_mix": 2.7075726687908173, "l_a": 2.789640337228775, "l_c": 2.74145445227623, "z_loss": 47.910523891448975, "lr": 0.00035140856511463054, "wd": 0.01, "grad_norm": 0.08910411596298218, "tokens_per_second": 1475424.6803344074, "eta_seconds": 987.5098142385482, "data_wait_seconds": 1.5437602996826172e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16880439, "gate_a2b/L1_absmax": 0.6715191, "gate_a2b/L1_rms": 0.17794806, "gate_a2b/L3_absmean": 0.15788779, "gate_a2b/L3_absmax": 0.50728559, "gate_a2b/L3_rms": 0.16331567, "gate_a2b/L5_absmean": 0.22597261, "gate_a2b/L5_absmax": 0.73649156, "gate_a2b/L5_rms": 0.23185825, "gate_c2b/L1_absmean": 0.04873775, "gate_c2b/L1_absmax": 0.5943265, "gate_c2b/L1_rms": 0.07270044, "gate_c2b/L3_absmean": 0.1627627, "gate_c2b/L3_absmax": 0.38186279, "gate_c2b/L3_rms": 0.16615897, "gate_c2b/L5_absmean": 0.46978053, "gate_c2b/L5_absmax": 0.68615818, "gate_c2b/L5_rms": 0.4784956, "gate_b2a/L7_absmean": 0.05296222, "gate_b2a/L7_absmax": 0.3380304, "gate_b2a/L7_rms": 0.05861536, "gate_b2a/L11_absmean": 0.08680829, "gate_b2a/L11_absmax": 0.61052948, "gate_b2a/L11_rms": 0.09544852, "gate_b2a/L14_absmean": 0.22423226, "gate_b2a/L14_absmax": 0.79165822, "gate_b2a/L14_rms": 0.232563, "gate_b2c/L3_absmean": 0.04042174, "gate_b2c/L3_absmax": 0.55496269, "gate_b2c/L3_rms": 0.04989393, "gate_b2c/L5_absmean": 0.04075475, "gate_b2c/L5_absmax": 0.53881687, "gate_b2c/L5_rms": 0.04942766, "gate_b2c/L6_absmean": 0.05865107, "gate_b2c/L6_absmax": 0.3294991, "gate_b2c/L6_rms": 0.06384761}, "streams": {"cos_a2b/round0": 0.355694, "cos_c2b/round0": 0.23738, "cos_b2a/round0": 0.176121, "cos_b2c/round0": 0.154672, "rms_a/round0": 6.770139, "rms_b/round0": 11.888373, "rms_c/round0": 4.805099, "cos_a2b/round1": 0.41074, "cos_c2b/round1": 0.305623, "cos_b2a/round1": 0.262245, "cos_b2c/round1": 0.103587, "rms_a/round1": 13.590154, "rms_b/round1": 28.535955, "rms_c/round1": 10.543355, "cos_a2b/round2": 0.533837, "cos_c2b/round2": 0.526138, "cos_b2a/round2": 0.095138, "cos_b2c/round2": 0.179373, "rms_a/round2": 41.598755, "rms_b/round2": 81.8004, "rms_c/round2": 21.392572}, "alpha_mean": 0.4188208818500385, "alpha_hist": [0.0126, 0.08896, 0.19096, 0.20775, 0.1743, 0.14044, 0.11031, 0.06454, 0.00977, 0.00037], "alpha_by_type": {"special": {"share": 0.00131, "mean": 0.54729, "std": 0.11969}, "whitespace": {"share": 0.02858, "mean": 0.52071, "std": 0.13413}, "digit": {"share": 0.04205, "mean": 0.44653, "std": 0.12866}, "word_start": {"share": 0.56036, "mean": 0.43268, "std": 0.1715}, "word_cont": {"share": 0.26207, "mean": 0.31152, "std": 0.13039}, "punct": {"share": 0.10562, "mean": 0.57132, "std": 0.16979}}, "alpha_p05": 0.15078, "alpha_p50": 0.39187, "alpha_p95": 0.72873}
{"step": 69710, "total_steps": 72479, "tokens": 36548116480, "loss": 4.2937270402908325, "l_mix": 2.662106931209564, "l_a": 2.742301881313324, "l_c": 2.696431666612625, "z_loss": 48.2300329208374, "lr": 0.0003490203128322723, "wd": 0.01, "grad_norm": 0.09099408239126205, "tokens_per_second": 1397622.4751791165, "eta_seconds": 1038.7307715654374, "data_wait_seconds": 2.5272369384765625e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 69720, "total_steps": 72479, "tokens": 36553359360, "loss": 4.3765310645103455, "l_mix": 2.713452309370041, "l_a": 2.794354200363159, "l_c": 2.7492414116859436, "z_loss": 47.899675369262695, "lr": 0.0003466387511004814, "wd": 0.01, "grad_norm": 0.09375074505805969, "tokens_per_second": 1427431.1823828164, "eta_seconds": 1013.3662553071975, "data_wait_seconds": 1.7970800399780273e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 69730, "total_steps": 72479, "tokens": 36558602240, "loss": 2.7969446182250977, "l_mix": 1.7294387221336365, "l_a": 1.802814245223999, "l_c": 1.7555385828018188, "z_loss": 60.8773889541626, "lr": 0.0003442638998123448, "wd": 0.01, "grad_norm": 0.11538845300674438, "tokens_per_second": 1435924.0870760232, "eta_seconds": 1003.7213840007782, "data_wait_seconds": 3.0100345611572266e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 69740, "total_steps": 72479, "tokens": 36563845120, "loss": 4.328172922134399, "l_mix": 2.6830885112285614, "l_a": 2.7663497924804688, "l_c": 2.7172646820545197, "z_loss": 47.78020763397217, "lr": 0.00034189577880489876, "wd": 0.01, "grad_norm": 0.07994059473276138, "tokens_per_second": 1334834.2252683449, "eta_seconds": 1075.8076207637787, "data_wait_seconds": 2.956390380859375e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 69750, "total_steps": 72479, "tokens": 36569088000, "loss": 4.376858174800873, "l_mix": 2.7141180634498596, "l_a": 2.793247312307358, "l_c": 2.7492193579673767, "z_loss": 48.2208890914917, "lr": 0.00033953440785896086, "wd": 0.01, "grad_norm": 0.08461001515388489, "tokens_per_second": 1403742.2232781919, "eta_seconds": 1019.2626026868819, "data_wait_seconds": 1.52587890625e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 69760, "total_steps": 72479, "tokens": 36574330880, "loss": 4.30641108751297, "l_mix": 2.6697959899902344, "l_a": 2.7523700892925262, "l_c": 2.7030131220817566, "z_loss": 48.132405281066895, "lr": 0.0003371798066989664, "wd": 0.01, "grad_norm": 0.0811474621295929, "tokens_per_second": 1446375.745996742, "eta_seconds": 985.5938720941543, "data_wait_seconds": 1.6361474990844727e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 69770, "total_steps": 72479, "tokens": 36579573760, "loss": 4.34790176153183, "l_mix": 2.6963380575180054, "l_a": 2.772618919610977, "l_c": 2.732593387365341, "z_loss": 47.97774696350098, "lr": 0.0003348319949928028, "wd": 0.01, "grad_norm": 0.09354820847511292, "tokens_per_second": 1470166.6663002989, "eta_seconds": 966.0783532619477, "data_wait_seconds": 1.5795230865478516e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 69780, "total_steps": 72479, "tokens": 36584816640, "loss": 4.335411489009857, "l_mix": 2.6878620982170105, "l_a": 2.769701659679413, "l_c": 2.7221294045448303, "z_loss": 48.07322597503662, "lr": 0.00033249099235164423, "wd": 0.01, "grad_norm": 0.08834648877382278, "tokens_per_second": 1425532.7495033538, "eta_seconds": 992.6487571001053, "data_wait_seconds": 1.8835067749023438e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 69790, "total_steps": 72479, "tokens": 36590059520, "loss": 4.343308091163635, "l_mix": 2.6936274766921997, "l_a": 2.7713071405887604, "l_c": 2.727628380060196, "z_loss": 48.42431879043579, "lr": 0.00033015681832979077, "wd": 0.01, "grad_norm": 0.08990220725536346, "tokens_per_second": 1474198.8799639384, "eta_seconds": 956.3230926036836, "data_wait_seconds": 1.5795230865478516e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 69800, "total_steps": 72479, "tokens": 36595302400, "loss": 4.7155845165252686, "l_mix": 2.927277773618698, "l_a": 2.9980679750442505, "l_c": 2.9629543125629425, "z_loss": 46.175071716308594, "lr": 0.00032782949242450224, "wd": 0.01, "grad_norm": 0.09970276057720184, "tokens_per_second": 1343448.9810649008, "eta_seconds": 1045.4937789201738, "data_wait_seconds": 2.2023916244506836e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16893886, "gate_a2b/L1_absmax": 0.67154813, "gate_a2b/L1_rms": 0.17805469, "gate_a2b/L3_absmean": 0.15803163, "gate_a2b/L3_absmax": 0.50708038, "gate_a2b/L3_rms": 0.163459, "gate_a2b/L5_absmean": 0.22617081, "gate_a2b/L5_absmax": 0.73545754, "gate_a2b/L5_rms": 0.23203975, "gate_c2b/L1_absmean": 0.04890584, "gate_c2b/L1_absmax": 0.59360832, "gate_c2b/L1_rms": 0.07270514, "gate_c2b/L3_absmean": 0.16271964, "gate_c2b/L3_absmax": 0.38200939, "gate_c2b/L3_rms": 0.16612558, "gate_c2b/L5_absmean": 0.46972367, "gate_c2b/L5_absmax": 0.68591803, "gate_c2b/L5_rms": 0.47843888, "gate_b2a/L7_absmean": 0.05295634, "gate_b2a/L7_absmax": 0.33813107, "gate_b2a/L7_rms": 0.05859412, "gate_b2a/L11_absmean": 0.0869088, "gate_b2a/L11_absmax": 0.6098389, "gate_b2a/L11_rms": 0.09553118, "gate_b2a/L14_absmean": 0.22424296, "gate_b2a/L14_absmax": 0.79136789, "gate_b2a/L14_rms": 0.23256481, "gate_b2c/L3_absmean": 0.04048253, "gate_b2c/L3_absmax": 0.55466723, "gate_b2c/L3_rms": 0.04993472, "gate_b2c/L5_absmean": 0.0407888, "gate_b2c/L5_absmax": 0.53908432, "gate_b2c/L5_rms": 0.04944745, "gate_b2c/L6_absmean": 0.05869962, "gate_b2c/L6_absmax": 0.32913014, "gate_b2c/L6_rms": 0.06389319}, "streams": {"cos_a2b/round0": 0.352849, "cos_c2b/round0": 0.231617, "cos_b2a/round0": 0.175238, "cos_b2c/round0": 0.153085, "rms_a/round0": 6.675651, "rms_b/round0": 11.771018, "rms_c/round0": 4.706358, "cos_a2b/round1": 0.408963, "cos_c2b/round1": 0.310561, "cos_b2a/round1": 0.260619, "cos_b2c/round1": 0.102513, "rms_a/round1": 13.260837, "rms_b/round1": 27.82612, "rms_c/round1": 10.460593, "cos_a2b/round2": 0.540829, "cos_c2b/round2": 0.518053, "cos_b2a/round2": 0.100159, "cos_b2c/round2": 0.181649, "rms_a/round2": 40.733509, "rms_b/round2": 79.819016, "rms_c/round2": 20.8228}, "alpha_mean": 0.4450753610474649, "alpha_hist": [0.00761, 0.06143, 0.16586, 0.19034, 0.1832, 0.18074, 0.13454, 0.06854, 0.00766, 8e-05], "alpha_by_type": {"special": {"share": 0.00043, "mean": 0.56596, "std": 0.11508}, "whitespace": {"share": 0.0638, "mean": 0.52018, "std": 0.12373}, "digit": {"share": 0.03586, "mean": 0.42626, "std": 0.11737}, "word_start": {"share": 0.54504, "mean": 0.46889, "std": 0.16463}, "word_cont": {"share": 0.25412, "mean": 0.32706, "std": 0.1293}, "punct": {"share": 0.10075, "mean": 0.57257, "std": 0.16115}}, "alpha_p05": 0.17526, "alpha_p50": 0.44951, "alpha_p95": 0.70781}
{"step": 69810, "total_steps": 72479, "tokens": 36600545280, "loss": 4.547296643257141, "l_mix": 2.822055160999298, "l_a": 2.8953152000904083, "l_c": 2.85548934340477, "z_loss": 46.14189529418945, "lr": 0.0003255090340758373, "wd": 0.01, "grad_norm": 0.08639567345380783, "tokens_per_second": 1430134.1063684649, "eta_seconds": 978.4569613218307, "data_wait_seconds": 2.0444393157958984e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 69820, "total_steps": 72479, "tokens": 36605788160, "loss": 4.57363760471344, "l_mix": 2.8383291959762573, "l_a": 2.9113222658634186, "l_c": 2.8730389177799225, "z_loss": 47.05522918701172, "lr": 0.0003231954626664889, "wd": 0.01, "grad_norm": 0.10728253424167633, "tokens_per_second": 1464822.5522148097, "eta_seconds": 951.7069421768188, "data_wait_seconds": 4.056096076965332e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 69830, "total_steps": 72479, "tokens": 36611031040, "loss": 4.637792527675629, "l_mix": 2.878364533185959, "l_a": 2.950584441423416, "l_c": 2.9141753911972046, "z_loss": 46.78921365737915, "lr": 0.0003208887975216246, "wd": 0.01, "grad_norm": 0.0840572714805603, "tokens_per_second": 1360356.9427219694, "eta_seconds": 1020.9371293544768, "data_wait_seconds": 2.765655517578125e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 69840, "total_steps": 72479, "tokens": 36616273920, "loss": 2.814994692802429, "l_mix": 1.7432070225477219, "l_a": 1.8055064678192139, "l_c": 1.7671188861131668, "z_loss": 60.90019512176514, "lr": 0.0003185890579087241, "wd": 0.01, "grad_norm": 0.12738415598869324, "tokens_per_second": 1442120.1496693718, "eta_seconds": 959.4180015563965, "data_wait_seconds": 2.2083520889282227e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 69850, "total_steps": 72479, "tokens": 36621516800, "loss": 4.356882750988007, "l_mix": 2.701152116060257, "l_a": 2.7836439311504364, "l_c": 2.7354580760002136, "z_loss": 47.767062187194824, "lr": 0.00031629626303741876, "wd": 0.01, "grad_norm": 0.09218186140060425, "tokens_per_second": 1340859.3357407977, "eta_seconds": 1027.9625276565553, "data_wait_seconds": 1.4603137969970703e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 69860, "total_steps": 72479, "tokens": 36626759680, "loss": 4.351994633674622, "l_mix": 2.698550432920456, "l_a": 2.778933137655258, "l_c": 2.73254731297493, "z_loss": 47.896912574768066, "lr": 0.0003140104320593296, "wd": 0.01, "grad_norm": 0.09168662875890732, "tokens_per_second": 1371908.4383093177, "eta_seconds": 1000.8760305404662, "data_wait_seconds": 2.1964311599731445e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 69870, "total_steps": 72479, "tokens": 36632002560, "loss": 4.336006462574005, "l_mix": 2.6880707442760468, "l_a": 2.768985688686371, "l_c": 2.724133402109146, "z_loss": 48.47606945037842, "lr": 0.0003117315840679096, "wd": 0.01, "grad_norm": 0.09722265601158142, "tokens_per_second": 1410454.8656342681, "eta_seconds": 969.8058586120607, "data_wait_seconds": 2.0742416381835938e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 69880, "total_steps": 72479, "tokens": 36637245440, "loss": 4.2921032309532166, "l_mix": 2.6606869399547577, "l_a": 2.7429131269454956, "l_c": 2.6951408982276917, "z_loss": 48.36054563522339, "lr": 0.00030945973809828317, "wd": 0.01, "grad_norm": 0.08303194493055344, "tokens_per_second": 1398519.6806982476, "eta_seconds": 974.3334547281265, "data_wait_seconds": 1.5467405319213867e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 69890, "total_steps": 72479, "tokens": 36642488320, "loss": 4.304304420948029, "l_mix": 2.6681388318538666, "l_a": 2.7493431866168976, "l_c": 2.704541891813278, "z_loss": 49.399147033691406, "lr": 0.0003071949131270858, "wd": 0.01, "grad_norm": 0.08390241861343384, "tokens_per_second": 1448036.9663347148, "eta_seconds": 937.3943231821061, "data_wait_seconds": 1.5914440155029297e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 69900, "total_steps": 72479, "tokens": 36647731200, "loss": 4.374029159545898, "l_mix": 2.7120444774627686, "l_a": 2.792855829000473, "l_c": 2.7470928132534027, "z_loss": 47.90409851074219, "lr": 0.0003049371280723093, "wd": 0.01, "grad_norm": 0.09104685485363007, "tokens_per_second": 1290919.0327895838, "eta_seconds": 1047.4233609199523, "data_wait_seconds": 1.58846378326416e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.16908203, "gate_a2b/L1_absmax": 0.67094874, "gate_a2b/L1_rms": 0.17820176, "gate_a2b/L3_absmean": 0.15810578, "gate_a2b/L3_absmax": 0.50717121, "gate_a2b/L3_rms": 0.16351256, "gate_a2b/L5_absmean": 0.22622795, "gate_a2b/L5_absmax": 0.73560119, "gate_a2b/L5_rms": 0.23208016, "gate_c2b/L1_absmean": 0.04886835, "gate_c2b/L1_absmax": 0.59256798, "gate_c2b/L1_rms": 0.07261512, "gate_c2b/L3_absmean": 0.16288617, "gate_c2b/L3_absmax": 0.38166231, "gate_c2b/L3_rms": 0.16628937, "gate_c2b/L5_absmean": 0.46965894, "gate_c2b/L5_absmax": 0.68634403, "gate_c2b/L5_rms": 0.47836089, "gate_b2a/L7_absmean": 0.05288373, "gate_b2a/L7_absmax": 0.33795825, "gate_b2a/L7_rms": 0.05852947, "gate_b2a/L11_absmean": 0.08689405, "gate_b2a/L11_absmax": 0.61025691, "gate_b2a/L11_rms": 0.09550974, "gate_b2a/L14_absmean": 0.22423303, "gate_b2a/L14_absmax": 0.79059976, "gate_b2a/L14_rms": 0.23253618, "gate_b2c/L3_absmean": 0.04056381, "gate_b2c/L3_absmax": 0.55508196, "gate_b2c/L3_rms": 0.0500104, "gate_b2c/L5_absmean": 0.04079225, "gate_b2c/L5_absmax": 0.53938669, "gate_b2c/L5_rms": 0.04946399, "gate_b2c/L6_absmean": 0.05873166, "gate_b2c/L6_absmax": 0.32799631, "gate_b2c/L6_rms": 0.06390685}, "streams": {"cos_a2b/round0": 0.354945, "cos_c2b/round0": 0.234406, "cos_b2a/round0": 0.177822, "cos_b2c/round0": 0.152963, "rms_a/round0": 6.725388, "rms_b/round0": 11.928123, "rms_c/round0": 4.798153, "cos_a2b/round1": 0.408803, "cos_c2b/round1": 0.305351, "cos_b2a/round1": 0.262899, "cos_b2c/round1": 0.103846, "rms_a/round1": 13.589275, "rms_b/round1": 28.454266, "rms_c/round1": 10.578049, "cos_a2b/round2": 0.539071, "cos_c2b/round2": 0.521289, "cos_b2a/round2": 0.097901, "cos_b2c/round2": 0.176021, "rms_a/round2": 41.582756, "rms_b/round2": 80.829315, "rms_c/round2": 21.420963}, "alpha_mean": 0.427375005606919, "alpha_hist": [0.01544, 0.08656, 0.17972, 0.1949, 0.17001, 0.15164, 0.11986, 0.07014, 0.01141, 0.00031], "alpha_by_type": {"special": {"share": 0.00142, "mean": 0.56159, "std": 0.13356}, "whitespace": {"share": 0.03459, "mean": 0.52118, "std": 0.1323}, "digit": {"share": 0.0387, "mean": 0.43263, "std": 0.13863}, "word_start": {"share": 0.55222, "mean": 0.44383, "std": 0.17457}, "word_cont": {"share": 0.25165, "mean": 0.3183, "std": 0.14099}, "punct": {"share": 0.12143, "mean": 0.54862, "std": 0.1826}}, "alpha_p05": 0.17413, "alpha_p50": 0.40263, "alpha_p95": 0.72096}
{"step": 69910, "total_steps": 72479, "tokens": 36652974080, "loss": 4.590977311134338, "l_mix": 2.849519729614258, "l_a": 2.9207784831523895, "l_c": 2.8840799629688263, "z_loss": 46.769046783447266, "lr": 0.00030268640179313867, "wd": 0.01, "grad_norm": 0.0812242180109024, "tokens_per_second": 1483837.279733298, "eta_seconds": 907.7113039255142, "data_wait_seconds": 2.3424625396728516e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 69920, "total_steps": 72479, "tokens": 36658216960, "loss": 4.552518308162689, "l_mix": 2.8246743977069855, "l_a": 2.9006670713424683, "l_c": 2.8588122129440308, "z_loss": 46.516180992126465, "lr": 0.00030044275308979923, "wd": 0.01, "grad_norm": 0.08419745415449142, "tokens_per_second": 1226187.2551321862, "eta_seconds": 1094.1664793729783, "data_wait_seconds": 2.1845102310180664e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 69930, "total_steps": 72479, "tokens": 36663459840, "loss": 4.625854015350342, "l_mix": 2.8708163499832153, "l_a": 2.94486865401268, "l_c": 2.905256927013397, "z_loss": 45.764384269714355, "lr": 0.00029820620070339615, "wd": 0.01, "grad_norm": 0.08546794950962067, "tokens_per_second": 1434794.2844194146, "eta_seconds": 931.4297711610794, "data_wait_seconds": 2.0742416381835938e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 69940, "total_steps": 72479, "tokens": 36668702720, "loss": 4.54463517665863, "l_mix": 2.8202565014362335, "l_a": 2.8928880095481873, "l_c": 2.8550408482551575, "z_loss": 47.82917404174805, "lr": 0.0002959767633157605, "wd": 0.01, "grad_norm": 0.09056097269058228, "tokens_per_second": 1459593.4869711252, "eta_seconds": 912.0123129367829, "data_wait_seconds": 2.130866050720215e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 69950, "total_steps": 72479, "tokens": 36673945600, "loss": 4.364410221576691, "l_mix": 2.7082464396953583, "l_a": 2.77873295545578, "l_c": 2.7418122589588165, "z_loss": 48.18089485168457, "lr": 0.00029375445954929133, "wd": 0.01, "grad_norm": 0.08611045032739639, "tokens_per_second": 1369536.1553733859, "eta_seconds": 968.1557852983475, "data_wait_seconds": 2.4616718292236328e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 69960, "total_steps": 72479, "tokens": 36679188480, "loss": 4.3851693868637085, "l_mix": 2.718787908554077, "l_a": 2.801286369562149, "l_c": 2.75331848859787, "z_loss": 48.54407358169556, "lr": 0.0002915393079668014, "wd": 0.01, "grad_norm": 0.09354464709758759, "tokens_per_second": 1403885.878980316, "eta_seconds": 940.732784461975, "data_wait_seconds": 2.0951032638549805e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 69970, "total_steps": 72479, "tokens": 36684431360, "loss": 4.4200146198272705, "l_mix": 2.7404878437519073, "l_a": 2.8229771554470062, "l_c": 2.7754454612731934, "z_loss": 47.74352407455444, "lr": 0.0002893313270713614, "wd": 0.01, "grad_norm": 0.08529796451330185, "tokens_per_second": 1365028.3409860602, "eta_seconds": 963.6712678432465, "data_wait_seconds": 1.6897916793823242e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 69980, "total_steps": 72479, "tokens": 36689674240, "loss": 4.326773524284363, "l_mix": 2.682512730360031, "l_a": 2.7631926834583282, "l_c": 2.71767657995224, "z_loss": 48.81898212432861, "lr": 0.00028713053530614487, "wd": 0.01, "grad_norm": 0.08130266517400742, "tokens_per_second": 1454463.6684254755, "eta_seconds": 900.8102027177811, "data_wait_seconds": 1.9669532775878906e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 69990, "total_steps": 72479, "tokens": 36694917120, "loss": 4.354675650596619, "l_mix": 2.6996177434921265, "l_a": 2.7832011580467224, "l_c": 2.7336581349372864, "z_loss": 48.09485483169556, "lr": 0.0002849369510542757, "wd": 0.01, "grad_norm": 0.08800418674945831, "tokens_per_second": 1463724.189854535, "eta_seconds": 891.5291836023331, "data_wait_seconds": 2.2679567337036133e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 70000, "total_steps": 72479, "tokens": 36700160000, "loss": 4.344081163406372, "l_mix": 2.6934127509593964, "l_a": 2.774505138397217, "l_c": 2.727722704410553, "z_loss": 47.977538108825684, "lr": 0.00028275059263867356, "wd": 0.01, "grad_norm": 0.09267056733369827, "tokens_per_second": 1427085.005312412, "eta_seconds": 910.7445927619934, "data_wait_seconds": 2.6106834411621094e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16927235, "gate_a2b/L1_absmax": 0.67121691, "gate_a2b/L1_rms": 0.1783798, "gate_a2b/L3_absmean": 0.15825671, "gate_a2b/L3_absmax": 0.50712907, "gate_a2b/L3_rms": 0.16369063, "gate_a2b/L5_absmean": 0.2264196, "gate_a2b/L5_absmax": 0.73510963, "gate_a2b/L5_rms": 0.23226631, "gate_c2b/L1_absmean": 0.04923224, "gate_c2b/L1_absmax": 0.59237856, "gate_c2b/L1_rms": 0.07269543, "gate_c2b/L3_absmean": 0.16264513, "gate_c2b/L3_absmax": 0.38161704, "gate_c2b/L3_rms": 0.16606615, "gate_c2b/L5_absmean": 0.46963724, "gate_c2b/L5_absmax": 0.68643236, "gate_c2b/L5_rms": 0.47833544, "gate_b2a/L7_absmean": 0.0528848, "gate_b2a/L7_absmax": 0.33870387, "gate_b2a/L7_rms": 0.05855449, "gate_b2a/L11_absmean": 0.08689792, "gate_b2a/L11_absmax": 0.61086673, "gate_b2a/L11_rms": 0.09553669, "gate_b2a/L14_absmean": 0.22434404, "gate_b2a/L14_absmax": 0.79061913, "gate_b2a/L14_rms": 0.23264693, "gate_b2c/L3_absmean": 0.04066794, "gate_b2c/L3_absmax": 0.55523193, "gate_b2c/L3_rms": 0.05010875, "gate_b2c/L5_absmean": 0.040849, "gate_b2c/L5_absmax": 0.53908139, "gate_b2c/L5_rms": 0.04950074, "gate_b2c/L6_absmean": 0.05882123, "gate_b2c/L6_absmax": 0.32854223, "gate_b2c/L6_rms": 0.06398513}, "streams": {"cos_a2b/round0": 0.352322, "cos_c2b/round0": 0.227496, "cos_b2a/round0": 0.171441, "cos_b2c/round0": 0.151011, "rms_a/round0": 6.654046, "rms_b/round0": 11.789877, "rms_c/round0": 4.795465, "cos_a2b/round1": 0.407138, "cos_c2b/round1": 0.303274, "cos_b2a/round1": 0.260972, "cos_b2c/round1": 0.100869, "rms_a/round1": 13.259488, "rms_b/round1": 28.435896, "rms_c/round1": 10.366899, "cos_a2b/round2": 0.524002, "cos_c2b/round2": 0.529097, "cos_b2a/round2": 0.093441, "cos_b2c/round2": 0.178466, "rms_a/round2": 40.874409, "rms_b/round2": 81.852066, "rms_c/round2": 20.785286}, "alpha_mean": 0.40759442799247836, "alpha_hist": [0.01483, 0.10515, 0.20071, 0.1993, 0.17203, 0.13885, 0.10329, 0.0582, 0.00758, 6e-05], "alpha_by_type": {"special": {"share": 0.00066, "mean": 0.53499, "std": 0.12982}, "whitespace": {"share": 0.02792, "mean": 0.49229, "std": 0.12992}, "digit": {"share": 0.05115, "mean": 0.44076, "std": 0.09801}, "word_start": {"share": 0.5291, "mean": 0.4321, "std": 0.17395}, "word_cont": {"share": 0.28661, "mean": 0.2939, "std": 0.12771}, "punct": {"share": 0.10457, "mean": 0.55557, "std": 0.16866}}, "alpha_p05": 0.15355, "alpha_p50": 0.38053, "alpha_p95": 0.71721}
{"step": 70010, "total_steps": 72479, "tokens": 36705402880, "loss": 4.3830026388168335, "l_mix": 2.71767395734787, "l_a": 2.799333482980728, "l_c": 2.7517620027065277, "z_loss": 48.11003494262695, "lr": 0.00028057147832190104, "wd": 0.01, "grad_norm": 0.08752851188182831, "tokens_per_second": 1457908.7534806458, "eta_seconds": 887.8930652618408, "data_wait_seconds": 1.946091651916504e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 70020, "total_steps": 72479, "tokens": 36710645760, "loss": 2.899225354194641, "l_mix": 1.795959323644638, "l_a": 1.8584125936031342, "l_c": 1.8191405832767487, "z_loss": 61.00856351852417, "lr": 0.0002783996263060103, "wd": 0.01, "grad_norm": 0.09830516576766968, "tokens_per_second": 1457286.3584144309, "eta_seconds": 884.6745765209198, "data_wait_seconds": 2.4139881134033203e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 70030, "total_steps": 72479, "tokens": 36715888640, "loss": 4.5276166796684265, "l_mix": 2.8094891607761383, "l_a": 2.882046639919281, "l_c": 2.8450450897216797, "z_loss": 47.47295093536377, "lr": 0.0002762350547323931, "wd": 0.01, "grad_norm": 0.09016010910272598, "tokens_per_second": 1388725.9603765751, "eta_seconds": 924.5750051736832, "data_wait_seconds": 2.16066837310791e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 70040, "total_steps": 72479, "tokens": 36721131520, "loss": 4.683228552341461, "l_mix": 2.9066569805145264, "l_a": 2.979549914598465, "l_c": 2.9423550367355347, "z_loss": 46.31675720214844, "lr": 0.00027407778168162623, "wd": 0.01, "grad_norm": 0.10129300504922867, "tokens_per_second": 1465375.13476744, "eta_seconds": 872.6355468034744, "data_wait_seconds": 1.6570091247558594e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 70050, "total_steps": 72479, "tokens": 36726374400, "loss": 4.553651988506317, "l_mix": 2.825503647327423, "l_a": 2.901095360517502, "l_c": 2.8593988716602325, "z_loss": 47.51474618911743, "lr": 0.00027192782517332457, "wd": 0.01, "grad_norm": 0.0969197005033493, "tokens_per_second": 1445378.8660448997, "eta_seconds": 881.0807892084122, "data_wait_seconds": 3.4421682357788086e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 70060, "total_steps": 72479, "tokens": 36731617280, "loss": 4.422668993473053, "l_mix": 2.7439800202846527, "l_a": 2.81780469417572, "l_c": 2.7778250873088837, "z_loss": 48.24251079559326, "lr": 0.0002697852031659859, "wd": 0.01, "grad_norm": 0.08996383100748062, "tokens_per_second": 1402131.4950485006, "eta_seconds": 904.5176408052445, "data_wait_seconds": 2.4616718292236328e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 70070, "total_steps": 72479, "tokens": 36736860160, "loss": 4.217253863811493, "l_mix": 2.6138581037521362, "l_a": 2.6958828270435333, "l_c": 2.648769587278366, "z_loss": 49.309818267822266, "lr": 0.0002676499335568445, "wd": 0.01, "grad_norm": 0.08935606479644775, "tokens_per_second": 1339955.8517257948, "eta_seconds": 942.5756754398346, "data_wait_seconds": 1.564621925354004e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 70080, "total_steps": 72479, "tokens": 36742103040, "loss": 4.339223623275757, "l_mix": 2.690283626317978, "l_a": 2.7707898914813995, "l_c": 2.725676715373993, "z_loss": 48.48825550079346, "lr": 0.0002655220341817207, "wd": 0.01, "grad_norm": 0.08531951159238815, "tokens_per_second": 1334046.064051855, "eta_seconds": 942.8212007761001, "data_wait_seconds": 1.6748905181884766e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 70090, "total_steps": 72479, "tokens": 36747345920, "loss": 4.356656491756439, "l_mix": 2.7011573016643524, "l_a": 2.782008558511734, "l_c": 2.736321985721588, "z_loss": 48.66517877578735, "lr": 0.00026340152281486954, "wd": 0.01, "grad_norm": 0.09213873744010925, "tokens_per_second": 1425354.3258249045, "eta_seconds": 878.7457331180573, "data_wait_seconds": 1.71661376953125e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 70100, "total_steps": 72479, "tokens": 36752588800, "loss": 4.284250438213348, "l_mix": 2.6561608016490936, "l_a": 2.7364283204078674, "l_c": 2.6905367374420166, "z_loss": 48.892133712768555, "lr": 0.00026128841716883727, "wd": 0.01, "grad_norm": 0.08608782291412354, "tokens_per_second": 1445595.5985149506, "eta_seconds": 862.8147133827209, "data_wait_seconds": 1.8149614334106445e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16936302, "gate_a2b/L1_absmax": 0.67094249, "gate_a2b/L1_rms": 0.17846052, "gate_a2b/L3_absmean": 0.15830997, "gate_a2b/L3_absmax": 0.50755799, "gate_a2b/L3_rms": 0.16373736, "gate_a2b/L5_absmean": 0.22646566, "gate_a2b/L5_absmax": 0.73517013, "gate_a2b/L5_rms": 0.2323146, "gate_c2b/L1_absmean": 0.04946426, "gate_c2b/L1_absmax": 0.59205544, "gate_c2b/L1_rms": 0.07279576, "gate_c2b/L3_absmean": 0.1626246, "gate_c2b/L3_absmax": 0.38147992, "gate_c2b/L3_rms": 0.16601463, "gate_c2b/L5_absmean": 0.46952939, "gate_c2b/L5_absmax": 0.68652976, "gate_c2b/L5_rms": 0.47821313, "gate_b2a/L7_absmean": 0.05298673, "gate_b2a/L7_absmax": 0.33811596, "gate_b2a/L7_rms": 0.05861933, "gate_b2a/L11_absmean": 0.08685635, "gate_b2a/L11_absmax": 0.61018449, "gate_b2a/L11_rms": 0.0954885, "gate_b2a/L14_absmean": 0.22435194, "gate_b2a/L14_absmax": 0.79092807, "gate_b2a/L14_rms": 0.23264217, "gate_b2c/L3_absmean": 0.04071148, "gate_b2c/L3_absmax": 0.55551058, "gate_b2c/L3_rms": 0.0501418, "gate_b2c/L5_absmean": 0.04091602, "gate_b2c/L5_absmax": 0.53955942, "gate_b2c/L5_rms": 0.04957926, "gate_b2c/L6_absmean": 0.05888465, "gate_b2c/L6_absmax": 0.32794249, "gate_b2c/L6_rms": 0.06404234}, "streams": {"cos_a2b/round0": 0.356322, "cos_c2b/round0": 0.233114, "cos_b2a/round0": 0.158635, "cos_b2c/round0": 0.147415, "rms_a/round0": 6.748906, "rms_b/round0": 11.771588, "rms_c/round0": 4.73398, "cos_a2b/round1": 0.386638, "cos_c2b/round1": 0.284793, "cos_b2a/round1": 0.238801, "cos_b2c/round1": 0.095293, "rms_a/round1": 13.602866, "rms_b/round1": 28.673275, "rms_c/round1": 10.412927, "cos_a2b/round2": 0.528687, "cos_c2b/round2": 0.505359, "cos_b2a/round2": 0.087565, "cos_b2c/round2": 0.168157, "rms_a/round2": 41.510307, "rms_b/round2": 82.529572, "rms_c/round2": 20.719872}, "alpha_mean": 0.42465627973691156, "alpha_hist": [0.02258, 0.09471, 0.17967, 0.18265, 0.16193, 0.15321, 0.12242, 0.06718, 0.01408, 0.00156], "alpha_by_type": {"special": {"share": 0.00082, "mean": 0.54488, "std": 0.1469}, "whitespace": {"share": 0.03484, "mean": 0.56124, "std": 0.14472}, "digit": {"share": 0.12125, "mean": 0.46445, "std": 0.16012}, "word_start": {"share": 0.44754, "mean": 0.43085, "std": 0.17571}, "word_cont": {"share": 0.24995, "mean": 0.29515, "std": 0.13266}, "punct": {"share": 0.1456, "mean": 0.56146, "std": 0.18308}}, "alpha_p05": 0.13478, "alpha_p50": 0.43446, "alpha_p95": 0.74798}
{"step": 70110, "total_steps": 72479, "tokens": 36757831680, "loss": 4.350756943225861, "l_mix": 2.6983343362808228, "l_a": 2.773548275232315, "l_c": 2.7345269918441772, "z_loss": 48.11585569381714, "lr": 0.000259182734894308, "wd": 0.01, "grad_norm": 0.09526285529136658, "tokens_per_second": 1424865.2065540256, "eta_seconds": 871.6882595539092, "data_wait_seconds": 1.7970800399780273e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 70120, "total_steps": 72479, "tokens": 36763074560, "loss": 4.3597646951675415, "l_mix": 2.704206347465515, "l_a": 2.780401051044464, "l_c": 2.7381263375282288, "z_loss": 48.42002534866333, "lr": 0.00025708449357996014, "wd": 0.01, "grad_norm": 0.08679313957691193, "tokens_per_second": 1405488.9647286718, "eta_seconds": 879.9751709461212, "data_wait_seconds": 1.58846378326416e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 70130, "total_steps": 72479, "tokens": 36768317440, "loss": 2.9240353405475616, "l_mix": 1.8110360950231552, "l_a": 1.8756546080112457, "l_c": 1.8343426287174225, "z_loss": 60.30533313751221, "lr": 0.00025499371075231677, "wd": 0.01, "grad_norm": 0.21512621641159058, "tokens_per_second": 1471631.937938076, "eta_seconds": 836.8617724657059, "data_wait_seconds": 2.396106719970703e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 70140, "total_steps": 72479, "tokens": 36773560320, "loss": 2.713605135679245, "l_mix": 1.680408164858818, "l_a": 1.7422651052474976, "l_c": 1.7017246186733246, "z_loss": 60.623459815979004, "lr": 0.0002529104038756026, "wd": 0.01, "grad_norm": 0.11623281985521317, "tokens_per_second": 1134842.5679600432, "eta_seconds": 1080.5989012241364, "data_wait_seconds": 2.148747444152832e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 70150, "total_steps": 72479, "tokens": 36778803200, "loss": 4.621314585208893, "l_mix": 2.867963880300522, "l_a": 2.9436636865139008, "l_c": 2.9008383452892303, "z_loss": 46.468050956726074, "lr": 0.0002508345903515937, "wd": 0.01, "grad_norm": 0.09483733028173447, "tokens_per_second": 1467772.3293280504, "eta_seconds": 831.9183619976044, "data_wait_seconds": 2.1278858184814453e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 70160, "total_steps": 72479, "tokens": 36784046080, "loss": 4.258672297000885, "l_mix": 2.6407129764556885, "l_a": 2.7187574803829193, "l_c": 2.6744399070739746, "z_loss": 48.91541051864624, "lr": 0.0002487662875194759, "wd": 0.01, "grad_norm": 0.10687964409589767, "tokens_per_second": 1457386.7056536132, "eta_seconds": 834.2493226289748, "data_wait_seconds": 2.154707908630371e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 70170, "total_steps": 72479, "tokens": 36789288960, "loss": 4.317159593105316, "l_mix": 2.676833540201187, "l_a": 2.756437361240387, "l_c": 2.711316019296646, "z_loss": 49.10866069793701, "lr": 0.00024670551265569833, "wd": 0.01, "grad_norm": 0.0822749063372612, "tokens_per_second": 1395601.9871232803, "eta_seconds": 867.425672340393, "data_wait_seconds": 1.8328428268432617e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 70180, "total_steps": 72479, "tokens": 36794531840, "loss": 4.298174321651459, "l_mix": 2.664582282304764, "l_a": 2.7451213002204895, "l_c": 2.7001854181289673, "z_loss": 49.00266361236572, "lr": 0.00024465228297382876, "wd": 0.01, "grad_norm": 0.08614008128643036, "tokens_per_second": 1400415.8252014231, "eta_seconds": 860.700150847435, "data_wait_seconds": 1.7851591110229492e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 70190, "total_steps": 72479, "tokens": 36799774720, "loss": 4.3567328453063965, "l_mix": 2.7016303837299347, "l_a": 2.780230402946472, "l_c": 2.736777752637863, "z_loss": 48.90747022628784, "lr": 0.000242606615624411, "wd": 0.01, "grad_norm": 0.09909281134605408, "tokens_per_second": 1344716.6802739059, "eta_seconds": 892.4521050453186, "data_wait_seconds": 1.582503318786621e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 70200, "total_steps": 72479, "tokens": 36805017600, "loss": 4.325077474117279, "l_mix": 2.6810276210308075, "l_a": 2.7642791271209717, "l_c": 2.715886950492859, "z_loss": 48.52349853515625, "lr": 0.0002405685276948207, "wd": 0.01, "grad_norm": 0.08575043827295303, "tokens_per_second": 1435892.302002242, "eta_seconds": 832.1322917699814, "data_wait_seconds": 2.962350845336914e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.1694856, "gate_a2b/L1_absmax": 0.67159641, "gate_a2b/L1_rms": 0.1785816, "gate_a2b/L3_absmean": 0.15840068, "gate_a2b/L3_absmax": 0.50817877, "gate_a2b/L3_rms": 0.16383919, "gate_a2b/L5_absmean": 0.22660862, "gate_a2b/L5_absmax": 0.73506445, "gate_a2b/L5_rms": 0.23245427, "gate_c2b/L1_absmean": 0.04963433, "gate_c2b/L1_absmax": 0.59161305, "gate_c2b/L1_rms": 0.07280532, "gate_c2b/L3_absmean": 0.16255935, "gate_c2b/L3_absmax": 0.38125327, "gate_c2b/L3_rms": 0.16595341, "gate_c2b/L5_absmean": 0.46949235, "gate_c2b/L5_absmax": 0.6867516, "gate_c2b/L5_rms": 0.47817898, "gate_b2a/L7_absmean": 0.05294022, "gate_b2a/L7_absmax": 0.3377474, "gate_b2a/L7_rms": 0.05857439, "gate_b2a/L11_absmean": 0.08695633, "gate_b2a/L11_absmax": 0.61016846, "gate_b2a/L11_rms": 0.09556655, "gate_b2a/L14_absmean": 0.22443616, "gate_b2a/L14_absmax": 0.78985852, "gate_b2a/L14_rms": 0.23271731, "gate_b2c/L3_absmean": 0.04075574, "gate_b2c/L3_absmax": 0.55538452, "gate_b2c/L3_rms": 0.05020361, "gate_b2c/L5_absmean": 0.04094107, "gate_b2c/L5_absmax": 0.53955179, "gate_b2c/L5_rms": 0.04959625, "gate_b2c/L6_absmean": 0.05894602, "gate_b2c/L6_absmax": 0.3278451, "gate_b2c/L6_rms": 0.06410304}, "streams": {"cos_a2b/round0": 0.352168, "cos_c2b/round0": 0.232641, "cos_b2a/round0": 0.175847, "cos_b2c/round0": 0.152205, "rms_a/round0": 6.733989, "rms_b/round0": 11.919245, "rms_c/round0": 4.763443, "cos_a2b/round1": 0.409057, "cos_c2b/round1": 0.305678, "cos_b2a/round1": 0.258644, "cos_b2c/round1": 0.103776, "rms_a/round1": 13.679053, "rms_b/round1": 28.564404, "rms_c/round1": 10.613948, "cos_a2b/round2": 0.538159, "cos_c2b/round2": 0.517337, "cos_b2a/round2": 0.099494, "cos_b2c/round2": 0.178088, "rms_a/round2": 41.642239, "rms_b/round2": 81.4496, "rms_c/round2": 21.556021}, "alpha_mean": 0.4321922854697746, "alpha_hist": [0.02138, 0.09222, 0.16876, 0.1758, 0.16824, 0.15721, 0.12505, 0.07591, 0.01476, 0.00067], "alpha_by_type": {"special": {"share": 0.00153, "mean": 0.53, "std": 0.15383}, "whitespace": {"share": 0.02673, "mean": 0.51262, "std": 0.13609}, "digit": {"share": 0.02673, "mean": 0.449, "std": 0.13581}, "word_start": {"share": 0.57251, "mean": 0.4547, "std": 0.17793}, "word_cont": {"share": 0.26407, "mean": 0.30867, "std": 0.14605}, "punct": {"share": 0.10843, "mean": 0.58882, "std": 0.16624}}, "alpha_p05": 0.1373, "alpha_p50": 0.40993, "alpha_p95": 0.73488}
{"step": 70210, "total_steps": 72479, "tokens": 36810260480, "loss": 4.371106922626495, "l_mix": 2.7103624641895294, "l_a": 2.79073965549469, "l_c": 2.7450750172138214, "z_loss": 48.77180337905884, "lr": 0.00023853803620912337, "wd": 0.01, "grad_norm": 0.0840344950556755, "tokens_per_second": 1440114.846177518, "eta_seconds": 826.0518077135085, "data_wait_seconds": 1.6927719116210938e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 70220, "total_steps": 72479, "tokens": 36815503360, "loss": 4.308101832866669, "l_mix": 2.6707232892513275, "l_a": 2.7520673274993896, "l_c": 2.705860733985901, "z_loss": 49.05335569381714, "lr": 0.0002365151581279305, "wd": 0.01, "grad_norm": 0.0840735137462616, "tokens_per_second": 1404962.3945263908, "eta_seconds": 842.9881088733673, "data_wait_seconds": 1.7076730728149414e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 70230, "total_steps": 72479, "tokens": 36820746240, "loss": 4.312858402729034, "l_mix": 2.6735485792160034, "l_a": 2.7559628188610077, "l_c": 2.7084028124809265, "z_loss": 49.304858684539795, "lr": 0.0002344999103482605, "wd": 0.01, "grad_norm": 0.08342863619327545, "tokens_per_second": 1332171.6012442436, "eta_seconds": 885.1139829874039, "data_wait_seconds": 2.5719404220581055e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 70240, "total_steps": 72479, "tokens": 36825989120, "loss": 2.4400806799530983, "l_mix": 1.5112811736762524, "l_a": 1.5630398988723755, "l_c": 1.5329585149884224, "z_loss": 62.454301834106445, "lr": 0.00023249230970339563, "wd": 0.01, "grad_norm": 0.10571742802858353, "tokens_per_second": 1436025.5461946053, "eta_seconds": 817.4512181282043, "data_wait_seconds": 6.327033042907715e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 70250, "total_steps": 72479, "tokens": 36831232000, "loss": 2.4732392840087414, "l_mix": 1.531614787876606, "l_a": 1.5871717855334282, "l_c": 1.5515764877200127, "z_loss": 62.73900508880615, "lr": 0.00023049237296274215, "wd": 0.01, "grad_norm": 0.1299264281988144, "tokens_per_second": 1374789.8563412933, "eta_seconds": 850.0484249353409, "data_wait_seconds": 2.586841583251953e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 70260, "total_steps": 72479, "tokens": 36836474880, "loss": 4.255963087081909, "l_mix": 2.638338267803192, "l_a": 2.720748156309128, "l_c": 2.671334385871887, "z_loss": 49.13856744766235, "lr": 0.00022850011683168942, "wd": 0.01, "grad_norm": 0.08403609693050385, "tokens_per_second": 1423868.7974477082, "eta_seconds": 817.0662030696868, "data_wait_seconds": 1.58846378326416e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 70270, "total_steps": 72479, "tokens": 36841717760, "loss": 4.376485109329224, "l_mix": 2.713758736848831, "l_a": 2.7940956950187683, "l_c": 2.748325139284134, "z_loss": 48.256572246551514, "lr": 0.00022651555795147156, "wd": 0.01, "grad_norm": 0.09704700112342834, "tokens_per_second": 1482800.3089316702, "eta_seconds": 781.0574256181717, "data_wait_seconds": 1.6033649444580078e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 70280, "total_steps": 72479, "tokens": 36846960640, "loss": 4.292378604412079, "l_mix": 2.6611260771751404, "l_a": 2.7424125373363495, "l_c": 2.69509556889534, "z_loss": 49.08181667327881, "lr": 0.00022453871289902774, "wd": 0.01, "grad_norm": 0.0893888995051384, "tokens_per_second": 835330.3769074125, "eta_seconds": 1380.1836301803587, "data_wait_seconds": 1.6480684280395508e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 70290, "total_steps": 72479, "tokens": 36852203520, "loss": 4.249949336051941, "l_mix": 2.6344085335731506, "l_a": 2.7163952589035034, "l_c": 2.6687404215335846, "z_loss": 49.32137584686279, "lr": 0.0002225695981868639, "wd": 0.01, "grad_norm": 0.08518630266189575, "tokens_per_second": 1473984.1568500963, "eta_seconds": 778.6151748418808, "data_wait_seconds": 1.963973045349121e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 70300, "total_steps": 72479, "tokens": 36857446400, "loss": 4.3900551199913025, "l_mix": 2.7220757603645325, "l_a": 2.803139865398407, "l_c": 2.756791204214096, "z_loss": 48.096869468688965, "lr": 0.0002206082302629151, "wd": 0.01, "grad_norm": 0.0934659019112587, "tokens_per_second": 1316430.5530973265, "eta_seconds": 867.819080400467, "data_wait_seconds": 1.913309097290039e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.16946578, "gate_a2b/L1_absmax": 0.67142719, "gate_a2b/L1_rms": 0.17854445, "gate_a2b/L3_absmean": 0.15839981, "gate_a2b/L3_absmax": 0.50732023, "gate_a2b/L3_rms": 0.16382277, "gate_a2b/L5_absmean": 0.22670408, "gate_a2b/L5_absmax": 0.73443204, "gate_a2b/L5_rms": 0.23255908, "gate_c2b/L1_absmean": 0.0496945, "gate_c2b/L1_absmax": 0.59149987, "gate_c2b/L1_rms": 0.07282877, "gate_c2b/L3_absmean": 0.16244273, "gate_c2b/L3_absmax": 0.38094756, "gate_c2b/L3_rms": 0.16585009, "gate_c2b/L5_absmean": 0.46947852, "gate_c2b/L5_absmax": 0.68647856, "gate_c2b/L5_rms": 0.47816342, "gate_b2a/L7_absmean": 0.05295061, "gate_b2a/L7_absmax": 0.33848014, "gate_b2a/L7_rms": 0.05859451, "gate_b2a/L11_absmean": 0.08704624, "gate_b2a/L11_absmax": 0.61039376, "gate_b2a/L11_rms": 0.0956486, "gate_b2a/L14_absmean": 0.22458869, "gate_b2a/L14_absmax": 0.78951836, "gate_b2a/L14_rms": 0.23284358, "gate_b2c/L3_absmean": 0.04084748, "gate_b2c/L3_absmax": 0.55599076, "gate_b2c/L3_rms": 0.05027036, "gate_b2c/L5_absmean": 0.04095794, "gate_b2c/L5_absmax": 0.53949958, "gate_b2c/L5_rms": 0.0496038, "gate_b2c/L6_absmean": 0.05903327, "gate_b2c/L6_absmax": 0.32831168, "gate_b2c/L6_rms": 0.06417791}, "streams": {"cos_a2b/round0": 0.355541, "cos_c2b/round0": 0.234521, "cos_b2a/round0": 0.177085, "cos_b2c/round0": 0.153594, "rms_a/round0": 6.677639, "rms_b/round0": 11.818713, "rms_c/round0": 4.786352, "cos_a2b/round1": 0.41259, "cos_c2b/round1": 0.311812, "cos_b2a/round1": 0.266148, "cos_b2c/round1": 0.099863, "rms_a/round1": 13.459126, "rms_b/round1": 28.161776, "rms_c/round1": 10.536312, "cos_a2b/round2": 0.535448, "cos_c2b/round2": 0.529186, "cos_b2a/round2": 0.097071, "cos_b2c/round2": 0.174525, "rms_a/round2": 41.13599, "rms_b/round2": 80.857903, "rms_c/round2": 21.329062}, "alpha_mean": 0.42121871935108857, "alpha_hist": [0.0141, 0.08012, 0.1924, 0.20738, 0.16887, 0.15068, 0.11832, 0.06055, 0.00735, 0.00023], "alpha_by_type": {"special": {"share": 0.00114, "mean": 0.55027, "std": 0.1372}, "whitespace": {"share": 0.02861, "mean": 0.51617, "std": 0.12563}, "digit": {"share": 0.02676, "mean": 0.41678, "std": 0.14507}, "word_start": {"share": 0.55901, "mean": 0.44042, "std": 0.17019}, "word_cont": {"share": 0.27548, "mean": 0.31479, "std": 0.12834}, "punct": {"share": 0.10899, "mean": 0.56654, "std": 0.16048}}, "alpha_p05": 0.17781, "alpha_p50": 0.39794, "alpha_p95": 0.71542}
{"step": 70310, "total_steps": 72479, "tokens": 36862689280, "loss": 4.297030091285706, "l_mix": 2.663812577724457, "l_a": 2.7464066445827484, "l_c": 2.6976514160633087, "z_loss": 49.02507543563843, "lr": 0.0002186546255104066, "wd": 0.01, "grad_norm": 0.08410182595252991, "tokens_per_second": 1459287.700970625, "eta_seconds": 779.2710589170456, "data_wait_seconds": 2.0802021026611328e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 70320, "total_steps": 72479, "tokens": 36867932160, "loss": 4.5792845487594604, "l_mix": 2.8420841097831726, "l_a": 2.9128548204898834, "l_c": 2.877813220024109, "z_loss": 46.92212772369385, "lr": 0.0002167088002477196, "wd": 0.01, "grad_norm": 0.08788225054740906, "tokens_per_second": 1418669.2156113123, "eta_seconds": 797.8870476245879, "data_wait_seconds": 1.7434358596801758e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 70330, "total_steps": 72479, "tokens": 36873175040, "loss": 4.664469838142395, "l_mix": 2.89534530043602, "l_a": 2.966949611902237, "l_c": 2.9301320016384125, "z_loss": 46.31653547286987, "lr": 0.0002147707707282521, "wd": 0.01, "grad_norm": 0.0826188400387764, "tokens_per_second": 1460650.825349988, "eta_seconds": 771.3649918556213, "data_wait_seconds": 1.7017126083374023e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 70340, "total_steps": 72479, "tokens": 36878417920, "loss": 4.52342289686203, "l_mix": 2.8070741295814514, "l_a": 2.8791806995868683, "l_c": 2.8419816493988037, "z_loss": 48.08614921569824, "lr": 0.00021284055314028667, "wd": 0.01, "grad_norm": 0.08749882131814957, "tokens_per_second": 1269656.1540472128, "eta_seconds": 883.2722374677658, "data_wait_seconds": 1.5735626220703125e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 70350, "total_steps": 72479, "tokens": 36883660800, "loss": 2.627452939748764, "l_mix": 1.6263567805290222, "l_a": 1.688418373465538, "l_c": 1.6485686004161835, "z_loss": 62.140034675598145, "lr": 0.00021091816360685087, "wd": 0.01, "grad_norm": 0.09753495454788208, "tokens_per_second": 1174174.257231574, "eta_seconds": 950.6333026170731, "data_wait_seconds": 1.901388168334961e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 70360, "total_steps": 72479, "tokens": 36888903680, "loss": 2.8449560403823853, "l_mix": 1.7622475773096085, "l_a": 1.8237170577049255, "l_c": 1.7853109687566757, "z_loss": 61.14054489135742, "lr": 0.00020900361818558663, "wd": 0.01, "grad_norm": 0.09257984161376953, "tokens_per_second": 1385374.951987721, "eta_seconds": 801.9246128320694, "data_wait_seconds": 2.5480985641479492e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 70370, "total_steps": 72479, "tokens": 36894146560, "loss": 4.360794723033905, "l_mix": 2.7038250267505646, "l_a": 2.7848212718963623, "l_c": 2.7384107410907745, "z_loss": 48.475542068481445, "lr": 0.0002070969328686127, "wd": 0.01, "grad_norm": 0.08056536316871643, "tokens_per_second": 1420686.2362362382, "eta_seconds": 778.3023188352585, "data_wait_seconds": 1.8149614334106445e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 70380, "total_steps": 72479, "tokens": 36899389440, "loss": 4.332042396068573, "l_mix": 2.6860586404800415, "l_a": 2.764654368162155, "l_c": 2.721957892179489, "z_loss": 49.387547969818115, "lr": 0.00020519812358239396, "wd": 0.01, "grad_norm": 0.08282311260700226, "tokens_per_second": 1354937.7102176873, "eta_seconds": 812.2000765800476, "data_wait_seconds": 1.5616416931152344e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 70390, "total_steps": 72479, "tokens": 36904632320, "loss": 4.402941644191742, "l_mix": 2.7303333580493927, "l_a": 2.8116932809352875, "l_c": 2.763667047023773, "z_loss": 48.428903579711914, "lr": 0.00020330720618760666, "wd": 0.01, "grad_norm": 0.09190429002046585, "tokens_per_second": 1080359.8191715905, "eta_seconds": 1013.771164536476, "data_wait_seconds": 2.5063753128051758e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 70400, "total_steps": 72479, "tokens": 36909875200, "loss": 4.356176435947418, "l_mix": 2.700646460056305, "l_a": 2.781275689601898, "l_c": 2.7371576130390167, "z_loss": 48.57382678985596, "lr": 0.00020142419647900664, "wd": 0.01, "grad_norm": 0.08723582327365875, "tokens_per_second": 1467266.593016312, "eta_seconds": 742.8743741512299, "data_wait_seconds": 2.3692846298217773e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16967204, "gate_a2b/L1_absmax": 0.67119461, "gate_a2b/L1_rms": 0.1787387, "gate_a2b/L3_absmean": 0.15853293, "gate_a2b/L3_absmax": 0.50782752, "gate_a2b/L3_rms": 0.1639536, "gate_a2b/L5_absmean": 0.22681837, "gate_a2b/L5_absmax": 0.73461628, "gate_a2b/L5_rms": 0.23266454, "gate_c2b/L1_absmean": 0.04984934, "gate_c2b/L1_absmax": 0.59122485, "gate_c2b/L1_rms": 0.0728889, "gate_c2b/L3_absmean": 0.16243184, "gate_c2b/L3_absmax": 0.38084561, "gate_c2b/L3_rms": 0.16584447, "gate_c2b/L5_absmean": 0.46944237, "gate_c2b/L5_absmax": 0.68688929, "gate_c2b/L5_rms": 0.47811836, "gate_b2a/L7_absmean": 0.05299507, "gate_b2a/L7_absmax": 0.33820465, "gate_b2a/L7_rms": 0.05862432, "gate_b2a/L11_absmean": 0.08698881, "gate_b2a/L11_absmax": 0.61009675, "gate_b2a/L11_rms": 0.09559523, "gate_b2a/L14_absmean": 0.22460517, "gate_b2a/L14_absmax": 0.78889662, "gate_b2a/L14_rms": 0.23286897, "gate_b2c/L3_absmean": 0.04091832, "gate_b2c/L3_absmax": 0.5558303, "gate_b2c/L3_rms": 0.05032605, "gate_b2c/L5_absmean": 0.04107371, "gate_b2c/L5_absmax": 0.53966403, "gate_b2c/L5_rms": 0.04970137, "gate_b2c/L6_absmean": 0.05904833, "gate_b2c/L6_absmax": 0.32803816, "gate_b2c/L6_rms": 0.0641901}, "streams": {"cos_a2b/round0": 0.354119, "cos_c2b/round0": 0.234062, "cos_b2a/round0": 0.179407, "cos_b2c/round0": 0.153785, "rms_a/round0": 6.673022, "rms_b/round0": 11.867578, "rms_c/round0": 4.756845, "cos_a2b/round1": 0.409225, "cos_c2b/round1": 0.309456, "cos_b2a/round1": 0.265164, "cos_b2c/round1": 0.104404, "rms_a/round1": 13.504194, "rms_b/round1": 28.3466, "rms_c/round1": 10.587057, "cos_a2b/round2": 0.538732, "cos_c2b/round2": 0.522631, "cos_b2a/round2": 0.098131, "cos_b2c/round2": 0.178823, "rms_a/round2": 41.5508, "rms_b/round2": 80.99247, "rms_c/round2": 21.526051}, "alpha_mean": 0.4253913440897108, "alpha_hist": [0.01624, 0.09253, 0.17992, 0.18889, 0.16991, 0.15308, 0.11745, 0.07059, 0.01108, 0.00034], "alpha_by_type": {"special": {"share": 0.0014, "mean": 0.56503, "std": 0.14045}, "whitespace": {"share": 0.02979, "mean": 0.50032, "std": 0.13597}, "digit": {"share": 0.03232, "mean": 0.43787, "std": 0.12602}, "word_start": {"share": 0.57028, "mean": 0.4441, "std": 0.17444}, "word_cont": {"share": 0.26108, "mean": 0.31135, "std": 0.14168}, "punct": {"share": 0.10513, "mean": 0.58017, "std": 0.1691}}, "alpha_p05": 0.14416, "alpha_p50": 0.41632, "alpha_p95": 0.75274}
{"step": 70410, "total_steps": 72479, "tokens": 36915118080, "loss": 4.376031577587128, "l_mix": 2.7132620811462402, "l_a": 2.794760823249817, "l_c": 2.747803717851639, "z_loss": 48.39692258834839, "lr": 0.00019954911018529712, "wd": 0.01, "grad_norm": 0.0846535861492157, "tokens_per_second": 1434287.3464529777, "eta_seconds": 756.3002453327179, "data_wait_seconds": 3.555417060852051e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 70420, "total_steps": 72479, "tokens": 36920360960, "loss": 4.13175967335701, "l_mix": 2.563897356390953, "l_a": 2.62957164645195, "l_c": 2.596635803580284, "z_loss": 50.61572599411011, "lr": 0.00019768196296899668, "wd": 0.01, "grad_norm": 0.17276184260845184, "tokens_per_second": 1365405.3382698516, "eta_seconds": 790.6143045902253, "data_wait_seconds": 1.9490718841552734e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 70430, "total_steps": 72479, "tokens": 36925603840, "loss": 4.337329387664795, "l_mix": 2.6917752623558044, "l_a": 2.7586451172828674, "l_c": 2.7265352308750153, "z_loss": 49.29127550125122, "lr": 0.00019582277042630986, "wd": 0.01, "grad_norm": 0.10063377767801285, "tokens_per_second": 1450067.2771031456, "eta_seconds": 740.8388072490693, "data_wait_seconds": 2.1755695343017578e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 70440, "total_steps": 72479, "tokens": 36930846720, "loss": 4.549171328544617, "l_mix": 2.823158025741577, "l_a": 2.8951238095760345, "l_c": 2.858253538608551, "z_loss": 47.65469694137573, "lr": 0.00019397154808699604, "wd": 0.01, "grad_norm": 0.08019162714481354, "tokens_per_second": 1444613.079392441, "eta_seconds": 740.0066130161285, "data_wait_seconds": 1.9222497940063477e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 70450, "total_steps": 72479, "tokens": 36936089600, "loss": 4.489865243434906, "l_mix": 2.7859539687633514, "l_a": 2.860342025756836, "l_c": 2.8193618953227997, "z_loss": 48.251455783843994, "lr": 0.00019212831141423995, "wd": 0.01, "grad_norm": 0.08473463356494904, "tokens_per_second": 1445857.0751876521, "eta_seconds": 735.7437814950943, "data_wait_seconds": 2.4706125259399414e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 70460, "total_steps": 72479, "tokens": 36941332480, "loss": 4.457374572753906, "l_mix": 2.7658756971359253, "l_a": 2.8383569419384003, "l_c": 2.7999722063541412, "z_loss": 47.40648603439331, "lr": 0.0001902930758045216, "wd": 0.01, "grad_norm": 0.08349652588367462, "tokens_per_second": 1426015.854062601, "eta_seconds": 742.3041398763656, "data_wait_seconds": 2.5838613510131836e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 70470, "total_steps": 72479, "tokens": 36946575360, "loss": 2.8495509326457977, "l_mix": 1.7654360085725784, "l_a": 1.8253839612007141, "l_c": 1.788332223892212, "z_loss": 62.138415813446045, "lr": 0.00018846585658748925, "wd": 0.01, "grad_norm": 0.09160996973514557, "tokens_per_second": 1342267.6438434545, "eta_seconds": 784.7127931833268, "data_wait_seconds": 1.627206802368164e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 70480, "total_steps": 72479, "tokens": 36951818240, "loss": 4.4543843269348145, "l_mix": 2.7628906965255737, "l_a": 2.8403456807136536, "l_c": 2.7979663014411926, "z_loss": 48.69053888320923, "lr": 0.00018664666902583018, "wd": 0.01, "grad_norm": 0.08169307559728622, "tokens_per_second": 1476428.9539708837, "eta_seconds": 709.855837750435, "data_wait_seconds": 2.065300941467285e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 70490, "total_steps": 72479, "tokens": 36957061120, "loss": 4.362063646316528, "l_mix": 2.704772025346756, "l_a": 2.785538613796234, "l_c": 2.7387665808200836, "z_loss": 48.77180814743042, "lr": 0.00018483552831514402, "wd": 0.01, "grad_norm": 0.08967774361371994, "tokens_per_second": 1278887.5665888311, "eta_seconds": 815.4030575037003, "data_wait_seconds": 1.5616416931152344e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 70500, "total_steps": 72479, "tokens": 36962304000, "loss": 4.331589221954346, "l_mix": 2.6854593753814697, "l_a": 2.7678831815719604, "l_c": 2.719216138124466, "z_loss": 48.52221155166626, "lr": 0.0001830324495838146, "wd": 0.01, "grad_norm": 0.08179713040590286, "tokens_per_second": 1469820.7716911903, "eta_seconds": 705.913246011734, "data_wait_seconds": 1.8984079360961914e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16954951, "gate_a2b/L1_absmax": 0.67143697, "gate_a2b/L1_rms": 0.17864658, "gate_a2b/L3_absmean": 0.15857847, "gate_a2b/L3_absmax": 0.50778109, "gate_a2b/L3_rms": 0.16400422, "gate_a2b/L5_absmean": 0.22693913, "gate_a2b/L5_absmax": 0.73464429, "gate_a2b/L5_rms": 0.2327738, "gate_c2b/L1_absmean": 0.04990728, "gate_c2b/L1_absmax": 0.59151614, "gate_c2b/L1_rms": 0.07286258, "gate_c2b/L3_absmean": 0.16245408, "gate_c2b/L3_absmax": 0.38128257, "gate_c2b/L3_rms": 0.16585882, "gate_c2b/L5_absmean": 0.4694007, "gate_c2b/L5_absmax": 0.68755245, "gate_c2b/L5_rms": 0.47806939, "gate_b2a/L7_absmean": 0.05301923, "gate_b2a/L7_absmax": 0.33814618, "gate_b2a/L7_rms": 0.05864307, "gate_b2a/L11_absmean": 0.08706495, "gate_b2a/L11_absmax": 0.61037737, "gate_b2a/L11_rms": 0.09566236, "gate_b2a/L14_absmean": 0.22464216, "gate_b2a/L14_absmax": 0.78925419, "gate_b2a/L14_rms": 0.23289575, "gate_b2c/L3_absmean": 0.04093632, "gate_b2c/L3_absmax": 0.55584413, "gate_b2c/L3_rms": 0.05033109, "gate_b2c/L5_absmean": 0.04102982, "gate_b2c/L5_absmax": 0.53955281, "gate_b2c/L5_rms": 0.04966245, "gate_b2c/L6_absmean": 0.05907058, "gate_b2c/L6_absmax": 0.3281464, "gate_b2c/L6_rms": 0.06422334}, "streams": {"cos_a2b/round0": 0.354611, "cos_c2b/round0": 0.23363, "cos_b2a/round0": 0.177594, "cos_b2c/round0": 0.15295, "rms_a/round0": 6.667616, "rms_b/round0": 11.824008, "rms_c/round0": 4.790926, "cos_a2b/round1": 0.409814, "cos_c2b/round1": 0.307461, "cos_b2a/round1": 0.263088, "cos_b2c/round1": 0.101325, "rms_a/round1": 13.474002, "rms_b/round1": 28.34, "rms_c/round1": 10.552678, "cos_a2b/round2": 0.534819, "cos_c2b/round2": 0.525364, "cos_b2a/round2": 0.096685, "cos_b2c/round2": 0.179861, "rms_a/round2": 41.267487, "rms_b/round2": 81.121506, "rms_c/round2": 21.39922}, "alpha_mean": 0.41771796236510284, "alpha_hist": [0.0127, 0.08861, 0.19423, 0.21071, 0.17212, 0.13794, 0.10841, 0.06532, 0.00975, 0.00021], "alpha_by_type": {"special": {"share": 0.00133, "mean": 0.54437, "std": 0.15788}, "whitespace": {"share": 0.02747, "mean": 0.50667, "std": 0.13372}, "digit": {"share": 0.03241, "mean": 0.46168, "std": 0.12964}, "word_start": {"share": 0.55272, "mean": 0.43222, "std": 0.17227}, "word_cont": {"share": 0.27454, "mean": 0.31068, "std": 0.1275}, "punct": {"share": 0.11154, "mean": 0.57314, "std": 0.16379}}, "alpha_p05": 0.15155, "alpha_p50": 0.40946, "alpha_p95": 0.72798}
{"step": 70510, "total_steps": 72479, "tokens": 36967546880, "loss": 4.383374035358429, "l_mix": 2.7176287472248077, "l_a": 2.801347255706787, "l_c": 2.7511367201805115, "z_loss": 48.71473932266235, "lr": 0.0001812374478928851, "wd": 0.01, "grad_norm": 0.08426406234502792, "tokens_per_second": 1425571.1011801192, "eta_seconds": 724.1470251083374, "data_wait_seconds": 2.7060508728027344e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 70520, "total_steps": 72479, "tokens": 36972789760, "loss": 4.367426633834839, "l_mix": 2.707891047000885, "l_a": 2.789119392633438, "l_c": 2.742666006088257, "z_loss": 48.14975166320801, "lr": 0.00017945053823593103, "wd": 0.01, "grad_norm": 0.08946697413921356, "tokens_per_second": 1308738.468940971, "eta_seconds": 784.7864308834077, "data_wait_seconds": 1.9073486328125e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 70530, "total_steps": 72479, "tokens": 36978032640, "loss": 2.677424192428589, "l_mix": 1.6576693654060364, "l_a": 1.7184436470270157, "l_c": 1.6807389855384827, "z_loss": 61.81204795837402, "lr": 0.00017767173553893576, "wd": 0.01, "grad_norm": 0.08959250897169113, "tokens_per_second": 1429837.191155832, "eta_seconds": 714.6529117584229, "data_wait_seconds": 3.5822391510009766e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 70540, "total_steps": 72479, "tokens": 36983275520, "loss": 4.587490618228912, "l_mix": 2.8468457460403442, "l_a": 2.920369476079941, "l_c": 2.8817798793315887, "z_loss": 46.9475622177124, "lr": 0.0001759010546601659, "wd": 0.01, "grad_norm": 0.07947085797786713, "tokens_per_second": 1482394.880036154, "eta_seconds": 685.7784290075302, "data_wait_seconds": 1.9043684005737305e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 70550, "total_steps": 72479, "tokens": 36988518400, "loss": 4.558785915374756, "l_mix": 2.8290749192237854, "l_a": 2.9016618132591248, "l_c": 2.8640411496162415, "z_loss": 47.58950710296631, "lr": 0.00017413851039004553, "wd": 0.01, "grad_norm": 0.08136703819036484, "tokens_per_second": 1433630.18023129, "eta_seconds": 705.4480060100556, "data_wait_seconds": 2.6404857635498047e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 70560, "total_steps": 72479, "tokens": 36993761280, "loss": 4.549991965293884, "l_mix": 2.8231753408908844, "l_a": 2.8979865312576294, "l_c": 2.8580684065818787, "z_loss": 47.27672529220581, "lr": 0.00017238411745103575, "wd": 0.01, "grad_norm": 0.08521509170532227, "tokens_per_second": 1464968.6371132522, "eta_seconds": 686.7783012628555, "data_wait_seconds": 3.483891487121582e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 70570, "total_steps": 72479, "tokens": 36999004160, "loss": 4.54828268289566, "l_mix": 2.822496920824051, "l_a": 2.8958382308483124, "l_c": 2.856780558824539, "z_loss": 47.86443042755127, "lr": 0.00017063789049750797, "wd": 0.01, "grad_norm": 0.08161856979131699, "tokens_per_second": 1440767.7773250446, "eta_seconds": 694.6753028154374, "data_wait_seconds": 3.504753112792969e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 70580, "total_steps": 72479, "tokens": 37004247040, "loss": 4.462487161159515, "l_mix": 2.768989622592926, "l_a": 2.841339260339737, "l_c": 2.8036522567272186, "z_loss": 48.59638786315918, "lr": 0.00016889984411562578, "wd": 0.01, "grad_norm": 0.18629474937915802, "tokens_per_second": 1384178.0090443336, "eta_seconds": 719.2882024526597, "data_wait_seconds": 2.5838613510131836e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 70590, "total_steps": 72479, "tokens": 37009489920, "loss": 4.3323333859443665, "l_mix": 2.68673175573349, "l_a": 2.764892578125, "l_c": 2.7204459607601166, "z_loss": 48.00653696060181, "lr": 0.00016716999282321857, "wd": 0.01, "grad_norm": 0.08480294793844223, "tokens_per_second": 1429242.337045836, "eta_seconds": 692.9405925989151, "data_wait_seconds": 2.4974346160888672e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08233}}
{"step": 70600, "total_steps": 72479, "tokens": 37014732800, "loss": 4.286384224891663, "l_mix": 2.657783627510071, "l_a": 2.7356813848018646, "l_c": 2.6929870545864105, "z_loss": 48.995598793029785, "lr": 0.00016544835106966366, "wd": 0.01, "grad_norm": 0.08419784903526306, "tokens_per_second": 1332011.3440567087, "eta_seconds": 739.5861577272416, "data_wait_seconds": 1.8596649169921875e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16967654, "gate_a2b/L1_absmax": 0.67048746, "gate_a2b/L1_rms": 0.17874466, "gate_a2b/L3_absmean": 0.15866844, "gate_a2b/L3_absmax": 0.50791073, "gate_a2b/L3_rms": 0.16409431, "gate_a2b/L5_absmean": 0.22693518, "gate_a2b/L5_absmax": 0.73431498, "gate_a2b/L5_rms": 0.23275957, "gate_c2b/L1_absmean": 0.04999181, "gate_c2b/L1_absmax": 0.59079641, "gate_c2b/L1_rms": 0.07283454, "gate_c2b/L3_absmean": 0.16234462, "gate_c2b/L3_absmax": 0.38105893, "gate_c2b/L3_rms": 0.16573845, "gate_c2b/L5_absmean": 0.4694626, "gate_c2b/L5_absmax": 0.68699324, "gate_c2b/L5_rms": 0.47814697, "gate_b2a/L7_absmean": 0.0531054, "gate_b2a/L7_absmax": 0.33841312, "gate_b2a/L7_rms": 0.0587295, "gate_b2a/L11_absmean": 0.08702002, "gate_b2a/L11_absmax": 0.60998267, "gate_b2a/L11_rms": 0.09562036, "gate_b2a/L14_absmean": 0.22471328, "gate_b2a/L14_absmax": 0.7889666, "gate_b2a/L14_rms": 0.23295075, "gate_b2c/L3_absmean": 0.04098377, "gate_b2c/L3_absmax": 0.55622321, "gate_b2c/L3_rms": 0.05039274, "gate_b2c/L5_absmean": 0.04108129, "gate_b2c/L5_absmax": 0.53973055, "gate_b2c/L5_rms": 0.04969982, "gate_b2c/L6_absmean": 0.05914789, "gate_b2c/L6_absmax": 0.32778999, "gate_b2c/L6_rms": 0.06427608}, "streams": {"cos_a2b/round0": 0.354751, "cos_c2b/round0": 0.233663, "cos_b2a/round0": 0.171195, "cos_b2c/round0": 0.153916, "rms_a/round0": 6.677637, "rms_b/round0": 11.859558, "rms_c/round0": 4.763083, "cos_a2b/round1": 0.408055, "cos_c2b/round1": 0.305409, "cos_b2a/round1": 0.259966, "cos_b2c/round1": 0.098292, "rms_a/round1": 13.463164, "rms_b/round1": 28.311768, "rms_c/round1": 10.571834, "cos_a2b/round2": 0.535474, "cos_c2b/round2": 0.519815, "cos_b2a/round2": 0.095297, "cos_b2c/round2": 0.176105, "rms_a/round2": 41.337376, "rms_b/round2": 80.990356, "rms_c/round2": 21.424498}, "alpha_mean": 0.42993770768260475, "alpha_hist": [0.01559, 0.0784, 0.18127, 0.19189, 0.1716, 0.16124, 0.12187, 0.06757, 0.01018, 0.00038], "alpha_by_type": {"special": {"share": 0.00131, "mean": 0.54596, "std": 0.12964}, "whitespace": {"share": 0.02544, "mean": 0.52958, "std": 0.14516}, "digit": {"share": 0.04465, "mean": 0.44829, "std": 0.13501}, "word_start": {"share": 0.55708, "mean": 0.44625, "std": 0.16907}, "word_cont": {"share": 0.25717, "mean": 0.31185, "std": 0.13735}, "punct": {"share": 0.11435, "mean": 0.58541, "std": 0.15717}}, "alpha_p05": 0.15482, "alpha_p50": 0.41798, "alpha_p95": 0.71958}
{"step": 70610, "total_steps": 72479, "tokens": 37019975680, "loss": 4.3536704778671265, "l_mix": 2.699707418680191, "l_a": 2.77822083234787, "l_c": 2.7349894642829895, "z_loss": 49.04434108734131, "lr": 0.00016373493323576464, "wd": 0.01, "grad_norm": 0.08682533353567123, "tokens_per_second": 1381186.1792598653, "eta_seconds": 709.4584978580475, "data_wait_seconds": 1.7344951629638672e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 70620, "total_steps": 72479, "tokens": 37025218560, "loss": 4.34600567817688, "l_mix": 2.694234997034073, "l_a": 2.7786676585674286, "l_c": 2.727234423160553, "z_loss": 49.129517555236816, "lr": 0.00016202975363362976, "wd": 0.01, "grad_norm": 0.09102191776037216, "tokens_per_second": 1411942.2569571612, "eta_seconds": 690.2912546157836, "data_wait_seconds": 1.58846378326416e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 70630, "total_steps": 72479, "tokens": 37030461440, "loss": 4.369021654129028, "l_mix": 2.7090110778808594, "l_a": 2.7895990014076233, "l_c": 2.743769198656082, "z_loss": 48.34767436981201, "lr": 0.00016033282650655543, "wd": 0.01, "grad_norm": 0.08208605647087097, "tokens_per_second": 1314718.7246442526, "eta_seconds": 737.3505023002624, "data_wait_seconds": 1.9729137420654297e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 70640, "total_steps": 72479, "tokens": 37035704320, "loss": 2.6838749647140503, "l_mix": 1.6613385528326035, "l_a": 1.7248756140470505, "l_c": 1.6835789680480957, "z_loss": 61.96906661987305, "lr": 0.00015864416602890392, "wd": 0.01, "grad_norm": 0.08925018459558487, "tokens_per_second": 1444558.8926453155, "eta_seconds": 667.4463996648788, "data_wait_seconds": 1.7881393432617188e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 70650, "total_steps": 72479, "tokens": 37040947200, "loss": 2.9963055849075317, "l_mix": 1.8432159125804901, "l_a": 1.9568813890218735, "l_c": 1.8867506384849548, "z_loss": 54.04716491699219, "lr": 0.00015696378630598748, "wd": 0.01, "grad_norm": 0.08500217646360397, "tokens_per_second": 1284371.920476083, "eta_seconds": 746.6083123683928, "data_wait_seconds": 2.5063753128051758e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 70660, "total_steps": 72479, "tokens": 37046190080, "loss": 2.932242512702942, "l_mix": 1.8042097240686417, "l_a": 1.913654014468193, "l_c": 1.846455156803131, "z_loss": 54.316598892211914, "lr": 0.0001552917013739488, "wd": 0.01, "grad_norm": 0.08356725424528122, "tokens_per_second": 1329621.9932219624, "eta_seconds": 717.2563908100128, "data_wait_seconds": 2.9027462005615234e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 70670, "total_steps": 72479, "tokens": 37051432960, "loss": 2.965466558933258, "l_mix": 1.8242940306663513, "l_a": 1.937298759818077, "l_c": 1.866609439253807, "z_loss": 53.76411581039429, "lr": 0.00015362792519964515, "wd": 0.01, "grad_norm": 0.08656960725784302, "tokens_per_second": 1461114.2446713913, "eta_seconds": 649.1189826250077, "data_wait_seconds": 2.3871660232543945e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 70680, "total_steps": 72479, "tokens": 37056675840, "loss": 4.515783846378326, "l_mix": 2.8019392788410187, "l_a": 2.8768053352832794, "l_c": 2.8360095620155334, "z_loss": 48.04542589187622, "lr": 0.00015197247168053087, "wd": 0.01, "grad_norm": 0.08507145196199417, "tokens_per_second": 1475977.8622790964, "eta_seconds": 639.0299855470657, "data_wait_seconds": 2.0831823348999023e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 70690, "total_steps": 72479, "tokens": 37061918720, "loss": 4.268243074417114, "l_mix": 2.6462135910987854, "l_a": 2.726433575153351, "l_c": 2.6803311705589294, "z_loss": 48.76805257797241, "lr": 0.00015032535464454185, "wd": 0.01, "grad_norm": 0.09054199606180191, "tokens_per_second": 1439866.0955402274, "eta_seconds": 651.4155968427658, "data_wait_seconds": 1.8149614334106445e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 70700, "total_steps": 72479, "tokens": 37067161600, "loss": 4.435771822929382, "l_mix": 2.750759541988373, "l_a": 2.8301746249198914, "l_c": 2.786532700061798, "z_loss": 48.72918748855591, "lr": 0.0001486865878499786, "wd": 0.01, "grad_norm": 0.08387240022420883, "tokens_per_second": 1458369.5637524917, "eta_seconds": 639.5555524349213, "data_wait_seconds": 1.6450881958007812e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.16972296, "gate_a2b/L1_absmax": 0.67025095, "gate_a2b/L1_rms": 0.17879924, "gate_a2b/L3_absmean": 0.15873733, "gate_a2b/L3_absmax": 0.50741649, "gate_a2b/L3_rms": 0.16416928, "gate_a2b/L5_absmean": 0.22696014, "gate_a2b/L5_absmax": 0.73418885, "gate_a2b/L5_rms": 0.23279099, "gate_c2b/L1_absmean": 0.05002902, "gate_c2b/L1_absmax": 0.59035575, "gate_c2b/L1_rms": 0.0728957, "gate_c2b/L3_absmean": 0.1624469, "gate_c2b/L3_absmax": 0.38111645, "gate_c2b/L3_rms": 0.16583635, "gate_c2b/L5_absmean": 0.46946952, "gate_c2b/L5_absmax": 0.68676132, "gate_c2b/L5_rms": 0.47813058, "gate_b2a/L7_absmean": 0.0530469, "gate_b2a/L7_absmax": 0.33846587, "gate_b2a/L7_rms": 0.05868637, "gate_b2a/L11_absmean": 0.08711088, "gate_b2a/L11_absmax": 0.61013877, "gate_b2a/L11_rms": 0.09569584, "gate_b2a/L14_absmean": 0.22482824, "gate_b2a/L14_absmax": 0.78911453, "gate_b2a/L14_rms": 0.23304956, "gate_b2c/L3_absmean": 0.04101279, "gate_b2c/L3_absmax": 0.55634719, "gate_b2c/L3_rms": 0.05042337, "gate_b2c/L5_absmean": 0.04110367, "gate_b2c/L5_absmax": 0.53953868, "gate_b2c/L5_rms": 0.04971964, "gate_b2c/L6_absmean": 0.05919709, "gate_b2c/L6_absmax": 0.32750148, "gate_b2c/L6_rms": 0.06431822}, "streams": {"cos_a2b/round0": 0.353852, "cos_c2b/round0": 0.23456, "cos_b2a/round0": 0.177861, "cos_b2c/round0": 0.154153, "rms_a/round0": 6.655904, "rms_b/round0": 11.826339, "rms_c/round0": 4.757413, "cos_a2b/round1": 0.410743, "cos_c2b/round1": 0.31153, "cos_b2a/round1": 0.2657, "cos_b2c/round1": 0.103257, "rms_a/round1": 13.428603, "rms_b/round1": 28.198792, "rms_c/round1": 10.504647, "cos_a2b/round2": 0.542225, "cos_c2b/round2": 0.519247, "cos_b2a/round2": 0.101561, "cos_b2c/round2": 0.179854, "rms_a/round2": 41.395977, "rms_b/round2": 80.547035, "rms_c/round2": 21.20717}, "alpha_mean": 0.431571747120131, "alpha_hist": [0.01471, 0.08165, 0.17424, 0.18703, 0.18005, 0.16096, 0.12318, 0.06836, 0.0096, 0.00021], "alpha_by_type": {"special": {"share": 0.00113, "mean": 0.5452, "std": 0.14624}, "whitespace": {"share": 0.02879, "mean": 0.50292, "std": 0.11963}, "digit": {"share": 0.02982, "mean": 0.46386, "std": 0.11498}, "word_start": {"share": 0.57536, "mean": 0.45114, "std": 0.1705}, "word_cont": {"share": 0.25876, "mean": 0.31584, "std": 0.13938}, "punct": {"share": 0.10614, "mean": 0.57799, "std": 0.16489}}, "alpha_p05": 0.14199, "alpha_p50": 0.41111, "alpha_p95": 0.7068}
{"step": 70710, "total_steps": 72479, "tokens": 37072404480, "loss": 4.344605207443237, "l_mix": 2.6938525438308716, "l_a": 2.77420637011528, "l_c": 2.7283020317554474, "z_loss": 49.37216091156006, "lr": 0.00014705618498539368, "wd": 0.01, "grad_norm": 0.0828668400645256, "tokens_per_second": 1460900.0162045914, "eta_seconds": 634.8589648246765, "data_wait_seconds": 2.0503997802734375e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 70720, "total_steps": 72479, "tokens": 37077647360, "loss": 4.363896012306213, "l_mix": 2.7057818472385406, "l_a": 2.786040872335434, "l_c": 2.7410058081150055, "z_loss": 48.88655614852905, "lr": 0.00014543415966947526, "wd": 0.01, "grad_norm": 0.0820624977350235, "tokens_per_second": 1287230.7391149227, "eta_seconds": 716.4392241239548, "data_wait_seconds": 1.7881393432617188e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 70730, "total_steps": 72479, "tokens": 37082890240, "loss": 4.3774460554122925, "l_mix": 2.7141659259796143, "l_a": 2.7937098145484924, "l_c": 2.750557154417038, "z_loss": 49.247079849243164, "lr": 0.00014382052545093457, "wd": 0.01, "grad_norm": 0.08863207697868347, "tokens_per_second": 1412305.615304992, "eta_seconds": 649.2785287141801, "data_wait_seconds": 1.8417835235595703e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 70740, "total_steps": 72479, "tokens": 37088133120, "loss": 4.375940263271332, "l_mix": 2.713035970926285, "l_a": 2.796210676431656, "l_c": 2.746803402900696, "z_loss": 48.50723218917847, "lr": 0.00014221529580839244, "wd": 0.01, "grad_norm": 0.08243230730295181, "tokens_per_second": 1439400.0374096048, "eta_seconds": 633.4144840240479, "data_wait_seconds": 1.6182661056518555e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 70750, "total_steps": 72479, "tokens": 37093376000, "loss": 3.088863879442215, "l_mix": 1.9127002358436584, "l_a": 1.982464611530304, "l_c": 1.9380808025598526, "z_loss": 60.47512722015381, "lr": 0.000140618484150266, "wd": 0.01, "grad_norm": 0.10360598564147949, "tokens_per_second": 1442267.9844304915, "eta_seconds": 628.5197770357132, "data_wait_seconds": 3.921985626220703e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 70760, "total_steps": 72479, "tokens": 37098618880, "loss": 2.9009054601192474, "l_mix": 1.7844327539205551, "l_a": 1.894278571009636, "l_c": 1.8272967785596848, "z_loss": 55.041781425476074, "lr": 0.0001390301038146582, "wd": 0.01, "grad_norm": 0.08169751614332199, "tokens_per_second": 1361305.264092595, "eta_seconds": 662.0492080450058, "data_wait_seconds": 2.2023916244506836e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 70770, "total_steps": 72479, "tokens": 37103861760, "loss": 2.9273700416088104, "l_mix": 1.8008621484041214, "l_a": 1.9117047488689423, "l_c": 1.843321293592453, "z_loss": 54.726757526397705, "lr": 0.00013745016806924432, "wd": 0.01, "grad_norm": 0.08171442151069641, "tokens_per_second": 1393620.8050188578, "eta_seconds": 642.9354303359985, "data_wait_seconds": 2.238154411315918e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 70780, "total_steps": 72479, "tokens": 37109104640, "loss": 2.9507833123207092, "l_mix": 1.815403938293457, "l_a": 1.9272735118865967, "l_c": 1.8573242127895355, "z_loss": 54.2569375038147, "lr": 0.00013587869011116396, "wd": 0.01, "grad_norm": 0.08653149753808975, "tokens_per_second": 1291425.7609829344, "eta_seconds": 689.7534019470214, "data_wait_seconds": 2.014636993408203e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 70790, "total_steps": 72479, "tokens": 37114347520, "loss": 4.36680543422699, "l_mix": 2.707257628440857, "l_a": 2.7910296022892, "l_c": 2.7407961785793304, "z_loss": 49.266709327697754, "lr": 0.00013431568306690738, "wd": 0.01, "grad_norm": 0.08645410090684891, "tokens_per_second": 1485311.582553373, "eta_seconds": 596.1863102674484, "data_wait_seconds": 1.9162893295288086e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 70800, "total_steps": 72479, "tokens": 37119590400, "loss": 4.300494581460953, "l_mix": 2.6663300693035126, "l_a": 2.7482913732528687, "l_c": 2.69892355799675, "z_loss": 48.921894550323486, "lr": 0.00013276115999220886, "wd": 0.01, "grad_norm": 0.10166969150304794, "tokens_per_second": 1451375.7666186048, "eta_seconds": 606.5138830661774, "data_wait_seconds": 1.4156103134155273e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16975926, "gate_a2b/L1_absmax": 0.67124742, "gate_a2b/L1_rms": 0.17882672, "gate_a2b/L3_absmean": 0.15876926, "gate_a2b/L3_absmax": 0.50768334, "gate_a2b/L3_rms": 0.16419493, "gate_a2b/L5_absmean": 0.22706321, "gate_a2b/L5_absmax": 0.73428673, "gate_a2b/L5_rms": 0.23289211, "gate_c2b/L1_absmean": 0.05013829, "gate_c2b/L1_absmax": 0.59072977, "gate_c2b/L1_rms": 0.07295178, "gate_c2b/L3_absmean": 0.16241892, "gate_c2b/L3_absmax": 0.38092563, "gate_c2b/L3_rms": 0.16583446, "gate_c2b/L5_absmean": 0.46939331, "gate_c2b/L5_absmax": 0.68676025, "gate_c2b/L5_rms": 0.47805732, "gate_b2a/L7_absmean": 0.05316266, "gate_b2a/L7_absmax": 0.33859983, "gate_b2a/L7_rms": 0.05878958, "gate_b2a/L11_absmean": 0.0870768, "gate_b2a/L11_absmax": 0.60983473, "gate_b2a/L11_rms": 0.09566525, "gate_b2a/L14_absmean": 0.22485113, "gate_b2a/L14_absmax": 0.78886092, "gate_b2a/L14_rms": 0.2330727, "gate_b2c/L3_absmean": 0.04102629, "gate_b2c/L3_absmax": 0.55679035, "gate_b2c/L3_rms": 0.05044845, "gate_b2c/L5_absmean": 0.04111988, "gate_b2c/L5_absmax": 0.53918594, "gate_b2c/L5_rms": 0.04972664, "gate_b2c/L6_absmean": 0.05922244, "gate_b2c/L6_absmax": 0.32733837, "gate_b2c/L6_rms": 0.06433957}, "streams": {"cos_a2b/round0": 0.356592, "cos_c2b/round0": 0.238049, "cos_b2a/round0": 0.183188, "cos_b2c/round0": 0.157978, "rms_a/round0": 6.607278, "rms_b/round0": 11.771558, "rms_c/round0": 4.743099, "cos_a2b/round1": 0.412622, "cos_c2b/round1": 0.313748, "cos_b2a/round1": 0.270781, "cos_b2c/round1": 0.10439, "rms_a/round1": 13.318903, "rms_b/round1": 28.087992, "rms_c/round1": 10.494704, "cos_a2b/round2": 0.54637, "cos_c2b/round2": 0.521453, "cos_b2a/round2": 0.09772, "cos_b2c/round2": 0.179366, "rms_a/round2": 41.313122, "rms_b/round2": 80.414772, "rms_c/round2": 21.193796}, "alpha_mean": 0.4298671223919861, "alpha_hist": [0.01149, 0.08055, 0.18161, 0.18793, 0.18201, 0.16, 0.1212, 0.06651, 0.00858, 0.00012], "alpha_by_type": {"special": {"share": 0.00119, "mean": 0.56086, "std": 0.12626}, "whitespace": {"share": 0.02606, "mean": 0.49991, "std": 0.12679}, "digit": {"share": 0.03259, "mean": 0.43622, "std": 0.13034}, "word_start": {"share": 0.59036, "mean": 0.44948, "std": 0.16602}, "word_cont": {"share": 0.24112, "mean": 0.30964, "std": 0.13451}, "punct": {"share": 0.10867, "mean": 0.5699, "std": 0.17184}}, "alpha_p05": 0.16452, "alpha_p50": 0.41466, "alpha_p95": 0.71721}
{"step": 70810, "total_steps": 72479, "tokens": 37124833280, "loss": 4.310137152671814, "l_mix": 2.672836124897003, "l_a": 2.7522230446338654, "l_c": 2.705446779727936, "z_loss": 48.90496492385864, "lr": 0.00013121513387193647, "wd": 0.01, "grad_norm": 0.08231213688850403, "tokens_per_second": 1294151.5711192915, "eta_seconds": 676.1469765424729, "data_wait_seconds": 1.5914440155029297e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 70820, "total_steps": 72479, "tokens": 37130076160, "loss": 4.368610441684723, "l_mix": 2.7087186872959137, "l_a": 2.7909849882125854, "l_c": 2.7419873774051666, "z_loss": 48.55269527435303, "lr": 0.0001296776176199823, "wd": 0.01, "grad_norm": 0.07879577577114105, "tokens_per_second": 1472054.8540826833, "eta_seconds": 590.870502948761, "data_wait_seconds": 1.7762184143066406e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 70830, "total_steps": 72479, "tokens": 37135319040, "loss": 4.376450955867767, "l_mix": 2.7135781347751617, "l_a": 2.7944769859313965, "l_c": 2.7484321892261505, "z_loss": 48.75540018081665, "lr": 0.00012814862407915724, "wd": 0.01, "grad_norm": 0.08022385835647583, "tokens_per_second": 1380635.703220834, "eta_seconds": 626.1977073192596, "data_wait_seconds": 1.5228986740112305e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 70840, "total_steps": 72479, "tokens": 37140561920, "loss": 4.48471611738205, "l_mix": 2.782814681529999, "l_a": 2.8561305105686188, "l_c": 2.8168744444847107, "z_loss": 48.1264386177063, "lr": 0.00012662816602108178, "wd": 0.01, "grad_norm": 0.08055226504802704, "tokens_per_second": 1448321.3616263443, "eta_seconds": 593.3130966424942, "data_wait_seconds": 1.6480684280395508e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 70850, "total_steps": 72479, "tokens": 37145804800, "loss": 4.551966547966003, "l_mix": 2.824553519487381, "l_a": 2.89923033118248, "l_c": 2.8588128983974457, "z_loss": 47.43272590637207, "lr": 0.00012511625614607992, "wd": 0.01, "grad_norm": 0.08229673653841019, "tokens_per_second": 1434999.3324628589, "eta_seconds": 595.1676301717758, "data_wait_seconds": 2.428889274597168e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 70860, "total_steps": 72479, "tokens": 37151047680, "loss": 4.5545734167099, "l_mix": 2.826647460460663, "l_a": 2.8977091908454895, "l_c": 2.862043857574463, "z_loss": 47.46199560165405, "lr": 0.0001236129070830727, "wd": 0.01, "grad_norm": 0.08310610055923462, "tokens_per_second": 1349923.8523018444, "eta_seconds": 628.7927060127258, "data_wait_seconds": 2.1666288375854492e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 70870, "total_steps": 72479, "tokens": 37156290560, "loss": 2.988691031932831, "l_mix": 1.838557630777359, "l_a": 1.9514328092336655, "l_c": 1.8823450356721878, "z_loss": 53.757946491241455, "lr": 0.00012211813138947376, "wd": 0.01, "grad_norm": 0.08265353739261627, "tokens_per_second": 1386847.8489813714, "eta_seconds": 608.2710461854934, "data_wait_seconds": 1.621246337890625e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 70880, "total_steps": 72479, "tokens": 37161533440, "loss": 2.997970700263977, "l_mix": 1.84428009390831, "l_a": 1.9566375315189362, "l_c": 1.8889977931976318, "z_loss": 53.914276123046875, "lr": 0.00012063194155108302, "wd": 0.01, "grad_norm": 0.08411762863397598, "tokens_per_second": 1453055.699913802, "eta_seconds": 576.9472650289536, "data_wait_seconds": 2.4765729904174805e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 70890, "total_steps": 72479, "tokens": 37166776320, "loss": 2.9586643278598785, "l_mix": 1.8197993338108063, "l_a": 1.9339701980352402, "l_c": 1.8622463941574097, "z_loss": 53.93616247177124, "lr": 0.00011915434998198375, "wd": 0.01, "grad_norm": 0.08441384881734848, "tokens_per_second": 1376868.6728197038, "eta_seconds": 605.0639748334885, "data_wait_seconds": 2.1219253540039062e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 70900, "total_steps": 72479, "tokens": 37172019200, "loss": 4.298585593700409, "l_mix": 2.6647031009197235, "l_a": 2.745608240365982, "l_c": 2.7006664276123047, "z_loss": 49.23568677902222, "lr": 0.00011768536902443755, "wd": 0.01, "grad_norm": 0.09241791069507599, "tokens_per_second": 1433439.7258671664, "eta_seconds": 577.5274237632751, "data_wait_seconds": 1.5527009963989258e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16988896, "gate_a2b/L1_absmax": 0.67082655, "gate_a2b/L1_rms": 0.17894492, "gate_a2b/L3_absmean": 0.15885426, "gate_a2b/L3_absmax": 0.50823748, "gate_a2b/L3_rms": 0.164272, "gate_a2b/L5_absmean": 0.22704792, "gate_a2b/L5_absmax": 0.73449248, "gate_a2b/L5_rms": 0.23287915, "gate_c2b/L1_absmean": 0.05017811, "gate_c2b/L1_absmax": 0.59001571, "gate_c2b/L1_rms": 0.07292391, "gate_c2b/L3_absmean": 0.16240917, "gate_c2b/L3_absmax": 0.38077253, "gate_c2b/L3_rms": 0.16581802, "gate_c2b/L5_absmean": 0.46937773, "gate_c2b/L5_absmax": 0.68709236, "gate_c2b/L5_rms": 0.47804561, "gate_b2a/L7_absmean": 0.0531722, "gate_b2a/L7_absmax": 0.33849323, "gate_b2a/L7_rms": 0.05880165, "gate_b2a/L11_absmean": 0.08707844, "gate_b2a/L11_absmax": 0.60922527, "gate_b2a/L11_rms": 0.09565387, "gate_b2a/L14_absmean": 0.22485301, "gate_b2a/L14_absmax": 0.78869599, "gate_b2a/L14_rms": 0.23306976, "gate_b2c/L3_absmean": 0.04111248, "gate_b2c/L3_absmax": 0.5567981, "gate_b2c/L3_rms": 0.05051224, "gate_b2c/L5_absmean": 0.04117109, "gate_b2c/L5_absmax": 0.53932732, "gate_b2c/L5_rms": 0.04976894, "gate_b2c/L6_absmean": 0.05922678, "gate_b2c/L6_absmax": 0.32725364, "gate_b2c/L6_rms": 0.0643413}, "streams": {"cos_a2b/round0": 0.353607, "cos_c2b/round0": 0.233803, "cos_b2a/round0": 0.174418, "cos_b2c/round0": 0.153115, "rms_a/round0": 6.633999, "rms_b/round0": 11.814274, "rms_c/round0": 4.747375, "cos_a2b/round1": 0.409925, "cos_c2b/round1": 0.305398, "cos_b2a/round1": 0.262381, "cos_b2c/round1": 0.099578, "rms_a/round1": 13.440368, "rms_b/round1": 28.169308, "rms_c/round1": 10.534628, "cos_a2b/round2": 0.53654, "cos_c2b/round2": 0.520073, "cos_b2a/round2": 0.096449, "cos_b2c/round2": 0.175721, "rms_a/round2": 41.306599, "rms_b/round2": 80.799576, "rms_c/round2": 21.405848}, "alpha_mean": 0.424805240568503, "alpha_hist": [0.01474, 0.09474, 0.18451, 0.19106, 0.16161, 0.14867, 0.12198, 0.07115, 0.011, 0.00055], "alpha_by_type": {"special": {"share": 0.00127, "mean": 0.56385, "std": 0.12538}, "whitespace": {"share": 0.02943, "mean": 0.52303, "std": 0.14227}, "digit": {"share": 0.04295, "mean": 0.432, "std": 0.14624}, "word_start": {"share": 0.55017, "mean": 0.44851, "std": 0.17605}, "word_cont": {"share": 0.2635, "mean": 0.30448, "std": 0.13548}, "punct": {"share": 0.11267, "mean": 0.56047, "std": 0.17374}}, "alpha_p05": 0.15204, "alpha_p50": 0.40851, "alpha_p95": 0.7364}
{"step": 70910, "total_steps": 72479, "tokens": 37177262080, "loss": 4.4175973534584045, "l_mix": 2.739904522895813, "l_a": 2.8181666135787964, "l_c": 2.77414271235466, "z_loss": 48.20302724838257, "lr": 0.00011622501094878229, "wd": 0.01, "grad_norm": 0.08684986084699631, "tokens_per_second": 1403576.558356892, "eta_seconds": 586.0798024177551, "data_wait_seconds": 1.8656253814697266e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 70920, "total_steps": 72479, "tokens": 37182504960, "loss": 4.29416698217392, "l_mix": 2.662316620349884, "l_a": 2.743965744972229, "l_c": 2.6955350637435913, "z_loss": 49.19681119918823, "lr": 0.00011477328795332952, "wd": 0.01, "grad_norm": 0.08244198560714722, "tokens_per_second": 1403796.9756647123, "eta_seconds": 582.2529939651489, "data_wait_seconds": 1.6808509826660156e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 70930, "total_steps": 72479, "tokens": 37187747840, "loss": 4.2895665764808655, "l_mix": 2.6593582034111023, "l_a": 2.741401970386505, "l_c": 2.6926256716251373, "z_loss": 49.40653705596924, "lr": 0.00011333021216426181, "wd": 0.01, "grad_norm": 0.09043413400650024, "tokens_per_second": 1329786.1791456416, "eta_seconds": 610.7163126945496, "data_wait_seconds": 2.0742416381835938e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 70940, "total_steps": 72479, "tokens": 37192990720, "loss": 2.986843556165695, "l_mix": 1.8377925157546997, "l_a": 1.9492360800504684, "l_c": 1.8809338510036469, "z_loss": 54.00016117095947, "lr": 0.0001118957956355326, "wd": 0.01, "grad_norm": 0.08182793110609055, "tokens_per_second": 1341291.4087637798, "eta_seconds": 601.5689258337021, "data_wait_seconds": 3.1888484954833984e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 70950, "total_steps": 72479, "tokens": 37198233600, "loss": 2.9561415910720825, "l_mix": 1.8183999508619308, "l_a": 1.9309902340173721, "l_c": 1.8614816963672638, "z_loss": 54.51723337173462, "lr": 0.00011047005034876382, "wd": 0.01, "grad_norm": 0.08471717685461044, "tokens_per_second": 1368004.9700245846, "eta_seconds": 585.9893564462662, "data_wait_seconds": 2.0742416381835938e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 70960, "total_steps": 72479, "tokens": 37203476480, "loss": 4.331404983997345, "l_mix": 2.687580555677414, "l_a": 2.7571267187595367, "l_c": 2.722287595272064, "z_loss": 48.473186016082764, "lr": 0.00010905298821314798, "wd": 0.01, "grad_norm": 0.08959271758794785, "tokens_per_second": 1429186.4177850522, "eta_seconds": 557.2355447053909, "data_wait_seconds": 4.106760025024414e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 70970, "total_steps": 72479, "tokens": 37208719360, "loss": 4.552401006221771, "l_mix": 2.8245712220668793, "l_a": 2.9009543657302856, "l_c": 2.858477771282196, "z_loss": 47.375489711761475, "lr": 0.00010764462106534713, "wd": 0.01, "grad_norm": 0.08959616720676422, "tokens_per_second": 1202130.9338813112, "eta_seconds": 658.1234786510468, "data_wait_seconds": 3.409385681152344e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 70980, "total_steps": 72479, "tokens": 37213962240, "loss": 4.484078586101532, "l_mix": 2.7823041677474976, "l_a": 2.855483204126358, "l_c": 2.817098319530487, "z_loss": 48.17359256744385, "lr": 0.00010624496066939448, "wd": 0.01, "grad_norm": 0.08297962695360184, "tokens_per_second": 1399755.109451226, "eta_seconds": 561.4608631849289, "data_wait_seconds": 2.777576446533203e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 70990, "total_steps": 72479, "tokens": 37219205120, "loss": 2.930602043867111, "l_mix": 1.803042158484459, "l_a": 1.9126927703619003, "l_c": 1.8458400666713715, "z_loss": 54.49669599533081, "lr": 0.00010485401871659581, "wd": 0.01, "grad_norm": 0.08581504970788956, "tokens_per_second": 1453267.9219129535, "eta_seconds": 537.1788781881332, "data_wait_seconds": 3.4809112548828125e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 71000, "total_steps": 72479, "tokens": 37224448000, "loss": 2.9417343735694885, "l_mix": 1.8098769336938858, "l_a": 1.9195557981729507, "l_c": 1.853302225470543, "z_loss": 54.916035175323486, "lr": 0.0001034718068254328, "wd": 0.01, "grad_norm": 0.07981602102518082, "tokens_per_second": 1374851.5707760039, "eta_seconds": 564.0041212320328, "data_wait_seconds": 2.2619962692260742e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16991691, "gate_a2b/L1_absmax": 0.67086756, "gate_a2b/L1_rms": 0.17897062, "gate_a2b/L3_absmean": 0.1588712, "gate_a2b/L3_absmax": 0.50834495, "gate_a2b/L3_rms": 0.16429788, "gate_a2b/L5_absmean": 0.22712572, "gate_a2b/L5_absmax": 0.73407072, "gate_a2b/L5_rms": 0.2329559, "gate_c2b/L1_absmean": 0.05024609, "gate_c2b/L1_absmax": 0.58969814, "gate_c2b/L1_rms": 0.07291479, "gate_c2b/L3_absmean": 0.16234837, "gate_c2b/L3_absmax": 0.38073707, "gate_c2b/L3_rms": 0.16576013, "gate_c2b/L5_absmean": 0.46940908, "gate_c2b/L5_absmax": 0.68723863, "gate_c2b/L5_rms": 0.47806725, "gate_b2a/L7_absmean": 0.05313142, "gate_b2a/L7_absmax": 0.33863226, "gate_b2a/L7_rms": 0.05877078, "gate_b2a/L11_absmean": 0.08714387, "gate_b2a/L11_absmax": 0.60971099, "gate_b2a/L11_rms": 0.09572228, "gate_b2a/L14_absmean": 0.22491959, "gate_b2a/L14_absmax": 0.78841323, "gate_b2a/L14_rms": 0.23312631, "gate_b2c/L3_absmean": 0.04114443, "gate_b2c/L3_absmax": 0.55695808, "gate_b2c/L3_rms": 0.05054378, "gate_b2c/L5_absmean": 0.04117337, "gate_b2c/L5_absmax": 0.53926504, "gate_b2c/L5_rms": 0.04976872, "gate_b2c/L6_absmean": 0.05926445, "gate_b2c/L6_absmax": 0.32717958, "gate_b2c/L6_rms": 0.06436425}, "streams": {"cos_a2b/round0": 0.352089, "cos_c2b/round0": 0.20752, "cos_b2a/round0": 0.163459, "cos_b2c/round0": 0.131232, "rms_a/round0": 6.901288, "rms_b/round0": 12.006268, "rms_c/round0": 4.890103, "cos_a2b/round1": 0.404016, "cos_c2b/round1": 0.306942, "cos_b2a/round1": 0.252709, "cos_b2c/round1": 0.110096, "rms_a/round1": 14.138326, "rms_b/round1": 29.495861, "rms_c/round1": 11.328385, "cos_a2b/round2": 0.519887, "cos_c2b/round2": 0.513997, "cos_b2a/round2": 0.088031, "cos_b2c/round2": 0.168829, "rms_a/round2": 42.169289, "rms_b/round2": 81.402168, "rms_c/round2": 22.887283}, "alpha_mean": 0.4037309406699663, "alpha_hist": [0.05653, 0.1732, 0.18111, 0.13551, 0.11203, 0.10316, 0.10007, 0.08478, 0.04614, 0.00746], "alpha_by_type": {"special": {"share": 0.00108, "mean": 0.65753, "std": 0.11637}, "whitespace": {"share": 0.03723, "mean": 0.62663, "std": 0.17991}, "digit": {"share": 0.0096, "mean": 0.38289, "std": 0.18346}, "word_start": {"share": 0.55852, "mean": 0.40916, "std": 0.21767}, "word_cont": {"share": 0.25816, "mean": 0.25556, "std": 0.14934}, "punct": {"share": 0.13541, "mean": 0.60202, "std": 0.19833}}, "alpha_p05": 0.09637, "alpha_p50": 0.34158, "alpha_p95": 0.81197}
{"step": 71010, "total_steps": 72479, "tokens": 37229690880, "loss": 4.348026394844055, "l_mix": 2.695570945739746, "l_a": 2.7794887721538544, "l_c": 2.72869610786438, "z_loss": 48.38524913787842, "lr": 0.0001020983365414641, "wd": 0.01, "grad_norm": 0.08648096024990082, "tokens_per_second": 1458557.3167180275, "eta_seconds": 528.0416910409928, "data_wait_seconds": 1.749396324157715e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 71020, "total_steps": 72479, "tokens": 37234933760, "loss": 4.379222393035889, "l_mix": 2.715353310108185, "l_a": 2.79633966088295, "l_c": 2.749890476465225, "z_loss": 48.18998146057129, "lr": 0.00010073361933723175, "wd": 0.01, "grad_norm": 0.08640366792678833, "tokens_per_second": 1224414.7248439584, "eta_seconds": 624.7361914873123, "data_wait_seconds": 3.0308961868286133e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 71030, "total_steps": 72479, "tokens": 37240176640, "loss": 4.288504958152771, "l_mix": 2.658763498067856, "l_a": 2.7388977110385895, "l_c": 2.693573921918869, "z_loss": 48.98660659790039, "lr": 9.937766661216198e-05, "wd": 0.01, "grad_norm": 0.08279687911272049, "tokens_per_second": 1166160.313556457, "eta_seconds": 651.4484356641769, "data_wait_seconds": 1.895427703857422e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 71040, "total_steps": 72479, "tokens": 37245419520, "loss": 4.31637716293335, "l_mix": 2.6765207052230835, "l_a": 2.7564798295497894, "l_c": 2.709708273410797, "z_loss": 48.71924877166748, "lr": 9.803048969247283e-05, "wd": 0.01, "grad_norm": 0.09030197560787201, "tokens_per_second": 1322042.9944708648, "eta_seconds": 570.6701182603836, "data_wait_seconds": 1.9758939743041992e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 71050, "total_steps": 72479, "tokens": 37250662400, "loss": 2.9652030766010284, "l_mix": 1.823791965842247, "l_a": 1.937913417816162, "l_c": 1.8667902648448944, "z_loss": 54.33678865432739, "lr": 9.669209983107804e-05, "wd": 0.01, "grad_norm": 0.0828084945678711, "tokens_per_second": 1430864.784025966, "eta_seconds": 523.604718184471, "data_wait_seconds": 2.5033950805664062e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 71060, "total_steps": 72479, "tokens": 37255905280, "loss": 2.971035808324814, "l_mix": 1.8273924440145493, "l_a": 1.9413405507802963, "l_c": 1.8708038628101349, "z_loss": 53.99280118942261, "lr": 9.536250820749301e-05, "wd": 0.01, "grad_norm": 0.08183639496564865, "tokens_per_second": 1298954.8430136049, "eta_seconds": 572.740981721878, "data_wait_seconds": 2.530217170715332e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 71070, "total_steps": 72479, "tokens": 37261148160, "loss": 2.9363619685173035, "l_mix": 1.8061532378196716, "l_a": 1.9179249852895737, "l_c": 1.8494371324777603, "z_loss": 54.287692070007324, "lr": 9.404172592774268e-05, "wd": 0.01, "grad_norm": 0.0812905952334404, "tokens_per_second": 1420388.276405798, "eta_seconds": 520.0844052791596, "data_wait_seconds": 1.570582389831543e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 71080, "total_steps": 72479, "tokens": 37266391040, "loss": 2.980800598859787, "l_mix": 1.833765670657158, "l_a": 1.9458941519260406, "l_c": 1.8775555342435837, "z_loss": 54.196922302246094, "lr": 9.272976402426714e-05, "wd": 0.01, "grad_norm": 0.0813625305891037, "tokens_per_second": 1437910.394708441, "eta_seconds": 510.10057003498076, "data_wait_seconds": 2.0265579223632812e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 71090, "total_steps": 72479, "tokens": 37271633920, "loss": 2.914262145757675, "l_mix": 1.7925702929496765, "l_a": 1.9044106602668762, "l_c": 1.8345620185136795, "z_loss": 54.886932373046875, "lr": 9.142663345583058e-05, "wd": 0.01, "grad_norm": 0.08133520931005478, "tokens_per_second": 1338301.4824511216, "eta_seconds": 544.1494622468948, "data_wait_seconds": 2.0742416381835938e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}}
{"step": 71100, "total_steps": 72479, "tokens": 37276876800, "loss": 2.964083284139633, "l_mix": 1.823632374405861, "l_a": 1.9344282746315002, "l_c": 1.8670744895935059, "z_loss": 54.17706108093262, "lr": 9.013234510742932e-05, "wd": 0.01, "grad_norm": 0.08284471929073334, "tokens_per_second": 1461712.5144014384, "eta_seconds": 494.6206212759018, "data_wait_seconds": 1.609325408935547e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20044, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.16994528, "gate_a2b/L1_absmax": 0.67085332, "gate_a2b/L1_rms": 0.178997, "gate_a2b/L3_absmean": 0.15889628, "gate_a2b/L3_absmax": 0.50830406, "gate_a2b/L3_rms": 0.16432317, "gate_a2b/L5_absmean": 0.22720921, "gate_a2b/L5_absmax": 0.73401004, "gate_a2b/L5_rms": 0.2330375, "gate_c2b/L1_absmean": 0.05039595, "gate_c2b/L1_absmax": 0.58941543, "gate_c2b/L1_rms": 0.07300641, "gate_c2b/L3_absmean": 0.16242956, "gate_c2b/L3_absmax": 0.38066342, "gate_c2b/L3_rms": 0.16583407, "gate_c2b/L5_absmean": 0.46939617, "gate_c2b/L5_absmax": 0.68725824, "gate_c2b/L5_rms": 0.47805968, "gate_b2a/L7_absmean": 0.05317357, "gate_b2a/L7_absmax": 0.33863151, "gate_b2a/L7_rms": 0.05879768, "gate_b2a/L11_absmean": 0.08714775, "gate_b2a/L11_absmax": 0.609685, "gate_b2a/L11_rms": 0.09572522, "gate_b2a/L14_absmean": 0.22496247, "gate_b2a/L14_absmax": 0.78801131, "gate_b2a/L14_rms": 0.23316079, "gate_b2c/L3_absmean": 0.04115597, "gate_b2c/L3_absmax": 0.55704618, "gate_b2c/L3_rms": 0.05054634, "gate_b2c/L5_absmean": 0.04120472, "gate_b2c/L5_absmax": 0.53956038, "gate_b2c/L5_rms": 0.04979916, "gate_b2c/L6_absmean": 0.05931768, "gate_b2c/L6_absmax": 0.32712233, "gate_b2c/L6_rms": 0.06441785}, "streams": {"cos_a2b/round0": 0.351908, "cos_c2b/round0": 0.204547, "cos_b2a/round0": 0.163155, "cos_b2c/round0": 0.130823, "rms_a/round0": 6.882398, "rms_b/round0": 12.048158, "rms_c/round0": 4.904549, "cos_a2b/round1": 0.405163, "cos_c2b/round1": 0.306416, "cos_b2a/round1": 0.254718, "cos_b2c/round1": 0.110266, "rms_a/round1": 14.132756, "rms_b/round1": 29.669285, "rms_c/round1": 11.345025, "cos_a2b/round2": 0.515606, "cos_c2b/round2": 0.513301, "cos_b2a/round2": 0.088387, "cos_b2c/round2": 0.16853, "rms_a/round2": 42.013531, "rms_b/round2": 81.167778, "rms_c/round2": 22.800339}, "alpha_mean": 0.3944223541212324, "alpha_hist": [0.06169, 0.18839, 0.1853, 0.12999, 0.10213, 0.10016, 0.09894, 0.08232, 0.04398, 0.00711], "alpha_by_type": {"special": {"share": 0.00108, "mean": 0.66481, "std": 0.12387}, "whitespace": {"share": 0.03395, "mean": 0.62864, "std": 0.18492}, "digit": {"share": 0.01294, "mean": 0.41347, "std": 0.18383}, "word_start": {"share": 0.55667, "mean": 0.40055, "std": 0.22035}, "word_cont": {"share": 0.27213, "mean": 0.25118, "std": 0.14541}, "punct": {"share": 0.12323, "mean": 0.61417, "std": 0.19575}}, "alpha_p05": 0.08152, "alpha_p50": 0.35488, "alpha_p95": 0.80162}
{"step": 71110, "total_steps": 72479, "tokens": 37282119680, "loss": 2.912616103887558, "l_mix": 1.7919081896543503, "l_a": 1.9008659273386002, "l_c": 1.834827035665512, "z_loss": 54.906736850738525, "lr": 8.884690979020083e-05, "wd": 0.01, "grad_norm": 0.08165095001459122, "tokens_per_second": 1440873.8873573532, "eta_seconds": 498.13538734912873, "data_wait_seconds": 1.6301870346069336e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 71120, "total_steps": 72479, "tokens": 37287362560, "loss": 4.253521800041199, "l_mix": 2.6368490755558014, "l_a": 2.7200258672237396, "l_c": 2.6688827872276306, "z_loss": 48.484697341918945, "lr": 8.757033824133397e-05, "wd": 0.01, "grad_norm": 0.08466103672981262, "tokens_per_second": 1443298.561660226, "eta_seconds": 493.66597523689273, "data_wait_seconds": 1.665949821472168e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 71130, "total_steps": 72479, "tokens": 37292605440, "loss": 4.306195318698883, "l_mix": 2.6693731546401978, "l_a": 2.752910405397415, "l_c": 2.7031630277633667, "z_loss": 48.46723556518555, "lr": 8.630264112397887e-05, "wd": 0.01, "grad_norm": 0.08509641885757446, "tokens_per_second": 1394667.3883800637, "eta_seconds": 507.12056357860564, "data_wait_seconds": 2.238154411315918e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 71140, "total_steps": 72479, "tokens": 37297848320, "loss": 4.277782499790192, "l_mix": 2.651540756225586, "l_a": 2.735536575317383, "l_c": 2.6852689385414124, "z_loss": 48.40417146682739, "lr": 8.504382902715712e-05, "wd": 0.01, "grad_norm": 0.080097995698452, "tokens_per_second": 1472327.469719184, "eta_seconds": 476.8107954502106, "data_wait_seconds": 1.862645149230957e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 71150, "total_steps": 72479, "tokens": 37303091200, "loss": 4.253441154956818, "l_mix": 2.6364638805389404, "l_a": 2.720697671175003, "l_c": 2.669226288795471, "z_loss": 48.47628164291382, "lr": 8.379391246567507e-05, "wd": 0.01, "grad_norm": 0.08423997461795807, "tokens_per_second": 1456480.508405009, "eta_seconds": 478.3989541769028, "data_wait_seconds": 1.665949821472168e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 71160, "total_steps": 72479, "tokens": 37308334080, "loss": 2.982009708881378, "l_mix": 1.8347213864326477, "l_a": 1.945666879415512, "l_c": 1.8786275535821915, "z_loss": 53.87964582443237, "lr": 8.255290188003454e-05, "wd": 0.01, "grad_norm": 0.08270987123250961, "tokens_per_second": 1273043.9389172306, "eta_seconds": 543.214456987381, "data_wait_seconds": 3.49581241607666e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 71170, "total_steps": 72479, "tokens": 37313576960, "loss": 2.992306113243103, "l_mix": 1.8413197845220566, "l_a": 1.9515692293643951, "l_c": 1.885051652789116, "z_loss": 53.77639961242676, "lr": 8.132080763634624e-05, "wd": 0.01, "grad_norm": 0.08476576209068298, "tokens_per_second": 1317409.2779810817, "eta_seconds": 520.9413683891296, "data_wait_seconds": 5.206465721130371e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 71180, "total_steps": 72479, "tokens": 37318819840, "loss": 3.0072193145751953, "l_mix": 1.8502454161643982, "l_a": 1.963100254535675, "l_c": 1.8934792280197144, "z_loss": 53.60465049743652, "lr": 8.009764002624321e-05, "wd": 0.01, "grad_norm": 0.08109266310930252, "tokens_per_second": 1348565.1029948806, "eta_seconds": 505.0183417081833, "data_wait_seconds": 2.804398536682129e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 71190, "total_steps": 72479, "tokens": 37324062720, "loss": 2.949599504470825, "l_mix": 1.814715251326561, "l_a": 1.9240476489067078, "l_c": 1.8588998466730118, "z_loss": 54.39987325668335, "lr": 7.888340926679402e-05, "wd": 0.01, "grad_norm": 0.08070164173841476, "tokens_per_second": 1448351.6006928303, "eta_seconds": 466.60440163612367, "data_wait_seconds": 2.5779008865356445e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 71200, "total_steps": 72479, "tokens": 37329305600, "loss": 2.984568804502487, "l_mix": 1.83611561357975, "l_a": 1.9490695744752884, "l_c": 1.8791073113679886, "z_loss": 53.75314712524414, "lr": 7.767812550041895e-05, "wd": 0.01, "grad_norm": 0.08221691846847534, "tokens_per_second": 1415896.997674644, "eta_seconds": 473.59684574604034, "data_wait_seconds": 2.4050474166870117e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.1699198, "gate_a2b/L1_absmax": 0.67085916, "gate_a2b/L1_rms": 0.17897977, "gate_a2b/L3_absmean": 0.15891464, "gate_a2b/L3_absmax": 0.50845695, "gate_a2b/L3_rms": 0.16434659, "gate_a2b/L5_absmean": 0.22721151, "gate_a2b/L5_absmax": 0.73381317, "gate_a2b/L5_rms": 0.23304702, "gate_c2b/L1_absmean": 0.05046188, "gate_c2b/L1_absmax": 0.58920097, "gate_c2b/L1_rms": 0.07305136, "gate_c2b/L3_absmean": 0.16238974, "gate_c2b/L3_absmax": 0.38060871, "gate_c2b/L3_rms": 0.16578591, "gate_c2b/L5_absmean": 0.46939832, "gate_c2b/L5_absmax": 0.6872077, "gate_c2b/L5_rms": 0.47804859, "gate_b2a/L7_absmean": 0.05318406, "gate_b2a/L7_absmax": 0.33848947, "gate_b2a/L7_rms": 0.0588097, "gate_b2a/L11_absmean": 0.08720221, "gate_b2a/L11_absmax": 0.60981363, "gate_b2a/L11_rms": 0.09577586, "gate_b2a/L14_absmean": 0.22495042, "gate_b2a/L14_absmax": 0.78813189, "gate_b2a/L14_rms": 0.23314637, "gate_b2c/L3_absmean": 0.0411694, "gate_b2c/L3_absmax": 0.55708236, "gate_b2c/L3_rms": 0.05056863, "gate_b2c/L5_absmean": 0.04124765, "gate_b2c/L5_absmax": 0.54008758, "gate_b2c/L5_rms": 0.04984964, "gate_b2c/L6_absmean": 0.05930838, "gate_b2c/L6_absmax": 0.32681891, "gate_b2c/L6_rms": 0.06441792}, "streams": {"cos_a2b/round0": 0.352129, "cos_c2b/round0": 0.206057, "cos_b2a/round0": 0.162842, "cos_b2c/round0": 0.131351, "rms_a/round0": 6.860559, "rms_b/round0": 12.021962, "rms_c/round0": 4.900252, "cos_a2b/round1": 0.407793, "cos_c2b/round1": 0.306447, "cos_b2a/round1": 0.254453, "cos_b2c/round1": 0.109408, "rms_a/round1": 14.121623, "rms_b/round1": 29.606878, "rms_c/round1": 11.311258, "cos_a2b/round2": 0.516513, "cos_c2b/round2": 0.515551, "cos_b2a/round2": 0.086777, "cos_b2c/round2": 0.16848, "rms_a/round2": 42.04681, "rms_b/round2": 81.544128, "rms_c/round2": 22.88633}, "alpha_mean": 0.39625007252529443, "alpha_hist": [0.06216, 0.18552, 0.17982, 0.13217, 0.10774, 0.10101, 0.097, 0.08247, 0.04492, 0.00717], "alpha_by_type": {"special": {"share": 0.00105, "mean": 0.63721, "std": 0.13193}, "whitespace": {"share": 0.0381, "mean": 0.62646, "std": 0.17617}, "digit": {"share": 0.01384, "mean": 0.39516, "std": 0.1699}, "word_start": {"share": 0.55658, "mean": 0.40288, "std": 0.21914}, "word_cont": {"share": 0.27353, "mean": 0.2524, "std": 0.14922}, "punct": {"share": 0.1169, "mean": 0.62421, "std": 0.19354}}, "alpha_p05": 0.10195, "alpha_p50": 0.36297, "alpha_p95": 0.80068}
{"step": 71210, "total_steps": 72479, "tokens": 37334548480, "loss": 4.234849572181702, "l_mix": 2.625080108642578, "l_a": 2.7084818482398987, "l_c": 2.657416433095932, "z_loss": 48.68263816833496, "lr": 7.648179879480335e-05, "wd": 0.01, "grad_norm": 0.08586934953927994, "tokens_per_second": 1454591.2410667446, "eta_seconds": 457.3941140413284, "data_wait_seconds": 1.558661460876465e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08233}}
{"step": 71220, "total_steps": 72479, "tokens": 37339791360, "loss": 4.249730467796326, "l_mix": 2.6343058943748474, "l_a": 2.7174333333969116, "l_c": 2.667314797639847, "z_loss": 48.5235013961792, "lr": 7.529443914281619e-05, "wd": 0.01, "grad_norm": 0.08758041262626648, "tokens_per_second": 1462059.0743754683, "eta_seconds": 451.4719025850296, "data_wait_seconds": 2.7120113372802734e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 71230, "total_steps": 72479, "tokens": 37345034240, "loss": 4.3715932965278625, "l_mix": 2.7105544805526733, "l_a": 2.7937610745429993, "l_c": 2.74303475022316, "z_loss": 47.733837604522705, "lr": 7.411605646242346e-05, "wd": 0.01, "grad_norm": 0.08111445605754852, "tokens_per_second": 1394266.81238915, "eta_seconds": 469.66312773227696, "data_wait_seconds": 2.777576446533203e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20044, "finemath": 0.08234}}
{"step": 71240, "total_steps": 72479, "tokens": 37350277120, "loss": 4.283571660518646, "l_mix": 2.6553190648555756, "l_a": 2.7393053472042084, "l_c": 2.688203275203705, "z_loss": 49.083048820495605, "lr": 7.294666059660753e-05, "wd": 0.01, "grad_norm": 0.08606208115816116, "tokens_per_second": 1442110.6923028696, "eta_seconds": 450.4458884239197, "data_wait_seconds": 2.2172927856445312e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 71250, "total_steps": 72479, "tokens": 37355520000, "loss": 4.351621150970459, "l_mix": 2.6981185376644135, "l_a": 2.7788553833961487, "l_c": 2.73281991481781, "z_loss": 48.24848222732544, "lr": 7.178626131328459e-05, "wd": 0.01, "grad_norm": 0.08441611379384995, "tokens_per_second": 1446782.2695661746, "eta_seconds": 445.3676033735275, "data_wait_seconds": 2.0056962966918945e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08234}}
{"step": 71260, "total_steps": 72479, "tokens": 37360762880, "loss": 4.233965635299683, "l_mix": 2.624544382095337, "l_a": 2.706624001264572, "l_c": 2.658113420009613, "z_loss": 49.05447769165039, "lr": 7.063486830522187e-05, "wd": 0.01, "grad_norm": 0.08026953786611557, "tokens_per_second": 1236492.4221651996, "eta_seconds": 516.8709977865219, "data_wait_seconds": 1.868605613708496e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 71270, "total_steps": 72479, "tokens": 37366005760, "loss": 2.9605980217456818, "l_mix": 1.8211093842983246, "l_a": 1.933639109134674, "l_c": 1.8646562099456787, "z_loss": 54.1101131439209, "lr": 6.94924911899587e-05, "wd": 0.01, "grad_norm": 0.08479796350002289, "tokens_per_second": 1292039.5374911653, "eta_seconds": 490.5919467687607, "data_wait_seconds": 2.1159648895263672e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 71280, "total_steps": 72479, "tokens": 37371248640, "loss": 2.985771268606186, "l_mix": 1.8367428183555603, "l_a": 1.9490312486886978, "l_c": 1.8810632824897766, "z_loss": 53.71679735183716, "lr": 6.83591395097239e-05, "wd": 0.01, "grad_norm": 0.08114202320575714, "tokens_per_second": 1410723.4218104389, "eta_seconds": 445.6020948410034, "data_wait_seconds": 3.415346145629883e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 71290, "total_steps": 72479, "tokens": 37376491520, "loss": 2.979015350341797, "l_mix": 1.8326181918382645, "l_a": 1.9461207836866379, "l_c": 1.875202938914299, "z_loss": 53.88313150405884, "lr": 6.723482273135804e-05, "wd": 0.01, "grad_norm": 0.08250181376934052, "tokens_per_second": 1387370.116371299, "eta_seconds": 449.3238139152527, "data_wait_seconds": 3.49581241607666e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 71300, "total_steps": 72479, "tokens": 37381734400, "loss": 2.9256772100925446, "l_mix": 1.7999216169118881, "l_a": 1.9095079600811005, "l_c": 1.8430101722478867, "z_loss": 54.894400119781494, "lr": 6.61195502462326e-05, "wd": 0.01, "grad_norm": 0.08487846702337265, "tokens_per_second": 1425348.4130135074, "eta_seconds": 433.6733014583587, "data_wait_seconds": 3.6090612411499023e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16996063, "gate_a2b/L1_absmax": 0.67098665, "gate_a2b/L1_rms": 0.17902279, "gate_a2b/L3_absmean": 0.15892878, "gate_a2b/L3_absmax": 0.50840491, "gate_a2b/L3_rms": 0.16436274, "gate_a2b/L5_absmean": 0.22725084, "gate_a2b/L5_absmax": 0.73374957, "gate_a2b/L5_rms": 0.23307872, "gate_c2b/L1_absmean": 0.05049611, "gate_c2b/L1_absmax": 0.58899015, "gate_c2b/L1_rms": 0.07302454, "gate_c2b/L3_absmean": 0.16236058, "gate_c2b/L3_absmax": 0.38060048, "gate_c2b/L3_rms": 0.16576089, "gate_c2b/L5_absmean": 0.46939331, "gate_c2b/L5_absmax": 0.68718207, "gate_c2b/L5_rms": 0.47804168, "gate_b2a/L7_absmean": 0.0531714, "gate_b2a/L7_absmax": 0.3384515, "gate_b2a/L7_rms": 0.05879557, "gate_b2a/L11_absmean": 0.08719105, "gate_b2a/L11_absmax": 0.60974431, "gate_b2a/L11_rms": 0.09576397, "gate_b2a/L14_absmean": 0.22499299, "gate_b2a/L14_absmax": 0.78797722, "gate_b2a/L14_rms": 0.23318344, "gate_b2c/L3_absmean": 0.04118196, "gate_b2c/L3_absmax": 0.55729765, "gate_b2c/L3_rms": 0.05058856, "gate_b2c/L5_absmean": 0.0412417, "gate_b2c/L5_absmax": 0.53988248, "gate_b2c/L5_rms": 0.04983528, "gate_b2c/L6_absmean": 0.05936244, "gate_b2c/L6_absmax": 0.32688496, "gate_b2c/L6_rms": 0.06446572}, "streams": {"cos_a2b/round0": 0.3513, "cos_c2b/round0": 0.204184, "cos_b2a/round0": 0.164092, "cos_b2c/round0": 0.130764, "rms_a/round0": 6.86537, "rms_b/round0": 12.016458, "rms_c/round0": 4.904805, "cos_a2b/round1": 0.404839, "cos_c2b/round1": 0.308374, "cos_b2a/round1": 0.255024, "cos_b2c/round1": 0.111827, "rms_a/round1": 14.062489, "rms_b/round1": 29.639755, "rms_c/round1": 11.336617, "cos_a2b/round2": 0.513822, "cos_c2b/round2": 0.515527, "cos_b2a/round2": 0.087492, "cos_b2c/round2": 0.168422, "rms_a/round2": 41.829037, "rms_b/round2": 81.313133, "rms_c/round2": 22.897978}, "alpha_mean": 0.3956867169033913, "alpha_hist": [0.06281, 0.1888, 0.17931, 0.13004, 0.10469, 0.09953, 0.09875, 0.08391, 0.04489, 0.00728], "alpha_by_type": {"special": {"share": 0.00104, "mean": 0.67023, "std": 0.12865}, "whitespace": {"share": 0.03317, "mean": 0.63358, "std": 0.18763}, "digit": {"share": 0.01201, "mean": 0.38815, "std": 0.19573}, "word_start": {"share": 0.56168, "mean": 0.39905, "std": 0.22066}, "word_cont": {"share": 0.26738, "mean": 0.25403, "std": 0.14801}, "punct": {"share": 0.12473, "mean": 0.61938, "std": 0.19252}}, "alpha_p05": 0.0882, "alpha_p50": 0.34378, "alpha_p95": 0.80716}
{"step": 71310, "total_steps": 72479, "tokens": 37386977280, "loss": 2.9763329923152924, "l_mix": 1.8309933692216873, "l_a": 1.9425733238458633, "l_c": 1.8752254247665405, "z_loss": 54.419095516204834, "lr": 6.501333137017392e-05, "wd": 0.01, "grad_norm": 0.08244160562753677, "tokens_per_second": 1354359.572413681, "eta_seconds": 452.5332005500793, "data_wait_seconds": 1.940131187438965e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 71320, "total_steps": 72479, "tokens": 37392220160, "loss": 4.293679356575012, "l_mix": 2.661832094192505, "l_a": 2.744411289691925, "l_c": 2.6950797140598297, "z_loss": 48.31260871887207, "lr": 6.391617534338231e-05, "wd": 0.01, "grad_norm": 0.09866119921207428, "tokens_per_second": 1409696.348945582, "eta_seconds": 431.0501282453537, "data_wait_seconds": 1.576542854309082e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 71330, "total_steps": 72479, "tokens": 37397463040, "loss": 4.262413680553436, "l_mix": 2.6425228118896484, "l_a": 2.724654644727707, "l_c": 2.6749813556671143, "z_loss": 48.9115252494812, "lr": 6.282809133035759e-05, "wd": 0.01, "grad_norm": 0.0816512182354927, "tokens_per_second": 1477270.5324338402, "eta_seconds": 407.7837462902069, "data_wait_seconds": 1.8775463104248047e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 71340, "total_steps": 72479, "tokens": 37402705920, "loss": 4.17983415722847, "l_mix": 2.5909001231193542, "l_a": 2.672886312007904, "l_c": 2.623559981584549, "z_loss": 48.57600736618042, "lr": 6.174908841982067e-05, "wd": 0.01, "grad_norm": 0.08536525815725327, "tokens_per_second": 1465682.5983271666, "eta_seconds": 407.4306624650955, "data_wait_seconds": 1.531839370727539e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 71350, "total_steps": 72479, "tokens": 37407948800, "loss": 4.310511410236359, "l_mix": 2.672105371952057, "l_a": 2.7561630308628082, "l_c": 2.7051901519298553, "z_loss": 49.076711654663086, "lr": 6.067917562463905e-05, "wd": 0.01, "grad_norm": 0.07803148031234741, "tokens_per_second": 1424064.7400260991, "eta_seconds": 415.656069111824, "data_wait_seconds": 1.6480684280395508e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 71360, "total_steps": 72479, "tokens": 37413191680, "loss": 4.160781502723694, "l_mix": 2.578583687543869, "l_a": 2.66188982129097, "l_c": 2.6121027171611786, "z_loss": 49.48059844970703, "lr": 5.961836188175032e-05, "wd": 0.01, "grad_norm": 0.0821157768368721, "tokens_per_second": 1481116.2740438902, "eta_seconds": 396.1054795503617, "data_wait_seconds": 1.481175422668457e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 71370, "total_steps": 72479, "tokens": 37418434560, "loss": 4.5789583921432495, "l_mix": 2.841229945421219, "l_a": 2.9153074622154236, "l_c": 2.877120465040207, "z_loss": 47.71634387969971, "lr": 5.856665605208842e-05, "wd": 0.01, "grad_norm": 0.07803580909967422, "tokens_per_second": 1468795.8362997263, "eta_seconds": 395.85855135917666, "data_wait_seconds": 1.7255544662475586e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20042, "finemath": 0.08235}}
{"step": 71380, "total_steps": 72479, "tokens": 37423677440, "loss": 2.9505741000175476, "l_mix": 1.8150915205478668, "l_a": 1.9263309836387634, "l_c": 1.858610987663269, "z_loss": 54.908472537994385, "lr": 5.752406692050922e-05, "wd": 0.01, "grad_norm": 0.08064162731170654, "tokens_per_second": 1245337.5404147117, "eta_seconds": 462.6797902584076, "data_wait_seconds": 2.765655517578125e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 71390, "total_steps": 72479, "tokens": 37428920320, "loss": 2.9289733469486237, "l_mix": 1.8016913384199142, "l_a": 1.9134093821048737, "l_c": 1.8441971838474274, "z_loss": 54.496864318847656, "lr": 5.6490603195716814e-05, "wd": 0.01, "grad_norm": 0.08037764579057693, "tokens_per_second": 1024695.532836974, "eta_seconds": 557.1895394325256, "data_wait_seconds": 2.5570392608642578e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 71400, "total_steps": 72479, "tokens": 37434163200, "loss": 2.9181419014930725, "l_mix": 1.7949054539203644, "l_a": 1.9070303738117218, "l_c": 1.8370908051729202, "z_loss": 54.53515005111694, "lr": 5.546627351019167e-05, "wd": 0.01, "grad_norm": 0.08378082513809204, "tokens_per_second": 1456726.348238973, "eta_seconds": 388.34112713336947, "data_wait_seconds": 1.7136335372924805e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16997333, "gate_a2b/L1_absmax": 0.67118746, "gate_a2b/L1_rms": 0.17903098, "gate_a2b/L3_absmean": 0.15899043, "gate_a2b/L3_absmax": 0.50861043, "gate_a2b/L3_rms": 0.16442037, "gate_a2b/L5_absmean": 0.22725895, "gate_a2b/L5_absmax": 0.73383838, "gate_a2b/L5_rms": 0.23308599, "gate_c2b/L1_absmean": 0.0505537, "gate_c2b/L1_absmax": 0.58917648, "gate_c2b/L1_rms": 0.07306741, "gate_c2b/L3_absmean": 0.16232489, "gate_c2b/L3_absmax": 0.38056472, "gate_c2b/L3_rms": 0.16572696, "gate_c2b/L5_absmean": 0.46940216, "gate_c2b/L5_absmax": 0.6871376, "gate_c2b/L5_rms": 0.47804487, "gate_b2a/L7_absmean": 0.05319967, "gate_b2a/L7_absmax": 0.33887699, "gate_b2a/L7_rms": 0.0588306, "gate_b2a/L11_absmean": 0.08715575, "gate_b2a/L11_absmax": 0.60994738, "gate_b2a/L11_rms": 0.09574018, "gate_b2a/L14_absmean": 0.22501929, "gate_b2a/L14_absmax": 0.78756464, "gate_b2a/L14_rms": 0.23320168, "gate_b2c/L3_absmean": 0.04120274, "gate_b2c/L3_absmax": 0.55718434, "gate_b2c/L3_rms": 0.05059732, "gate_b2c/L5_absmean": 0.04124551, "gate_b2c/L5_absmax": 0.53969282, "gate_b2c/L5_rms": 0.04984294, "gate_b2c/L6_absmean": 0.05936465, "gate_b2c/L6_absmax": 0.32693616, "gate_b2c/L6_rms": 0.06447461}, "streams": {"cos_a2b/round0": 0.351914, "cos_c2b/round0": 0.205192, "cos_b2a/round0": 0.164369, "cos_b2c/round0": 0.131011, "rms_a/round0": 6.876503, "rms_b/round0": 12.02773, "rms_c/round0": 4.906217, "cos_a2b/round1": 0.408011, "cos_c2b/round1": 0.308323, "cos_b2a/round1": 0.255449, "cos_b2c/round1": 0.111917, "rms_a/round1": 14.139365, "rms_b/round1": 29.718519, "rms_c/round1": 11.392181, "cos_a2b/round2": 0.515377, "cos_c2b/round2": 0.517004, "cos_b2a/round2": 0.087666, "cos_b2c/round2": 0.167587, "rms_a/round2": 41.969864, "rms_b/round2": 81.525879, "rms_c/round2": 23.060526}, "alpha_mean": 0.3906251248286594, "alpha_hist": [0.06561, 0.19751, 0.18207, 0.12683, 0.10152, 0.0965, 0.09659, 0.0817, 0.04422, 0.00746], "alpha_by_type": {"special": {"share": 0.00111, "mean": 0.64907, "std": 0.12821}, "whitespace": {"share": 0.03516, "mean": 0.63655, "std": 0.18069}, "digit": {"share": 0.00954, "mean": 0.4138, "std": 0.1894}, "word_start": {"share": 0.57062, "mean": 0.39468, "std": 0.22094}, "word_cont": {"share": 0.26775, "mean": 0.24933, "std": 0.14737}, "punct": {"share": 0.11583, "mean": 0.61824, "std": 0.2015}}, "alpha_p05": 0.09912, "alpha_p50": 0.35712, "alpha_p95": 0.80684}
{"step": 71410, "total_steps": 72479, "tokens": 37439406080, "loss": 2.939112573862076, "l_mix": 1.8079015165567398, "l_a": 1.918428674340248, "l_c": 1.8522747159004211, "z_loss": 54.74137830734253, "lr": 5.445108642011758e-05, "wd": 0.01, "grad_norm": 0.07861077785491943, "tokens_per_second": 1170342.4498366674, "eta_seconds": 478.8887834310531, "data_wait_seconds": 1.7404556274414062e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 71420, "total_steps": 72479, "tokens": 37444648960, "loss": 2.9263941049575806, "l_mix": 1.800238698720932, "l_a": 1.9106658846139908, "l_c": 1.843185469508171, "z_loss": 54.558743953704834, "lr": 5.344505040531107e-05, "wd": 0.01, "grad_norm": 0.07933678478002548, "tokens_per_second": 1348879.607880735, "eta_seconds": 411.6164176225662, "data_wait_seconds": 1.6003847122192383e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 71430, "total_steps": 72479, "tokens": 37449891840, "loss": 4.20344614982605, "l_mix": 2.605499804019928, "l_a": 2.6882340013980865, "l_c": 2.6382534205913544, "z_loss": 48.68863916397095, "lr": 5.24481738691495e-05, "wd": 0.01, "grad_norm": 0.08404970169067383, "tokens_per_second": 1163409.8794367758, "eta_seconds": 472.7294496297837, "data_wait_seconds": 2.4765729904174805e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 71440, "total_steps": 72479, "tokens": 37455134720, "loss": 4.31697890162468, "l_mix": 2.6762701869010925, "l_a": 2.76024267077446, "l_c": 2.708786278963089, "z_loss": 48.04284381866455, "lr": 5.146046513850202e-05, "wd": 0.01, "grad_norm": 0.08156555145978928, "tokens_per_second": 1372536.7816318946, "eta_seconds": 396.88206486701966, "data_wait_seconds": 1.5854835510253906e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 71450, "total_steps": 72479, "tokens": 37460377600, "loss": 4.333880364894867, "l_mix": 2.6875252723693848, "l_a": 2.767597943544388, "l_c": 2.7202519178390503, "z_loss": 48.36556959152222, "lr": 5.048193246365905e-05, "wd": 0.01, "grad_norm": 0.07680275291204453, "tokens_per_second": 1321316.7816284231, "eta_seconds": 408.29902374744415, "data_wait_seconds": 1.7821788787841797e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 71460, "total_steps": 72479, "tokens": 37465620480, "loss": 4.278819382190704, "l_mix": 2.652256727218628, "l_a": 2.736392229795456, "l_c": 2.6854832768440247, "z_loss": 48.94468116760254, "lr": 4.95125840182644e-05, "wd": 0.01, "grad_norm": 0.07972538471221924, "tokens_per_second": 1459762.6590102185, "eta_seconds": 365.98379106521605, "data_wait_seconds": 1.761317253112793e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 71470, "total_steps": 72479, "tokens": 37470863360, "loss": 4.125276029109955, "l_mix": 2.5570244193077087, "l_a": 2.637885183095932, "l_c": 2.5896200835704803, "z_loss": 48.977556228637695, "lr": 4.855242789924542e-05, "wd": 0.01, "grad_norm": 0.0809580609202385, "tokens_per_second": 1266372.8387678524, "eta_seconds": 417.73368458747865, "data_wait_seconds": 0.06688860058784485, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 71480, "total_steps": 72479, "tokens": 37476106240, "loss": 2.9433116912841797, "l_mix": 1.8104992359876633, "l_a": 1.922238752245903, "l_c": 1.8538026064634323, "z_loss": 54.2733211517334, "lr": 4.7601472126747345e-05, "wd": 0.01, "grad_norm": 0.0821019858121872, "tokens_per_second": 1463891.104929859, "eta_seconds": 357.7887113571167, "data_wait_seconds": 2.2411346435546875e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 71490, "total_steps": 72479, "tokens": 37481349120, "loss": 4.504658997058868, "l_mix": 2.7953203320503235, "l_a": 2.8696627616882324, "l_c": 2.828132390975952, "z_loss": 47.270487785339355, "lr": 4.665972464406477e-05, "wd": 0.01, "grad_norm": 0.07995086908340454, "tokens_per_second": 1279643.9750104875, "eta_seconds": 405.20710613727573, "data_wait_seconds": 1.940131187438965e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 71500, "total_steps": 72479, "tokens": 37486592000, "loss": 4.606218576431274, "l_mix": 2.858388274908066, "l_a": 2.9329788386821747, "l_c": 2.8931219577789307, "z_loss": 46.5535569190979, "lr": 4.5727193317575514e-05, "wd": 0.01, "grad_norm": 0.08726883679628372, "tokens_per_second": 1349121.5008502014, "eta_seconds": 380.4534667015076, "data_wait_seconds": 1.77919864654541e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.16999009, "gate_a2b/L1_absmax": 0.67143869, "gate_a2b/L1_rms": 0.17904556, "gate_a2b/L3_absmean": 0.1589866, "gate_a2b/L3_absmax": 0.50864923, "gate_a2b/L3_rms": 0.16441596, "gate_a2b/L5_absmean": 0.22726874, "gate_a2b/L5_absmax": 0.73364598, "gate_a2b/L5_rms": 0.23309647, "gate_c2b/L1_absmean": 0.05054612, "gate_c2b/L1_absmax": 0.5892089, "gate_c2b/L1_rms": 0.07302348, "gate_c2b/L3_absmean": 0.1622972, "gate_c2b/L3_absmax": 0.38052255, "gate_c2b/L3_rms": 0.16569906, "gate_c2b/L5_absmean": 0.46941677, "gate_c2b/L5_absmax": 0.68710089, "gate_c2b/L5_rms": 0.47806224, "gate_b2a/L7_absmean": 0.05318952, "gate_b2a/L7_absmax": 0.33890188, "gate_b2a/L7_rms": 0.0588257, "gate_b2a/L11_absmean": 0.08717014, "gate_b2a/L11_absmax": 0.60989493, "gate_b2a/L11_rms": 0.095749, "gate_b2a/L14_absmean": 0.22504434, "gate_b2a/L14_absmax": 0.78770274, "gate_b2a/L14_rms": 0.23323247, "gate_b2c/L3_absmean": 0.04122447, "gate_b2c/L3_absmax": 0.55705488, "gate_b2c/L3_rms": 0.05061356, "gate_b2c/L5_absmean": 0.04125616, "gate_b2c/L5_absmax": 0.53992349, "gate_b2c/L5_rms": 0.04985753, "gate_b2c/L6_absmean": 0.05938066, "gate_b2c/L6_absmax": 0.3267704, "gate_b2c/L6_rms": 0.0644805}, "streams": {"cos_a2b/round0": 0.352721, "cos_c2b/round0": 0.234382, "cos_b2a/round0": 0.181531, "cos_b2c/round0": 0.154621, "rms_a/round0": 6.535359, "rms_b/round0": 11.668454, "rms_c/round0": 4.691024, "cos_a2b/round1": 0.407725, "cos_c2b/round1": 0.311848, "cos_b2a/round1": 0.263021, "cos_b2c/round1": 0.106162, "rms_a/round1": 13.193011, "rms_b/round1": 27.688959, "rms_c/round1": 10.467374, "cos_a2b/round2": 0.545136, "cos_c2b/round2": 0.517298, "cos_b2a/round2": 0.099818, "cos_b2c/round2": 0.184016, "rms_a/round2": 40.741596, "rms_b/round2": 79.298714, "rms_c/round2": 21.02915}, "alpha_mean": 0.4440911635433622, "alpha_hist": [0.01062, 0.06804, 0.16074, 0.18677, 0.18166, 0.17871, 0.13498, 0.06874, 0.00949, 0.00026], "alpha_by_type": {"special": {"share": 0.00067, "mean": 0.566, "std": 0.15567}, "whitespace": {"share": 0.06685, "mean": 0.54318, "std": 0.1167}, "digit": {"share": 0.0349, "mean": 0.45276, "std": 0.14545}, "word_start": {"share": 0.53749, "mean": 0.46305, "std": 0.16465}, "word_cont": {"share": 0.25328, "mean": 0.32426, "std": 0.13814}, "punct": {"share": 0.10681, "mean": 0.56725, "std": 0.1653}}, "alpha_p05": 0.18952, "alpha_p50": 0.44348, "alpha_p95": 0.72174}
{"step": 71510, "total_steps": 72479, "tokens": 37491834880, "loss": 2.94429612159729, "l_mix": 1.8113974779844284, "l_a": 1.921775922179222, "l_c": 1.8545527160167694, "z_loss": 54.77719831466675, "lr": 4.4803885936676354e-05, "wd": 0.01, "grad_norm": 0.08277734369039536, "tokens_per_second": 1364802.564406487, "eta_seconds": 372.2407073736191, "data_wait_seconds": 2.4139881134033203e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 71520, "total_steps": 72479, "tokens": 37497077760, "loss": 2.9519046247005463, "l_mix": 1.815631479024887, "l_a": 1.9280946403741837, "l_c": 1.8594825714826584, "z_loss": 54.40532732009888, "lr": 4.388981021371542e-05, "wd": 0.01, "grad_norm": 0.08152393251657486, "tokens_per_second": 1373812.8767649778, "eta_seconds": 365.9830246925354, "data_wait_seconds": 2.816319465637207e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 71530, "total_steps": 72479, "tokens": 37502320640, "loss": 2.954332858324051, "l_mix": 1.8176571875810623, "l_a": 1.928156167268753, "l_c": 1.8607626557350159, "z_loss": 54.56493043899536, "lr": 4.2984973783930215e-05, "wd": 0.01, "grad_norm": 0.08135435730218887, "tokens_per_second": 1316212.4508143372, "eta_seconds": 378.0159591197968, "data_wait_seconds": 2.4259090423583984e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 71540, "total_steps": 72479, "tokens": 37507563520, "loss": 4.371596932411194, "l_mix": 2.710842549800873, "l_a": 2.791464388370514, "l_c": 2.7443831861019135, "z_loss": 48.810049533843994, "lr": 4.208938420538216e-05, "wd": 0.01, "grad_norm": 0.08560620248317719, "tokens_per_second": 1450051.404439392, "eta_seconds": 339.5096411705017, "data_wait_seconds": 2.5004148483276367e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 71550, "total_steps": 72479, "tokens": 37512806400, "loss": 4.277941286563873, "l_mix": 2.65192249417305, "l_a": 2.7351502180099487, "l_c": 2.6849122047424316, "z_loss": 48.59094429016113, "lr": 4.120304895889451e-05, "wd": 0.01, "grad_norm": 0.08083134889602661, "tokens_per_second": 1339027.0032137525, "eta_seconds": 363.74438366889956, "data_wait_seconds": 1.8388032913208008e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 71560, "total_steps": 72479, "tokens": 37518049280, "loss": 4.41200590133667, "l_mix": 2.7358813285827637, "l_a": 2.8154677152633667, "l_c": 2.771614223718643, "z_loss": 48.80438041687012, "lr": 4.032597544798921e-05, "wd": 0.01, "grad_norm": 0.09393911063671112, "tokens_per_second": 1464588.9940211568, "eta_seconds": 328.98012614250183, "data_wait_seconds": 3.2782554626464844e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 71570, "total_steps": 72479, "tokens": 37523292160, "loss": 4.338872671127319, "l_mix": 2.689958095550537, "l_a": 2.773843079805374, "l_c": 2.7225387692451477, "z_loss": 48.09541082382202, "lr": 3.945817099882565e-05, "wd": 0.01, "grad_norm": 0.08107297122478485, "tokens_per_second": 1477074.8555027677, "eta_seconds": 322.6497223377228, "data_wait_seconds": 3.165006637573242e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 71580, "total_steps": 72479, "tokens": 37528535040, "loss": 4.282187283039093, "l_mix": 2.6541313230991364, "l_a": 2.7392534613609314, "l_c": 2.6875995695590973, "z_loss": 48.606751918792725, "lr": 3.859964286013894e-05, "wd": 0.01, "grad_norm": 0.0824117437005043, "tokens_per_second": 1309059.1384588936, "eta_seconds": 360.05624051094054, "data_wait_seconds": 2.0384788513183594e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 71590, "total_steps": 72479, "tokens": 37533777920, "loss": 2.916368246078491, "l_mix": 1.7939794957637787, "l_a": 1.9055359810590744, "l_c": 1.8357597142457962, "z_loss": 54.93325662612915, "lr": 3.775039820317926e-05, "wd": 0.01, "grad_norm": 0.08158477395772934, "tokens_per_second": 1444752.6930503268, "eta_seconds": 322.6102531194687, "data_wait_seconds": 3.501772880554199e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 71600, "total_steps": 72479, "tokens": 37539020800, "loss": 2.6716100573539734, "l_mix": 1.6543956398963928, "l_a": 1.7142348885536194, "l_c": 1.6764797866344452, "z_loss": 62.08244705200195, "lr": 3.6910444121652456e-05, "wd": 0.01, "grad_norm": 0.154815673828125, "tokens_per_second": 1441217.6250020235, "eta_seconds": 319.7637497663498, "data_wait_seconds": 3.153085708618164e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}, "gates": {"gate_a2b/L1_absmean": 0.17002504, "gate_a2b/L1_absmax": 0.671242, "gate_a2b/L1_rms": 0.17908502, "gate_a2b/L3_absmean": 0.15898748, "gate_a2b/L3_absmax": 0.50845355, "gate_a2b/L3_rms": 0.16441576, "gate_a2b/L5_absmean": 0.22731309, "gate_a2b/L5_absmax": 0.73371714, "gate_a2b/L5_rms": 0.23313725, "gate_c2b/L1_absmean": 0.05059487, "gate_c2b/L1_absmax": 0.58919644, "gate_c2b/L1_rms": 0.07303734, "gate_c2b/L3_absmean": 0.16228829, "gate_c2b/L3_absmax": 0.38035333, "gate_c2b/L3_rms": 0.16568591, "gate_c2b/L5_absmean": 0.46941119, "gate_c2b/L5_absmax": 0.68719417, "gate_c2b/L5_rms": 0.47806069, "gate_b2a/L7_absmean": 0.05317757, "gate_b2a/L7_absmax": 0.33881271, "gate_b2a/L7_rms": 0.05881401, "gate_b2a/L11_absmean": 0.08719234, "gate_b2a/L11_absmax": 0.6097272, "gate_b2a/L11_rms": 0.09576705, "gate_b2a/L14_absmean": 0.22507495, "gate_b2a/L14_absmax": 0.78747791, "gate_b2a/L14_rms": 0.23325704, "gate_b2c/L3_absmean": 0.04121609, "gate_b2c/L3_absmax": 0.55731428, "gate_b2c/L3_rms": 0.05061173, "gate_b2c/L5_absmean": 0.04129356, "gate_b2c/L5_absmax": 0.5399285, "gate_b2c/L5_rms": 0.04988689, "gate_b2c/L6_absmean": 0.05939692, "gate_b2c/L6_absmax": 0.3269513, "gate_b2c/L6_rms": 0.06449433}, "streams": {"cos_a2b/round0": 0.362135, "cos_c2b/round0": 0.242383, "cos_b2a/round0": 0.169313, "cos_b2c/round0": 0.151886, "rms_a/round0": 6.754246, "rms_b/round0": 11.432059, "rms_c/round0": 4.706314, "cos_a2b/round1": 0.388216, "cos_c2b/round1": 0.285341, "cos_b2a/round1": 0.242299, "cos_b2c/round1": 0.089816, "rms_a/round1": 13.457608, "rms_b/round1": 27.506765, "rms_c/round1": 10.454076, "cos_a2b/round2": 0.501316, "cos_c2b/round2": 0.543979, "cos_b2a/round2": 0.080469, "cos_b2c/round2": 0.161282, "rms_a/round2": 39.840374, "rms_b/round2": 80.855034, "rms_c/round2": 21.217718}, "alpha_mean": 0.40372422295732235, "alpha_hist": [0.0372, 0.0948, 0.17525, 0.19867, 0.18407, 0.15836, 0.10039, 0.04385, 0.00681, 0.00061], "alpha_by_type": {"special": {"share": 0.00072, "mean": 0.5387, "std": 0.18896}, "whitespace": {"share": 0.0873, "mean": 0.48467, "std": 0.1591}, "digit": {"share": 0.08452, "mean": 0.35909, "std": 0.14706}, "word_start": {"share": 0.42282, "mean": 0.40449, "std": 0.17395}, "word_cont": {"share": 0.20094, "mean": 0.29767, "std": 0.13737}, "punct": {"share": 0.2037, "mean": 0.49011, "std": 0.17422}}, "alpha_p05": 0.11596, "alpha_p50": 0.397, "alpha_p95": 0.70999}
{"step": 71610, "total_steps": 72479, "tokens": 37544263680, "loss": 4.55299311876297, "l_mix": 2.8254798352718353, "l_a": 2.8987065851688385, "l_c": 2.859670966863632, "z_loss": 47.3352575302124, "lr": 3.607978763166082e-05, "wd": 0.01, "grad_norm": 0.08729507029056549, "tokens_per_second": 1420502.3251045195, "eta_seconds": 320.73602693080903, "data_wait_seconds": 1.615285873413086e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 71620, "total_steps": 72479, "tokens": 37549506560, "loss": 4.591493189334869, "l_mix": 2.8494880199432373, "l_a": 2.922031968832016, "l_c": 2.8846515715122223, "z_loss": 47.38874578475952, "lr": 3.52584356716441e-05, "wd": 0.01, "grad_norm": 0.07824863493442535, "tokens_per_second": 1274549.898855479, "eta_seconds": 353.350929927826, "data_wait_seconds": 2.6524066925048828e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 71630, "total_steps": 72479, "tokens": 37554749440, "loss": 4.644649922847748, "l_mix": 2.882015973329544, "l_a": 2.957951605319977, "l_c": 2.917494982481003, "z_loss": 47.04279184341431, "lr": 3.444639510232141e-05, "wd": 0.01, "grad_norm": 0.08252394944429398, "tokens_per_second": 1349968.8513188025, "eta_seconds": 329.72650558948516, "data_wait_seconds": 2.1189451217651367e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 71640, "total_steps": 72479, "tokens": 37559992320, "loss": 4.585590064525604, "l_mix": 2.845504343509674, "l_a": 2.9185700118541718, "l_c": 2.8817156553268433, "z_loss": 47.295762062072754, "lr": 3.3643672706634554e-05, "wd": 0.01, "grad_norm": 0.08167675882577896, "tokens_per_second": 1465749.421323049, "eta_seconds": 300.1042508363724, "data_wait_seconds": 2.03549861907959e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 71650, "total_steps": 72479, "tokens": 37565235200, "loss": 2.9489272832870483, "l_mix": 1.8139134794473648, "l_a": 1.9261334538459778, "l_c": 1.857245534658432, "z_loss": 54.55293607711792, "lr": 3.285027518969066e-05, "wd": 0.01, "grad_norm": 0.0795406624674797, "tokens_per_second": 1388041.1799828527, "eta_seconds": 313.1281393289566, "data_wait_seconds": 1.6391277313232422e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 71660, "total_steps": 72479, "tokens": 37570478080, "loss": 4.338724583387375, "l_mix": 2.6901532113552094, "l_a": 2.771729350090027, "l_c": 2.7235083878040314, "z_loss": 48.95941400527954, "lr": 3.2066209178707065e-05, "wd": 0.01, "grad_norm": 0.08972948789596558, "tokens_per_second": 1319213.3537486289, "eta_seconds": 325.49084708690646, "data_wait_seconds": 2.422928810119629e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 71670, "total_steps": 72479, "tokens": 37575720960, "loss": 4.311928749084473, "l_mix": 2.6734242737293243, "l_a": 2.7542455792427063, "l_c": 2.7074359357357025, "z_loss": 49.07567882537842, "lr": 3.129148122295487e-05, "wd": 0.01, "grad_norm": 0.08013180643320084, "tokens_per_second": 1367740.2654196622, "eta_seconds": 310.10931148529056, "data_wait_seconds": 1.5348196029663086e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 71680, "total_steps": 72479, "tokens": 37580963840, "loss": 4.29178386926651, "l_mix": 2.660421520471573, "l_a": 2.743747740983963, "l_c": 2.6941265165805817, "z_loss": 49.39164590835571, "lr": 3.0526097793704936e-05, "wd": 0.01, "grad_norm": 0.08411604166030884, "tokens_per_second": 1464517.4000355902, "eta_seconds": 286.03696479797367, "data_wait_seconds": 2.995133399963379e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 71690, "total_steps": 72479, "tokens": 37586206720, "loss": 4.231971025466919, "l_mix": 2.6236810088157654, "l_a": 2.7034134566783905, "l_c": 2.6575531661510468, "z_loss": 49.693856716156006, "lr": 2.977006528417398e-05, "wd": 0.01, "grad_norm": 0.08043535053730011, "tokens_per_second": 1389338.030424728, "eta_seconds": 297.74124290943143, "data_wait_seconds": 3.185868263244629e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 71700, "total_steps": 72479, "tokens": 37591449600, "loss": 2.9870554506778717, "l_mix": 1.8377524018287659, "l_a": 1.949137106537819, "l_c": 1.8818730115890503, "z_loss": 54.163862228393555, "lr": 2.9023390009470472e-05, "wd": 0.01, "grad_norm": 0.07765484601259232, "tokens_per_second": 1426430.719889073, "eta_seconds": 286.32330074310306, "data_wait_seconds": 2.822279930114746e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.17005821, "gate_a2b/L1_absmax": 0.67120504, "gate_a2b/L1_rms": 0.17910895, "gate_a2b/L3_absmean": 0.15900405, "gate_a2b/L3_absmax": 0.50877446, "gate_a2b/L3_rms": 0.16443054, "gate_a2b/L5_absmean": 0.22732221, "gate_a2b/L5_absmax": 0.73383248, "gate_a2b/L5_rms": 0.23314969, "gate_c2b/L1_absmean": 0.05059482, "gate_c2b/L1_absmax": 0.5893054, "gate_c2b/L1_rms": 0.07303409, "gate_c2b/L3_absmean": 0.16228297, "gate_c2b/L3_absmax": 0.38059956, "gate_c2b/L3_rms": 0.16568014, "gate_c2b/L5_absmean": 0.46941048, "gate_c2b/L5_absmax": 0.68726671, "gate_c2b/L5_rms": 0.47806156, "gate_b2a/L7_absmean": 0.0531783, "gate_b2a/L7_absmax": 0.33892381, "gate_b2a/L7_rms": 0.05880952, "gate_b2a/L11_absmean": 0.08719826, "gate_b2a/L11_absmax": 0.60983831, "gate_b2a/L11_rms": 0.09577572, "gate_b2a/L14_absmean": 0.22507806, "gate_b2a/L14_absmax": 0.78746158, "gate_b2a/L14_rms": 0.23325634, "gate_b2c/L3_absmean": 0.04125899, "gate_b2c/L3_absmax": 0.55742961, "gate_b2c/L3_rms": 0.05065059, "gate_b2c/L5_absmean": 0.04128242, "gate_b2c/L5_absmax": 0.54007655, "gate_b2c/L5_rms": 0.04988053, "gate_b2c/L6_absmean": 0.05940999, "gate_b2c/L6_absmax": 0.32679924, "gate_b2c/L6_rms": 0.06451003}, "streams": {"cos_a2b/round0": 0.352207, "cos_c2b/round0": 0.207453, "cos_b2a/round0": 0.166073, "cos_b2c/round0": 0.131959, "rms_a/round0": 6.866235, "rms_b/round0": 12.024113, "rms_c/round0": 4.891891, "cos_a2b/round1": 0.40693, "cos_c2b/round1": 0.309439, "cos_b2a/round1": 0.254767, "cos_b2c/round1": 0.11247, "rms_a/round1": 14.143884, "rms_b/round1": 29.571772, "rms_c/round1": 11.353917, "cos_a2b/round2": 0.518842, "cos_c2b/round2": 0.513681, "cos_b2a/round2": 0.091312, "cos_b2c/round2": 0.170628, "rms_a/round2": 42.128254, "rms_b/round2": 81.410904, "rms_c/round2": 22.950445}, "alpha_mean": 0.3947482549382393, "alpha_hist": [0.07072, 0.1871, 0.17178, 0.12817, 0.10632, 0.10199, 0.09871, 0.0838, 0.04446, 0.00693], "alpha_by_type": {"special": {"share": 0.0011, "mean": 0.64773, "std": 0.15696}, "whitespace": {"share": 0.03487, "mean": 0.62335, "std": 0.18623}, "digit": {"share": 0.00943, "mean": 0.37934, "std": 0.21955}, "word_start": {"share": 0.57217, "mean": 0.39718, "std": 0.21957}, "word_cont": {"share": 0.25757, "mean": 0.24935, "std": 0.15334}, "punct": {"share": 0.12486, "mean": 0.61866, "std": 0.1944}}, "alpha_p05": 0.07505, "alpha_p50": 0.34599, "alpha_p95": 0.80007}
{"step": 71710, "total_steps": 72479, "tokens": 37596692480, "loss": 2.9128082990646362, "l_mix": 1.7919552624225616, "l_a": 1.9020906388759613, "l_c": 1.8340860903263092, "z_loss": 54.998300075531006, "lr": 2.8286078206542536e-05, "wd": 0.01, "grad_norm": 0.07858776301145554, "tokens_per_second": 1475573.4817435443, "eta_seconds": 273.23442511558534, "data_wait_seconds": 2.5570392608642578e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 71720, "total_steps": 72479, "tokens": 37601935360, "loss": 2.918490171432495, "l_mix": 1.8100414425134659, "l_a": 1.8629667609930038, "l_c": 1.8318620920181274, "z_loss": 59.40826606750488, "lr": 2.755813603412546e-05, "wd": 0.01, "grad_norm": 0.08442625403404236, "tokens_per_second": 1458983.3982528741, "eta_seconds": 272.74785475730897, "data_wait_seconds": 2.9593706130981445e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 71730, "total_steps": 72479, "tokens": 37607178240, "loss": 4.29207319021225, "l_mix": 2.660474717617035, "l_a": 2.7429204285144806, "l_c": 2.695740669965744, "z_loss": 49.69747972488403, "lr": 2.6839569572690395e-05, "wd": 0.01, "grad_norm": 0.07972381263971329, "tokens_per_second": 1359999.633596843, "eta_seconds": 288.7439836740494, "data_wait_seconds": 1.9729137420654297e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 71740, "total_steps": 72479, "tokens": 37612421120, "loss": 4.224313318729401, "l_mix": 2.6186132431030273, "l_a": 2.7009796500205994, "l_c": 2.6513534784317017, "z_loss": 49.45491027832031, "lr": 2.6130384824393514e-05, "wd": 0.01, "grad_norm": 0.07982677221298218, "tokens_per_second": 1446288.0384378387, "eta_seconds": 267.89188716411593, "data_wait_seconds": 1.9848346710205078e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 71750, "total_steps": 72479, "tokens": 37617664000, "loss": 4.3473920822143555, "l_mix": 2.6955885887145996, "l_a": 2.776196211576462, "l_c": 2.729815125465393, "z_loss": 48.66058874130249, "lr": 2.5430587713026363e-05, "wd": 0.01, "grad_norm": 0.0881381407380104, "tokens_per_second": 1431673.754463191, "eta_seconds": 266.9644189596176, "data_wait_seconds": 1.6123056411743164e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 71760, "total_steps": 72479, "tokens": 37622906880, "loss": 4.338455975055695, "l_mix": 2.68967804312706, "l_a": 2.7715259790420532, "l_c": 2.7244004905223846, "z_loss": 48.872106075286865, "lr": 2.47401840839653e-05, "wd": 0.01, "grad_norm": 0.08315310627222061, "tokens_per_second": 1417545.9229754044, "eta_seconds": 265.92653253078464, "data_wait_seconds": 1.5407800674438477e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 71770, "total_steps": 72479, "tokens": 37628149760, "loss": 4.361053049564362, "l_mix": 2.7041451930999756, "l_a": 2.7843303084373474, "l_c": 2.738695740699768, "z_loss": 49.048279762268066, "lr": 2.4059179704123735e-05, "wd": 0.01, "grad_norm": 0.0859343633055687, "tokens_per_second": 1468985.7929747282, "eta_seconds": 253.04546427726746, "data_wait_seconds": 1.665949821472168e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 71780, "total_steps": 72479, "tokens": 37633392640, "loss": 4.301763117313385, "l_mix": 2.666867047548294, "l_a": 2.7481815218925476, "l_c": 2.701471656560898, "z_loss": 49.38870716094971, "lr": 2.338758026190324e-05, "wd": 0.01, "grad_norm": 0.08105400204658508, "tokens_per_second": 1380852.0951317283, "eta_seconds": 265.3993959903717, "data_wait_seconds": 2.5957822799682617e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 71790, "total_steps": 72479, "tokens": 37638635520, "loss": 4.334510803222656, "l_mix": 2.6869404017925262, "l_a": 2.770858973264694, "l_c": 2.7210423350334167, "z_loss": 49.29957294464111, "lr": 2.2725391367146563e-05, "wd": 0.01, "grad_norm": 0.08617238700389862, "tokens_per_second": 1402863.9017283495, "eta_seconds": 257.4978453397751, "data_wait_seconds": 2.3812055587768555e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 71800, "total_steps": 72479, "tokens": 37643878400, "loss": 4.380563497543335, "l_mix": 2.716295003890991, "l_a": 2.7975763380527496, "l_c": 2.7499851286411285, "z_loss": 48.7106614112854, "lr": 2.207261855109051e-05, "wd": 0.01, "grad_norm": 0.07987909764051437, "tokens_per_second": 1291765.9255325673, "eta_seconds": 275.58518533706666, "data_wait_seconds": 3.567337989807129e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.17009121, "gate_a2b/L1_absmax": 0.67106986, "gate_a2b/L1_rms": 0.17913747, "gate_a2b/L3_absmean": 0.15902179, "gate_a2b/L3_absmax": 0.50869977, "gate_a2b/L3_rms": 0.1644491, "gate_a2b/L5_absmean": 0.22732644, "gate_a2b/L5_absmax": 0.73371899, "gate_a2b/L5_rms": 0.23315558, "gate_c2b/L1_absmean": 0.05062209, "gate_c2b/L1_absmax": 0.58915102, "gate_c2b/L1_rms": 0.0730434, "gate_c2b/L3_absmean": 0.16227719, "gate_c2b/L3_absmax": 0.38054386, "gate_c2b/L3_rms": 0.16567078, "gate_c2b/L5_absmean": 0.46939132, "gate_c2b/L5_absmax": 0.68716323, "gate_c2b/L5_rms": 0.47804168, "gate_b2a/L7_absmean": 0.05319274, "gate_b2a/L7_absmax": 0.33901548, "gate_b2a/L7_rms": 0.05882515, "gate_b2a/L11_absmean": 0.08722288, "gate_b2a/L11_absmax": 0.60992467, "gate_b2a/L11_rms": 0.09579754, "gate_b2a/L14_absmean": 0.22506556, "gate_b2a/L14_absmax": 0.78740817, "gate_b2a/L14_rms": 0.23324321, "gate_b2c/L3_absmean": 0.04127592, "gate_b2c/L3_absmax": 0.55743134, "gate_b2c/L3_rms": 0.05066744, "gate_b2c/L5_absmean": 0.04128858, "gate_b2c/L5_absmax": 0.54016727, "gate_b2c/L5_rms": 0.04988799, "gate_b2c/L6_absmean": 0.05941851, "gate_b2c/L6_absmax": 0.32686126, "gate_b2c/L6_rms": 0.06451691}, "streams": {"cos_a2b/round0": 0.354246, "cos_c2b/round0": 0.235637, "cos_b2a/round0": 0.174984, "cos_b2c/round0": 0.154198, "rms_a/round0": 6.643123, "rms_b/round0": 11.765752, "rms_c/round0": 4.78107, "cos_a2b/round1": 0.408117, "cos_c2b/round1": 0.303498, "cos_b2a/round1": 0.259604, "cos_b2c/round1": 0.102547, "rms_a/round1": 13.431385, "rms_b/round1": 28.263651, "rms_c/round1": 10.558609, "cos_a2b/round2": 0.54073, "cos_c2b/round2": 0.516855, "cos_b2a/round2": 0.099263, "cos_b2c/round2": 0.177438, "rms_a/round2": 41.455257, "rms_b/round2": 80.870811, "rms_c/round2": 21.501484}, "alpha_mean": 0.42831901559761576, "alpha_hist": [0.01416, 0.09082, 0.18346, 0.18393, 0.16884, 0.15294, 0.12253, 0.07056, 0.01242, 0.00035], "alpha_by_type": {"special": {"share": 0.00143, "mean": 0.53072, "std": 0.13504}, "whitespace": {"share": 0.02698, "mean": 0.51995, "std": 0.14086}, "digit": {"share": 0.04041, "mean": 0.45563, "std": 0.12547}, "word_start": {"share": 0.56503, "mean": 0.4447, "std": 0.17457}, "word_cont": {"share": 0.25558, "mean": 0.30978, "std": 0.14051}, "punct": {"share": 0.11057, "mean": 0.58496, "std": 0.1631}}, "alpha_p05": 0.15817, "alpha_p50": 0.43254, "alpha_p95": 0.74872}
{"step": 71810, "total_steps": 72479, "tokens": 37649121280, "loss": 2.920002818107605, "l_mix": 1.7962346374988556, "l_a": 1.906795173883438, "l_c": 1.8390984535217285, "z_loss": 55.16800355911255, "lr": 2.1429267266319608e-05, "wd": 0.01, "grad_norm": 0.08063796162605286, "tokens_per_second": 1385796.3710996136, "eta_seconds": 253.10260534286502, "data_wait_seconds": 2.512335777282715e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 71820, "total_steps": 72479, "tokens": 37654364160, "loss": 2.953195035457611, "l_mix": 1.8165041953325272, "l_a": 1.9289172738790512, "l_c": 1.8600518256425858, "z_loss": 54.66713857650757, "lr": 2.0795342886721146e-05, "wd": 0.01, "grad_norm": 0.07916960120201111, "tokens_per_second": 1392950.8661902728, "eta_seconds": 248.03875024318694, "data_wait_seconds": 4.32431697845459e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 71830, "total_steps": 72479, "tokens": 37659607040, "loss": 2.780485153198242, "l_mix": 1.7219675332307816, "l_a": 1.7834778875112534, "l_c": 1.7449138909578323, "z_loss": 61.73889112472534, "lr": 2.017085070743911e-05, "wd": 0.01, "grad_norm": 0.10523445904254913, "tokens_per_second": 1456505.2044189067, "eta_seconds": 233.61599462032316, "data_wait_seconds": 2.7894973754882812e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 71840, "total_steps": 72479, "tokens": 37664849920, "loss": 4.302686870098114, "l_mix": 2.6673401594161987, "l_a": 2.7499495446681976, "l_c": 2.701205849647522, "z_loss": 49.196608543395996, "lr": 1.9555795944831256e-05, "wd": 0.01, "grad_norm": 0.07992789149284363, "tokens_per_second": 1465605.2319434045, "eta_seconds": 228.58817961215973, "data_wait_seconds": 1.901388168334961e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 71850, "total_steps": 72479, "tokens": 37670092800, "loss": 4.218896269798279, "l_mix": 2.6153308749198914, "l_a": 2.696637451648712, "l_c": 2.648580402135849, "z_loss": 48.74171590805054, "lr": 1.8950183736424956e-05, "wd": 0.01, "grad_norm": 0.08144832402467728, "tokens_per_second": 1461608.9476054958, "eta_seconds": 225.62611739635466, "data_wait_seconds": 1.7404556274414062e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 71860, "total_steps": 72479, "tokens": 37675335680, "loss": 4.315034627914429, "l_mix": 2.6753188371658325, "l_a": 2.755915015935898, "l_c": 2.709803879261017, "z_loss": 49.43629217147827, "lr": 1.8354019140874423e-05, "wd": 0.01, "grad_norm": 0.082386314868927, "tokens_per_second": 1167055.489399995, "eta_seconds": 278.07955572605135, "data_wait_seconds": 0.1188267171382904, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 71870, "total_steps": 72479, "tokens": 37680578560, "loss": 4.236792027950287, "l_mix": 2.626130908727646, "l_a": 2.7102872729301453, "l_c": 2.658582925796509, "z_loss": 48.97984170913696, "lr": 1.7767307137917666e-05, "wd": 0.01, "grad_norm": 0.08461745828390121, "tokens_per_second": 1464058.447971981, "eta_seconds": 218.08650634288787, "data_wait_seconds": 1.6182661056518555e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 71880, "total_steps": 72479, "tokens": 37685821440, "loss": 4.362606167793274, "l_mix": 2.7047819197177887, "l_a": 2.787264406681061, "l_c": 2.738816201686859, "z_loss": 48.843101024627686, "lr": 1.7190052628336293e-05, "wd": 0.01, "grad_norm": 0.08177444338798523, "tokens_per_second": 1253280.35428419, "eta_seconds": 250.58121347427368, "data_wait_seconds": 2.7179718017578125e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 71890, "total_steps": 72479, "tokens": 37691064320, "loss": 4.62174379825592, "l_mix": 2.8681637346744537, "l_a": 2.942834049463272, "l_c": 2.902432858943939, "z_loss": 47.183377742767334, "lr": 1.6622260433913633e-05, "wd": 0.01, "grad_norm": 0.08324892818927765, "tokens_per_second": 1367649.5015421188, "eta_seconds": 225.79296205043795, "data_wait_seconds": 1.9550323486328125e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 71900, "total_steps": 72479, "tokens": 37696307200, "loss": 4.57957649230957, "l_mix": 2.841731995344162, "l_a": 2.9164072275161743, "l_c": 2.8764069974422455, "z_loss": 47.85709571838379, "lr": 1.6063935297394626e-05, "wd": 0.01, "grad_norm": 0.078285813331604, "tokens_per_second": 1450110.2115311888, "eta_seconds": 209.33771073818207, "data_wait_seconds": 1.7464160919189453e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.17007922, "gate_a2b/L1_absmax": 0.67094719, "gate_a2b/L1_rms": 0.17912564, "gate_a2b/L3_absmean": 0.15902254, "gate_a2b/L3_absmax": 0.50858241, "gate_a2b/L3_rms": 0.16445078, "gate_a2b/L5_absmean": 0.22733787, "gate_a2b/L5_absmax": 0.73371953, "gate_a2b/L5_rms": 0.23316842, "gate_c2b/L1_absmean": 0.05062549, "gate_c2b/L1_absmax": 0.58897072, "gate_c2b/L1_rms": 0.07303667, "gate_c2b/L3_absmean": 0.16226903, "gate_c2b/L3_absmax": 0.38048828, "gate_c2b/L3_rms": 0.16566136, "gate_c2b/L5_absmean": 0.46938738, "gate_c2b/L5_absmax": 0.68723094, "gate_c2b/L5_rms": 0.4780395, "gate_b2a/L7_absmean": 0.05319339, "gate_b2a/L7_absmax": 0.33889419, "gate_b2a/L7_rms": 0.05882595, "gate_b2a/L11_absmean": 0.08722422, "gate_b2a/L11_absmax": 0.60995132, "gate_b2a/L11_rms": 0.09580049, "gate_b2a/L14_absmean": 0.22505553, "gate_b2a/L14_absmax": 0.78730518, "gate_b2a/L14_rms": 0.23323186, "gate_b2c/L3_absmean": 0.04128394, "gate_b2c/L3_absmax": 0.55736971, "gate_b2c/L3_rms": 0.05067083, "gate_b2c/L5_absmean": 0.04130195, "gate_b2c/L5_absmax": 0.54023659, "gate_b2c/L5_rms": 0.04989813, "gate_b2c/L6_absmean": 0.05941605, "gate_b2c/L6_absmax": 0.32687923, "gate_b2c/L6_rms": 0.06451447}, "streams": {"cos_a2b/round0": 0.351919, "cos_c2b/round0": 0.236504, "cos_b2a/round0": 0.180323, "cos_b2c/round0": 0.155431, "rms_a/round0": 6.546043, "rms_b/round0": 11.687, "rms_c/round0": 4.67274, "cos_a2b/round1": 0.410033, "cos_c2b/round1": 0.311602, "cos_b2a/round1": 0.26411, "cos_b2c/round1": 0.102899, "rms_a/round1": 13.253283, "rms_b/round1": 27.651548, "rms_c/round1": 10.514017, "cos_a2b/round2": 0.547427, "cos_c2b/round2": 0.517225, "cos_b2a/round2": 0.102888, "cos_b2c/round2": 0.182494, "rms_a/round2": 41.039001, "rms_b/round2": 79.818459, "rms_c/round2": 21.327553}, "alpha_mean": 0.44984308159092734, "alpha_hist": [0.00824, 0.06259, 0.16798, 0.17503, 0.17427, 0.19006, 0.13884, 0.07155, 0.01093, 0.0005], "alpha_by_type": {"special": {"share": 0.00085, "mean": 0.54223, "std": 0.14658}, "whitespace": {"share": 0.06651, "mean": 0.54045, "std": 0.11921}, "digit": {"share": 0.03514, "mean": 0.43906, "std": 0.1544}, "word_start": {"share": 0.54953, "mean": 0.47045, "std": 0.16237}, "word_cont": {"share": 0.24512, "mean": 0.32156, "std": 0.1372}, "punct": {"share": 0.10284, "mean": 0.58977, "std": 0.15878}}, "alpha_p05": 0.18185, "alpha_p50": 0.45968, "alpha_p95": 0.75563}
{"step": 71910, "total_steps": 72479, "tokens": 37701550080, "loss": 4.596550762653351, "l_mix": 2.8526435792446136, "l_a": 2.923627018928528, "l_c": 2.889396458864212, "z_loss": 48.147730350494385, "lr": 1.5515081882446397e-05, "wd": 0.01, "grad_norm": 0.08473030477762222, "tokens_per_second": 1461401.9551557037, "eta_seconds": 204.13266243934632, "data_wait_seconds": 1.677870750427246e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 71920, "total_steps": 72479, "tokens": 37706792960, "loss": 4.615048587322235, "l_mix": 2.863853871822357, "l_a": 2.937875896692276, "l_c": 2.8994397819042206, "z_loss": 47.60725116729736, "lr": 1.4975704773618908e-05, "wd": 0.01, "grad_norm": 0.07797883450984955, "tokens_per_second": 1440191.2425064512, "eta_seconds": 203.49866278171538, "data_wait_seconds": 1.8149614334106445e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 71930, "total_steps": 72479, "tokens": 37712035840, "loss": 2.9260614812374115, "l_mix": 1.8000558912754059, "l_a": 1.9108416140079498, "l_c": 1.8425100445747375, "z_loss": 54.80214166641235, "lr": 1.4445808476307187e-05, "wd": 0.01, "grad_norm": 0.0788554698228836, "tokens_per_second": 1464013.3190861512, "eta_seconds": 196.60621132850645, "data_wait_seconds": 2.2679567337036133e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 71940, "total_steps": 72479, "tokens": 37717278720, "loss": 2.48904250562191, "l_mix": 1.541401445865631, "l_a": 1.5965130031108856, "l_c": 1.562290534377098, "z_loss": 62.457380294799805, "lr": 1.3925397416713265e-05, "wd": 0.01, "grad_norm": 0.09039564430713654, "tokens_per_second": 1388375.2459869971, "eta_seconds": 203.54096114635468, "data_wait_seconds": 4.875659942626953e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 71950, "total_steps": 72479, "tokens": 37722521600, "loss": 4.3137654066085815, "l_mix": 2.674255847930908, "l_a": 2.7564855813980103, "l_c": 2.7085461616516113, "z_loss": 49.424498558044434, "lr": 1.3414475941809539e-05, "wd": 0.01, "grad_norm": 0.08153165876865387, "tokens_per_second": 1365371.9357702057, "eta_seconds": 203.13025684356688, "data_wait_seconds": 2.0176172256469727e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 71960, "total_steps": 72479, "tokens": 37727764480, "loss": 4.284646272659302, "l_mix": 2.6566096246242523, "l_a": 2.735902726650238, "l_c": 2.690886080265045, "z_loss": 49.2564492225647, "lr": 1.2913048319302134e-05, "wd": 0.01, "grad_norm": 0.10865508019924164, "tokens_per_second": 1443517.0395912034, "eta_seconds": 188.50173883438111, "data_wait_seconds": 1.5795230865478516e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 71970, "total_steps": 72479, "tokens": 37733007360, "loss": 4.320954084396362, "l_mix": 2.6781452298164368, "l_a": 2.762470781803131, "l_c": 2.7135583460330963, "z_loss": 49.23298215866089, "lr": 1.2421118737595526e-05, "wd": 0.01, "grad_norm": 0.0798019990324974, "tokens_per_second": 1377769.8931528504, "eta_seconds": 193.69169940948487, "data_wait_seconds": 1.704692840576172e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 71980, "total_steps": 72479, "tokens": 37738250240, "loss": 4.197260290384293, "l_mix": 2.6027213037014008, "l_a": 2.681707352399826, "l_c": 2.63342222571373, "z_loss": 49.439067363739014, "lr": 1.1938691305757315e-05, "wd": 0.01, "grad_norm": 0.1006898507475853, "tokens_per_second": 1471055.6345163132, "eta_seconds": 177.84487946033477, "data_wait_seconds": 2.3305416107177734e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 71990, "total_steps": 72479, "tokens": 37743493120, "loss": 4.101804226636887, "l_mix": 2.542576640844345, "l_a": 2.621705025434494, "l_c": 2.575719952583313, "z_loss": 48.62443208694458, "lr": 1.1465770053483997e-05, "wd": 0.01, "grad_norm": 0.08287464827299118, "tokens_per_second": 1450588.016615291, "eta_seconds": 176.73993515968323, "data_wait_seconds": 2.2113323211669922e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 72000, "total_steps": 72479, "tokens": 37748736000, "loss": 4.628140330314636, "l_mix": 2.8725678622722626, "l_a": 2.945225179195404, "l_c": 2.90668261051178, "z_loss": 47.39275312423706, "lr": 1.1002358931067779e-05, "wd": 0.01, "grad_norm": 0.0811363160610199, "tokens_per_second": 1315849.6849725212, "eta_seconds": 190.85306997299196, "data_wait_seconds": 2.199411392211914e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.17007719, "gate_a2b/L1_absmax": 0.67094195, "gate_a2b/L1_rms": 0.17912473, "gate_a2b/L3_absmean": 0.15903592, "gate_a2b/L3_absmax": 0.50849915, "gate_a2b/L3_rms": 0.16446154, "gate_a2b/L5_absmean": 0.22735277, "gate_a2b/L5_absmax": 0.73370135, "gate_a2b/L5_rms": 0.23318037, "gate_c2b/L1_absmean": 0.05063371, "gate_c2b/L1_absmax": 0.5889262, "gate_c2b/L1_rms": 0.07303376, "gate_c2b/L3_absmean": 0.16226308, "gate_c2b/L3_absmax": 0.3805106, "gate_c2b/L3_rms": 0.16565518, "gate_c2b/L5_absmean": 0.46939769, "gate_c2b/L5_absmax": 0.68720645, "gate_c2b/L5_rms": 0.47804895, "gate_b2a/L7_absmean": 0.05319251, "gate_b2a/L7_absmax": 0.33889717, "gate_b2a/L7_rms": 0.05882154, "gate_b2a/L11_absmean": 0.08722611, "gate_b2a/L11_absmax": 0.6098671, "gate_b2a/L11_rms": 0.09579812, "gate_b2a/L14_absmean": 0.22506572, "gate_b2a/L14_absmax": 0.78727657, "gate_b2a/L14_rms": 0.23324062, "gate_b2c/L3_absmean": 0.04127903, "gate_b2c/L3_absmax": 0.55744565, "gate_b2c/L3_rms": 0.05066834, "gate_b2c/L5_absmean": 0.04130766, "gate_b2c/L5_absmax": 0.54027426, "gate_b2c/L5_rms": 0.04990452, "gate_b2c/L6_absmean": 0.0594293, "gate_b2c/L6_absmax": 0.32684433, "gate_b2c/L6_rms": 0.06452523}, "streams": {"cos_a2b/round0": 0.352234, "cos_c2b/round0": 0.234725, "cos_b2a/round0": 0.181306, "cos_b2c/round0": 0.154289, "rms_a/round0": 6.558758, "rms_b/round0": 11.643709, "rms_c/round0": 4.675039, "cos_a2b/round1": 0.408761, "cos_c2b/round1": 0.310189, "cos_b2a/round1": 0.261692, "cos_b2c/round1": 0.104363, "rms_a/round1": 13.262851, "rms_b/round1": 27.672079, "rms_c/round1": 10.449416, "cos_a2b/round2": 0.545809, "cos_c2b/round2": 0.518127, "cos_b2a/round2": 0.098963, "cos_b2c/round2": 0.182, "rms_a/round2": 41.084396, "rms_b/round2": 79.924622, "rms_c/round2": 21.132788}, "alpha_mean": 0.443337890758551, "alpha_hist": [0.00972, 0.06107, 0.16209, 0.19991, 0.18343, 0.17772, 0.12823, 0.06689, 0.01027, 0.00067], "alpha_by_type": {"special": {"share": 0.00072, "mean": 0.53936, "std": 0.1526}, "whitespace": {"share": 0.07542, "mean": 0.49767, "std": 0.14995}, "digit": {"share": 0.03003, "mean": 0.46169, "std": 0.15029}, "word_start": {"share": 0.54411, "mean": 0.4649, "std": 0.16311}, "word_cont": {"share": 0.24406, "mean": 0.32634, "std": 0.13165}, "punct": {"share": 0.10565, "mean": 0.55792, "std": 0.16981}}, "alpha_p05": 0.16668, "alpha_p50": 0.44686, "alpha_p95": 0.74017}
{"step": 72010, "total_steps": 72479, "tokens": 37753978880, "loss": 4.577291429042816, "l_mix": 2.8409779965877533, "l_a": 2.9126471877098083, "l_c": 2.8750638365745544, "z_loss": 47.93100452423096, "lr": 1.054846180936248e-05, "wd": 0.01, "grad_norm": 0.08871760964393616, "tokens_per_second": 1431317.3184257473, "eta_seconds": 171.793542098999, "data_wait_seconds": 2.294778823852539e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 72020, "total_steps": 72479, "tokens": 37759221760, "loss": 4.653106689453125, "l_mix": 2.887777328491211, "l_a": 2.9615215957164764, "l_c": 2.922909438610077, "z_loss": 47.15479516983032, "lr": 1.0104082479752513e-05, "wd": 0.01, "grad_norm": 0.08241782337427139, "tokens_per_second": 1435216.1478918798, "eta_seconds": 167.67383251190185, "data_wait_seconds": 3.0219554901123047e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 72030, "total_steps": 72479, "tokens": 37764464640, "loss": 4.5207008719444275, "l_mix": 2.805205911397934, "l_a": 2.8789739310741425, "l_c": 2.8393424451351166, "z_loss": 48.07035827636719, "lr": 9.66922465411994e-06, "wd": 0.01, "grad_norm": 0.08749751001596451, "tokens_per_second": 1463512.5070442858, "eta_seconds": 160.84953894615174, "data_wait_seconds": 2.0384788513183594e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 72040, "total_steps": 72479, "tokens": 37769707520, "loss": 4.618064045906067, "l_mix": 2.8659798204898834, "l_a": 2.9396325051784515, "l_c": 2.9006479680538177, "z_loss": 47.21053647994995, "lr": 9.243891964814471e-06, "wd": 0.01, "grad_norm": 0.08065856248140335, "tokens_per_second": 1435188.1408323024, "eta_seconds": 160.37091267108917, "data_wait_seconds": 2.1278858184814453e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 72050, "total_steps": 72479, "tokens": 37774950400, "loss": 2.7562792003154755, "l_mix": 1.7065859884023666, "l_a": 1.7683930695056915, "l_c": 1.7305840849876404, "z_loss": 61.88525915145874, "lr": 8.828087964622822e-06, "wd": 0.01, "grad_norm": 0.1076849028468132, "tokens_per_second": 1216952.537112786, "eta_seconds": 184.821959066391, "data_wait_seconds": 1.862645149230957e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 72060, "total_steps": 72479, "tokens": 37780193280, "loss": 4.323094964027405, "l_mix": 2.680488020181656, "l_a": 2.7624614238739014, "l_c": 2.712894856929779, "z_loss": 49.32145977020264, "lr": 8.421816126738707e-06, "wd": 0.01, "grad_norm": 0.08909773081541061, "tokens_per_second": 1467999.3580337802, "eta_seconds": 149.6435749769211, "data_wait_seconds": 1.755356788635254e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 72070, "total_steps": 72479, "tokens": 37785436160, "loss": 4.302163302898407, "l_mix": 2.6670033633708954, "l_a": 2.750224471092224, "l_c": 2.700308233499527, "z_loss": 48.82358455657959, "lr": 8.02507984473399e-06, "wd": 0.01, "grad_norm": 0.08438190817832947, "tokens_per_second": 1446742.4825956877, "eta_seconds": 148.21835577487946, "data_wait_seconds": 1.9669532775878906e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 72080, "total_steps": 72479, "tokens": 37790679040, "loss": 4.397653222084045, "l_mix": 2.7270604968070984, "l_a": 2.8070135414600372, "l_c": 2.7616287171840668, "z_loss": 48.81200456619263, "lr": 7.637882432530847e-06, "wd": 0.01, "grad_norm": 0.08516698330640793, "tokens_per_second": 1324291.1853376802, "eta_seconds": 157.96443736553192, "data_wait_seconds": 1.7464160919189453e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 72090, "total_steps": 72479, "tokens": 37795921920, "loss": 4.265958845615387, "l_mix": 2.6449811160564423, "l_a": 2.724126398563385, "l_c": 2.679132640361786, "z_loss": 48.27294063568115, "lr": 7.26022712437342e-06, "wd": 0.01, "grad_norm": 0.08130434155464172, "tokens_per_second": 1456431.7941142498, "eta_seconds": 140.03266944885254, "data_wait_seconds": 1.7136335372924805e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 72100, "total_steps": 72479, "tokens": 37801164800, "loss": 4.309018909931183, "l_mix": 2.671183556318283, "l_a": 2.7548590898513794, "l_c": 2.704591780900955, "z_loss": 48.96620225906372, "lr": 6.892117074801013e-06, "wd": 0.01, "grad_norm": 0.08722154796123505, "tokens_per_second": 1347635.4338157438, "eta_seconds": 147.44725985527037, "data_wait_seconds": 3.68952751159668e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.17007652, "gate_a2b/L1_absmax": 0.67101294, "gate_a2b/L1_rms": 0.17912494, "gate_a2b/L3_absmean": 0.15903006, "gate_a2b/L3_absmax": 0.50860775, "gate_a2b/L3_rms": 0.16445664, "gate_a2b/L5_absmean": 0.22735511, "gate_a2b/L5_absmax": 0.73375428, "gate_a2b/L5_rms": 0.23318426, "gate_c2b/L1_absmean": 0.05064014, "gate_c2b/L1_absmax": 0.58896494, "gate_c2b/L1_rms": 0.07303666, "gate_c2b/L3_absmean": 0.16227268, "gate_c2b/L3_absmax": 0.38050011, "gate_c2b/L3_rms": 0.16566506, "gate_c2b/L5_absmean": 0.46939549, "gate_c2b/L5_absmax": 0.68724918, "gate_c2b/L5_rms": 0.47804624, "gate_b2a/L7_absmean": 0.05319653, "gate_b2a/L7_absmax": 0.33888885, "gate_b2a/L7_rms": 0.05882328, "gate_b2a/L11_absmean": 0.08722611, "gate_b2a/L11_absmax": 0.60983557, "gate_b2a/L11_rms": 0.09579893, "gate_b2a/L14_absmean": 0.22506766, "gate_b2a/L14_absmax": 0.78721654, "gate_b2a/L14_rms": 0.23324196, "gate_b2c/L3_absmean": 0.04127629, "gate_b2c/L3_absmax": 0.55742365, "gate_b2c/L3_rms": 0.05066508, "gate_b2c/L5_absmean": 0.04130206, "gate_b2c/L5_absmax": 0.54023588, "gate_b2c/L5_rms": 0.04990045, "gate_b2c/L6_absmean": 0.05943229, "gate_b2c/L6_absmax": 0.32686603, "gate_b2c/L6_rms": 0.06452885}, "streams": {"cos_a2b/round0": 0.3528, "cos_c2b/round0": 0.230951, "cos_b2a/round0": 0.178387, "cos_b2c/round0": 0.15283, "rms_a/round0": 6.623608, "rms_b/round0": 11.838855, "rms_c/round0": 4.759485, "cos_a2b/round1": 0.410557, "cos_c2b/round1": 0.312681, "cos_b2a/round1": 0.266265, "cos_b2c/round1": 0.105137, "rms_a/round1": 13.454261, "rms_b/round1": 28.276182, "rms_c/round1": 10.666698, "cos_a2b/round2": 0.538961, "cos_c2b/round2": 0.521641, "cos_b2a/round2": 0.096048, "cos_b2c/round2": 0.180158, "rms_a/round2": 41.572411, "rms_b/round2": 80.851128, "rms_c/round2": 21.730257}, "alpha_mean": 0.4248284307860324, "alpha_hist": [0.01855, 0.09639, 0.18196, 0.18346, 0.16293, 0.15094, 0.12067, 0.07407, 0.01082, 0.00021], "alpha_by_type": {"special": {"share": 0.0016, "mean": 0.55759, "std": 0.13139}, "whitespace": {"share": 0.02702, "mean": 0.48689, "std": 0.1338}, "digit": {"share": 0.03044, "mean": 0.45034, "std": 0.11625}, "word_start": {"share": 0.56836, "mean": 0.44701, "std": 0.17735}, "word_cont": {"share": 0.26794, "mean": 0.30644, "std": 0.14317}, "punct": {"share": 0.10463, "mean": 0.58202, "std": 0.17049}}, "alpha_p05": 0.14415, "alpha_p50": 0.43818, "alpha_p95": 0.75636}
{"step": 72110, "total_steps": 72479, "tokens": 37806407680, "loss": 2.851253092288971, "l_mix": 1.7659525126218796, "l_a": 1.8267458230257034, "l_c": 1.790922686457634, "z_loss": 60.41675567626953, "lr": 6.533555358621657e-06, "wd": 0.01, "grad_norm": 0.08422595262527466, "tokens_per_second": 1434603.4270623755, "eta_seconds": 134.85418224334717, "data_wait_seconds": 3.5256147384643555e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 72120, "total_steps": 72479, "tokens": 37811650560, "loss": 4.454952001571655, "l_mix": 2.764044374227524, "l_a": 2.837764471769333, "l_c": 2.7985939383506775, "z_loss": 48.126267433166504, "lr": 6.184544970886957e-06, "wd": 0.01, "grad_norm": 0.0804794654250145, "tokens_per_second": 1482552.2872607158, "eta_seconds": 126.95632634162904, "data_wait_seconds": 2.6047229766845703e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 72130, "total_steps": 72479, "tokens": 37816893440, "loss": 4.612261056900024, "l_mix": 2.8622653484344482, "l_a": 2.9357486367225647, "l_c": 2.8975702822208405, "z_loss": 48.11381673812866, "lr": 5.845088826866179e-06, "wd": 0.01, "grad_norm": 0.08337021619081497, "tokens_per_second": 1426155.4107413855, "eta_seconds": 128.30054187774658, "data_wait_seconds": 7.766485214233398e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 72140, "total_steps": 72479, "tokens": 37822136320, "loss": 4.554067075252533, "l_mix": 2.8265575170516968, "l_a": 2.8965786397457123, "l_c": 2.861786127090454, "z_loss": 47.927133083343506, "lr": 5.515189762022754e-06, "wd": 0.01, "grad_norm": 0.07830634713172913, "tokens_per_second": 1386941.8788265851, "eta_seconds": 128.1478587627411, "data_wait_seconds": 2.682209014892578e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 72150, "total_steps": 72479, "tokens": 37827379200, "loss": 4.461649417877197, "l_mix": 2.768401950597763, "l_a": 2.8404659032821655, "l_c": 2.80369234085083, "z_loss": 48.32426834106445, "lr": 5.194850531990019e-06, "wd": 0.01, "grad_norm": 0.0819898396730423, "tokens_per_second": 1413337.3127143804, "eta_seconds": 122.04499976634979, "data_wait_seconds": 4.2498111724853516e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 72160, "total_steps": 72479, "tokens": 37832622080, "loss": 4.634262561798096, "l_mix": 2.8764489591121674, "l_a": 2.9478252232074738, "l_c": 2.9115531742572784, "z_loss": 47.343987464904785, "lr": 4.884073812548495e-06, "wd": 0.01, "grad_norm": 0.0798279196023941, "tokens_per_second": 1327594.7247847964, "eta_seconds": 125.9781082868576, "data_wait_seconds": 1.9729137420654297e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 72170, "total_steps": 72479, "tokens": 37837864960, "loss": 4.262461721897125, "l_mix": 2.6419832706451416, "l_a": 2.726268321275711, "l_c": 2.675325959920883, "z_loss": 48.95393514633179, "lr": 4.582862199603411e-06, "wd": 0.01, "grad_norm": 0.0804518461227417, "tokens_per_second": 1434936.313744365, "eta_seconds": 112.90047540664673, "data_wait_seconds": 1.6927719116210938e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 72180, "total_steps": 72479, "tokens": 37843107840, "loss": 4.259429752826691, "l_mix": 2.6397754848003387, "l_a": 2.7250225841999054, "l_c": 2.6738246977329254, "z_loss": 49.12584638595581, "lr": 4.2912182091630005e-06, "wd": 0.01, "grad_norm": 0.08357458561658859, "tokens_per_second": 1375355.1199739168, "eta_seconds": 113.97937138080596, "data_wait_seconds": 1.519918441772461e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 72190, "total_steps": 72479, "tokens": 37848350720, "loss": 4.300867736339569, "l_mix": 2.666190952062607, "l_a": 2.7501335740089417, "l_c": 2.6987887620925903, "z_loss": 48.83728218078613, "lr": 4.009144277317694e-06, "wd": 0.01, "grad_norm": 0.08620670437812805, "tokens_per_second": 1394031.6144211458, "eta_seconds": 108.69138865470886, "data_wait_seconds": 2.053380012512207e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 72200, "total_steps": 72479, "tokens": 37853593600, "loss": 4.149422228336334, "l_mix": 2.571437567472458, "l_a": 2.6561965942382812, "l_c": 2.6037518084049225, "z_loss": 49.33070516586304, "lr": 3.7366427602193053e-06, "wd": 0.01, "grad_norm": 0.08248044550418854, "tokens_per_second": 1353065.6430190334, "eta_seconds": 108.1073580980301, "data_wait_seconds": 1.704692840576172e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.17008385, "gate_a2b/L1_absmax": 0.67104805, "gate_a2b/L1_rms": 0.17913255, "gate_a2b/L3_absmean": 0.15903419, "gate_a2b/L3_absmax": 0.50856572, "gate_a2b/L3_rms": 0.16445984, "gate_a2b/L5_absmean": 0.22735623, "gate_a2b/L5_absmax": 0.73372781, "gate_a2b/L5_rms": 0.23318495, "gate_c2b/L1_absmean": 0.05064328, "gate_c2b/L1_absmax": 0.58896506, "gate_c2b/L1_rms": 0.07304065, "gate_c2b/L3_absmean": 0.1622735, "gate_c2b/L3_absmax": 0.38047552, "gate_c2b/L3_rms": 0.16566639, "gate_c2b/L5_absmean": 0.46939594, "gate_c2b/L5_absmax": 0.68724287, "gate_c2b/L5_rms": 0.47804633, "gate_b2a/L7_absmean": 0.05319357, "gate_b2a/L7_absmax": 0.33890307, "gate_b2a/L7_rms": 0.05882112, "gate_b2a/L11_absmean": 0.08722556, "gate_b2a/L11_absmax": 0.60989624, "gate_b2a/L11_rms": 0.09579922, "gate_b2a/L14_absmean": 0.22507039, "gate_b2a/L14_absmax": 0.78724986, "gate_b2a/L14_rms": 0.23324384, "gate_b2c/L3_absmean": 0.04128241, "gate_b2c/L3_absmax": 0.55741769, "gate_b2c/L3_rms": 0.05066948, "gate_b2c/L5_absmean": 0.0413053, "gate_b2c/L5_absmax": 0.54023278, "gate_b2c/L5_rms": 0.04990274, "gate_b2c/L6_absmean": 0.05943461, "gate_b2c/L6_absmax": 0.32678899, "gate_b2c/L6_rms": 0.06453124}, "streams": {"cos_a2b/round0": 0.353984, "cos_c2b/round0": 0.231054, "cos_b2a/round0": 0.171162, "cos_b2c/round0": 0.149341, "rms_a/round0": 6.630185, "rms_b/round0": 11.747671, "rms_c/round0": 4.796555, "cos_a2b/round1": 0.402467, "cos_c2b/round1": 0.301098, "cos_b2a/round1": 0.25687, "cos_b2c/round1": 0.100461, "rms_a/round1": 13.477633, "rms_b/round1": 28.493889, "rms_c/round1": 10.590524, "cos_a2b/round2": 0.532259, "cos_c2b/round2": 0.521891, "cos_b2a/round2": 0.089943, "cos_b2c/round2": 0.174212, "rms_a/round2": 41.777531, "rms_b/round2": 81.946983, "rms_c/round2": 21.431017}, "alpha_mean": 0.4137737706260154, "alpha_hist": [0.02054, 0.10757, 0.19243, 0.18823, 0.15594, 0.14224, 0.11732, 0.06471, 0.01082, 0.00018], "alpha_by_type": {"special": {"share": 0.00122, "mean": 0.56223, "std": 0.11643}, "whitespace": {"share": 0.02502, "mean": 0.52279, "std": 0.12433}, "digit": {"share": 0.04924, "mean": 0.50194, "std": 0.14231}, "word_start": {"share": 0.53065, "mean": 0.42958, "std": 0.17447}, "word_cont": {"share": 0.27675, "mean": 0.29021, "std": 0.13472}, "punct": {"share": 0.11711, "mean": 0.57224, "std": 0.17268}}, "alpha_p05": 0.16238, "alpha_p50": 0.39443, "alpha_p95": 0.74651}
{"step": 72210, "total_steps": 72479, "tokens": 37858836480, "loss": 4.278428077697754, "l_mix": 2.652749687433243, "l_a": 2.733914852142334, "l_c": 2.6850129067897797, "z_loss": 48.472914695739746, "lr": 3.473715934062005e-06, "wd": 0.01, "grad_norm": 0.07866761833429337, "tokens_per_second": 1404752.7383777583, "eta_seconds": 100.39736399650575, "data_wait_seconds": 2.09808349609375e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 72220, "total_steps": 72479, "tokens": 37864079360, "loss": 2.996928483247757, "l_mix": 1.8564529567956924, "l_a": 1.9200370609760284, "l_c": 1.8815479129552841, "z_loss": 61.45008039474487, "lr": 3.220365995062918e-06, "wd": 0.01, "grad_norm": 0.08892109245061874, "tokens_per_second": 1357957.9884427201, "eta_seconds": 99.99616568088533, "data_wait_seconds": 1.9878149032592773e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 72230, "total_steps": 72479, "tokens": 37869322240, "loss": 2.768055737018585, "l_mix": 1.714390128850937, "l_a": 1.774614930152893, "l_c": 1.7376037389039993, "z_loss": 62.03691530227661, "lr": 2.976595059443482e-06, "wd": 0.01, "grad_norm": 0.08267146348953247, "tokens_per_second": 1363556.1662017873, "eta_seconds": 95.7406194448471, "data_wait_seconds": 2.4199485778808594e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 72240, "total_steps": 72479, "tokens": 37874565120, "loss": 4.586973607540131, "l_mix": 2.8467246294021606, "l_a": 2.9181504249572754, "l_c": 2.8826792538166046, "z_loss": 47.9857497215271, "lr": 2.7424051634125923e-06, "wd": 0.01, "grad_norm": 0.08193231374025345, "tokens_per_second": 1475718.25283249, "eta_seconds": 84.9110809326172, "data_wait_seconds": 1.7195940017700195e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 72250, "total_steps": 72479, "tokens": 37879808000, "loss": 4.444422423839569, "l_mix": 2.7575202584266663, "l_a": 2.830433338880539, "l_c": 2.7925733029842377, "z_loss": 48.47180938720703, "lr": 2.5177982631486018e-06, "wd": 0.01, "grad_norm": 0.07896940410137177, "tokens_per_second": 1434471.0080767234, "eta_seconds": 83.69771945476532, "data_wait_seconds": 2.8848648071289062e-05, "mix": {"fineweb_hq": 0.50956, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 72260, "total_steps": 72479, "tokens": 37885050880, "loss": 4.222708344459534, "l_mix": 2.6169160306453705, "l_a": 2.702106297016144, "l_c": 2.650534510612488, "z_loss": 49.212515354156494, "lr": 2.302776234784126e-06, "wd": 0.01, "grad_norm": 0.0807039812207222, "tokens_per_second": 1390858.3406166567, "eta_seconds": 82.55267171859741, "data_wait_seconds": 2.3633241653442383e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 72270, "total_steps": 72479, "tokens": 37890293760, "loss": 4.395234644412994, "l_mix": 2.7254600524902344, "l_a": 2.806948632001877, "l_c": 2.758966416120529, "z_loss": 48.67886972427368, "lr": 2.0973408743889364e-06, "wd": 0.01, "grad_norm": 0.09038624912500381, "tokens_per_second": 1439096.5315012217, "eta_seconds": 76.14235014915465, "data_wait_seconds": 1.7315149307250977e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 72280, "total_steps": 72479, "tokens": 37895536640, "loss": 4.241971552371979, "l_mix": 2.6294517517089844, "l_a": 2.712232083082199, "l_c": 2.6628336310386658, "z_loss": 49.234140396118164, "lr": 1.901493897956552e-06, "wd": 0.01, "grad_norm": 0.08184666931629181, "tokens_per_second": 1437426.9033347494, "eta_seconds": 72.58338615894317, "data_wait_seconds": 1.6242265701293945e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08233}}
{"step": 72290, "total_steps": 72479, "tokens": 37900779520, "loss": 4.100060939788818, "l_mix": 2.541270464658737, "l_a": 2.623328596353531, "l_c": 2.572639286518097, "z_loss": 49.98397874832153, "lr": 1.7152369413885372e-06, "wd": 0.01, "grad_norm": 0.08737258613109589, "tokens_per_second": 1469902.710415785, "eta_seconds": 67.41291875839234, "data_wait_seconds": 1.58846378326416e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 72300, "total_steps": 72479, "tokens": 37906022400, "loss": 4.215596556663513, "l_mix": 2.612769991159439, "l_a": 2.698274552822113, "l_c": 2.6444802284240723, "z_loss": 49.16059875488281, "lr": 1.5385715604818607e-06, "wd": 0.01, "grad_norm": 0.08266512304544449, "tokens_per_second": 1271743.7577080217, "eta_seconds": 73.79438776969909, "data_wait_seconds": 2.2530555725097656e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}, "gates": {"gate_a2b/L1_absmean": 0.17008953, "gate_a2b/L1_absmax": 0.67106032, "gate_a2b/L1_rms": 0.17913778, "gate_a2b/L3_absmean": 0.15903264, "gate_a2b/L3_absmax": 0.50855249, "gate_a2b/L3_rms": 0.16445796, "gate_a2b/L5_absmean": 0.22735797, "gate_a2b/L5_absmax": 0.73373026, "gate_a2b/L5_rms": 0.23318672, "gate_c2b/L1_absmean": 0.05064614, "gate_c2b/L1_absmax": 0.5889439, "gate_c2b/L1_rms": 0.07304306, "gate_c2b/L3_absmean": 0.1622747, "gate_c2b/L3_absmax": 0.38047001, "gate_c2b/L3_rms": 0.16566776, "gate_c2b/L5_absmean": 0.46939421, "gate_c2b/L5_absmax": 0.68724787, "gate_c2b/L5_rms": 0.47804439, "gate_b2a/L7_absmean": 0.05319336, "gate_b2a/L7_absmax": 0.33889082, "gate_b2a/L7_rms": 0.05882196, "gate_b2a/L11_absmean": 0.08722386, "gate_b2a/L11_absmax": 0.60987192, "gate_b2a/L11_rms": 0.09579718, "gate_b2a/L14_absmean": 0.2250689, "gate_b2a/L14_absmax": 0.78728527, "gate_b2a/L14_rms": 0.23324218, "gate_b2c/L3_absmean": 0.04128392, "gate_b2c/L3_absmax": 0.55741125, "gate_b2c/L3_rms": 0.05066996, "gate_b2c/L5_absmean": 0.04130564, "gate_b2c/L5_absmax": 0.54023772, "gate_b2c/L5_rms": 0.04990377, "gate_b2c/L6_absmean": 0.05943305, "gate_b2c/L6_absmax": 0.32680577, "gate_b2c/L6_rms": 0.06452996}, "streams": {"cos_a2b/round0": 0.353668, "cos_c2b/round0": 0.233155, "cos_b2a/round0": 0.169002, "cos_b2c/round0": 0.151068, "rms_a/round0": 6.626837, "rms_b/round0": 11.665818, "rms_c/round0": 4.791005, "cos_a2b/round1": 0.402886, "cos_c2b/round1": 0.301777, "cos_b2a/round1": 0.254493, "cos_b2c/round1": 0.097691, "rms_a/round1": 13.355957, "rms_b/round1": 28.359488, "rms_c/round1": 10.441337, "cos_a2b/round2": 0.524447, "cos_c2b/round2": 0.532621, "cos_b2a/round2": 0.089921, "cos_b2c/round2": 0.174357, "rms_a/round2": 41.060863, "rms_b/round2": 81.877396, "rms_c/round2": 21.034523}, "alpha_mean": 0.4059936123261565, "alpha_hist": [0.02202, 0.11067, 0.19884, 0.19173, 0.16339, 0.13393, 0.11163, 0.05928, 0.00841, 0.00011], "alpha_by_type": {"special": {"share": 0.00067, "mean": 0.54932, "std": 0.10355}, "whitespace": {"share": 0.02879, "mean": 0.54061, "std": 0.13648}, "digit": {"share": 0.04771, "mean": 0.45413, "std": 0.1465}, "word_start": {"share": 0.5416, "mean": 0.41651, "std": 0.17166}, "word_cont": {"share": 0.26984, "mean": 0.28918, "std": 0.13209}, "punct": {"share": 0.11139, "mean": 0.58156, "std": 0.1632}}, "alpha_p05": 0.14295, "alpha_p50": 0.37433, "alpha_p95": 0.70761}
{"step": 72310, "total_steps": 72479, "tokens": 37911265280, "loss": 4.278784990310669, "l_mix": 2.6525163650512695, "l_a": 2.7355513274669647, "l_c": 2.6853436827659607, "z_loss": 48.670947551727295, "lr": 1.37149923091498e-06, "wd": 0.01, "grad_norm": 0.08314541727304459, "tokens_per_second": 1288137.7623590797, "eta_seconds": 68.78509006500244, "data_wait_seconds": 1.868605613708496e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 72320, "total_steps": 72479, "tokens": 37916508160, "loss": 4.333905279636383, "l_mix": 2.6871252059936523, "l_a": 2.768672078847885, "l_c": 2.7205947637557983, "z_loss": 48.21352434158325, "lr": 1.2140213482363494e-06, "wd": 0.01, "grad_norm": 0.08007950335741043, "tokens_per_second": 1450723.619458438, "eta_seconds": 57.46221463680267, "data_wait_seconds": 1.9729137420654297e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 72330, "total_steps": 72479, "tokens": 37921751040, "loss": 4.230780482292175, "l_mix": 2.6221018731594086, "l_a": 2.707573652267456, "l_c": 2.6546880900859833, "z_loss": 49.01503086090088, "lr": 1.066139227852292e-06, "wd": 0.01, "grad_norm": 0.08747392892837524, "tokens_per_second": 1304142.3163920178, "eta_seconds": 59.90060365200043, "data_wait_seconds": 2.726912498474121e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 72340, "total_steps": 72479, "tokens": 37926993920, "loss": 2.832971155643463, "l_mix": 1.754695937037468, "l_a": 1.8155170232057571, "l_c": 1.7787334620952606, "z_loss": 62.39138650894165, "lr": 9.278541050161459e-07, "wd": 0.01, "grad_norm": 0.08452031016349792, "tokens_per_second": 1446018.894020956, "eta_seconds": 50.39770386219024, "data_wait_seconds": 2.7447938919067383e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 72350, "total_steps": 72479, "tokens": 37932236800, "loss": 4.139106452465057, "l_mix": 2.5642469227313995, "l_a": 2.650690257549286, "l_c": 2.598841279745102, "z_loss": 51.36046743392944, "lr": 7.991671348177964e-07, "wd": 0.01, "grad_norm": 0.09046439081430435, "tokens_per_second": 1451230.943342933, "eta_seconds": 46.6039897441864, "data_wait_seconds": 2.4527311325073242e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 72360, "total_steps": 72479, "tokens": 37937479680, "loss": 4.253830313682556, "l_mix": 2.636281907558441, "l_a": 2.7226971089839935, "l_c": 2.669131338596344, "z_loss": 48.87475824356079, "lr": 6.800793921742265e-07, "wd": 0.01, "grad_norm": 0.08137907087802887, "tokens_per_second": 1392227.8896270622, "eta_seconds": 44.81326115131378, "data_wait_seconds": 1.7881393432617188e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 72370, "total_steps": 72479, "tokens": 37942722560, "loss": 4.4018107652664185, "l_mix": 2.7294807136058807, "l_a": 2.8122622072696686, "l_c": 2.7621711790561676, "z_loss": 47.80255126953125, "lr": 5.705918718205805e-07, "wd": 0.01, "grad_norm": 0.08294223248958588, "tokens_per_second": 1334234.010977493, "eta_seconds": 42.83161089420319, "data_wait_seconds": 1.5527009963989258e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 72380, "total_steps": 72479, "tokens": 37947965440, "loss": 4.259543836116791, "l_mix": 2.640018582344055, "l_a": 2.72592431306839, "l_c": 2.6724930107593536, "z_loss": 48.97358322143555, "lr": 4.7070548830148163e-07, "wd": 0.01, "grad_norm": 0.08081761747598648, "tokens_per_second": 1469057.039395698, "eta_seconds": 35.331856155395506, "data_wait_seconds": 1.5467405319213867e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 72390, "total_steps": 72479, "tokens": 37953208320, "loss": 4.286946058273315, "l_mix": 2.6573528945446014, "l_a": 2.7409097850322723, "l_c": 2.691067188978195, "z_loss": 48.977386474609375, "lr": 3.804210759640103e-07, "wd": 0.01, "grad_norm": 0.08400021493434906, "tokens_per_second": 1386763.189294579, "eta_seconds": 33.647873234748836, "data_wait_seconds": 1.5944242477416992e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20765, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 72400, "total_steps": 72479, "tokens": 37958451200, "loss": 4.224549412727356, "l_mix": 2.618972659111023, "l_a": 2.6994504630565643, "l_c": 2.652471959590912, "z_loss": 49.471155643463135, "lr": 2.9973938895004323e-07, "wd": 0.01, "grad_norm": 0.09326978772878647, "tokens_per_second": 1424973.880972847, "eta_seconds": 29.06632363796234, "data_wait_seconds": 1.58846378326416e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20767, "cosmopedia_v2": 0.20043, "finemath": 0.08233}, "gates": {"gate_a2b/L1_absmean": 0.17009178, "gate_a2b/L1_absmax": 0.67105943, "gate_a2b/L1_rms": 0.17913978, "gate_a2b/L3_absmean": 0.15903316, "gate_a2b/L3_absmax": 0.50855827, "gate_a2b/L3_rms": 0.16445842, "gate_a2b/L5_absmean": 0.2273573, "gate_a2b/L5_absmax": 0.73373681, "gate_a2b/L5_rms": 0.23318584, "gate_c2b/L1_absmean": 0.05064714, "gate_c2b/L1_absmax": 0.58893824, "gate_c2b/L1_rms": 0.07304346, "gate_c2b/L3_absmean": 0.16227451, "gate_c2b/L3_absmax": 0.3804723, "gate_c2b/L3_rms": 0.16566761, "gate_c2b/L5_absmean": 0.46939522, "gate_c2b/L5_absmax": 0.68725038, "gate_c2b/L5_rms": 0.47804534, "gate_b2a/L7_absmean": 0.05319542, "gate_b2a/L7_absmax": 0.33888084, "gate_b2a/L7_rms": 0.05882395, "gate_b2a/L11_absmean": 0.08722316, "gate_b2a/L11_absmax": 0.60986727, "gate_b2a/L11_rms": 0.09579669, "gate_b2a/L14_absmean": 0.22506995, "gate_b2a/L14_absmax": 0.78728223, "gate_b2a/L14_rms": 0.23324299, "gate_b2c/L3_absmean": 0.04128422, "gate_b2c/L3_absmax": 0.55740887, "gate_b2c/L3_rms": 0.05066999, "gate_b2c/L5_absmean": 0.04130577, "gate_b2c/L5_absmax": 0.54024172, "gate_b2c/L5_rms": 0.04990394, "gate_b2c/L6_absmean": 0.05943257, "gate_b2c/L6_absmax": 0.32679144, "gate_b2c/L6_rms": 0.06452928}, "streams": {"cos_a2b/round0": 0.354856, "cos_c2b/round0": 0.239131, "cos_b2a/round0": 0.186132, "cos_b2c/round0": 0.157849, "rms_a/round0": 6.570509, "rms_b/round0": 11.646004, "rms_c/round0": 4.746445, "cos_a2b/round1": 0.415672, "cos_c2b/round1": 0.313629, "cos_b2a/round1": 0.268535, "cos_b2c/round1": 0.103961, "rms_a/round1": 13.317759, "rms_b/round1": 27.987432, "rms_c/round1": 10.426969, "cos_a2b/round2": 0.544998, "cos_c2b/round2": 0.52457, "cos_b2a/round2": 0.09684, "cos_b2c/round2": 0.181162, "rms_a/round2": 41.327362, "rms_b/round2": 80.751862, "rms_c/round2": 21.238026}, "alpha_mean": 0.427360797915334, "alpha_hist": [0.013, 0.0905, 0.1866, 0.18414, 0.1665, 0.15831, 0.12007, 0.07065, 0.01012, 0.00011], "alpha_by_type": {"special": {"share": 0.00105, "mean": 0.54876, "std": 0.09454}, "whitespace": {"share": 0.02332, "mean": 0.52859, "std": 0.11789}, "digit": {"share": 0.02342, "mean": 0.45826, "std": 0.14118}, "word_start": {"share": 0.60965, "mean": 0.44418, "std": 0.17048}, "word_cont": {"share": 0.23463, "mean": 0.30074, "std": 0.13321}, "punct": {"share": 0.10793, "mean": 0.57789, "std": 0.17126}}, "alpha_p05": 0.15974, "alpha_p50": 0.42203, "alpha_p95": 0.72835}
{"step": 72410, "total_steps": 72479, "tokens": 37963694080, "loss": 4.427783578634262, "l_mix": 2.747756242752075, "l_a": 2.8202958703041077, "l_c": 2.779795318841934, "z_loss": 48.6593713760376, "lr": 2.286611011903805e-07, "wd": 0.01, "grad_norm": 0.08977329730987549, "tokens_per_second": 1422050.8119778999, "eta_seconds": 25.439226007461546, "data_wait_seconds": 1.806020736694336e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 72420, "total_steps": 72479, "tokens": 37968936960, "loss": 4.580456376075745, "l_mix": 2.842564195394516, "l_a": 2.914573460817337, "l_c": 2.878400534391403, "z_loss": 47.82839107513428, "lr": 1.6718680639925542e-07, "wd": 0.01, "grad_norm": 0.08285044133663177, "tokens_per_second": 1475215.933741434, "eta_seconds": 20.96845030784607, "data_wait_seconds": 2.5212764739990234e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 72430, "total_steps": 72479, "tokens": 37974179840, "loss": 4.520753800868988, "l_mix": 2.805604636669159, "l_a": 2.8772143721580505, "l_c": 2.839949518442154, "z_loss": 47.71629095077515, "lr": 1.1531701806871686e-07, "wd": 0.01, "grad_norm": 0.08238998055458069, "tokens_per_second": 1406668.7133524776, "eta_seconds": 18.263086223602294, "data_wait_seconds": 2.6911497116088867e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 72440, "total_steps": 72479, "tokens": 37979422720, "loss": 4.442746222019196, "l_mix": 2.756330668926239, "l_a": 2.831119865179062, "l_c": 2.790265053510666, "z_loss": 49.01694583892822, "lr": 7.305216946492665e-08, "wd": 0.01, "grad_norm": 0.08834974467754364, "tokens_per_second": 1341627.8218283406, "eta_seconds": 15.240614175796507, "data_wait_seconds": 2.3603439331054688e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 72450, "total_steps": 72479, "tokens": 37984665600, "loss": 2.7089558839797974, "l_mix": 1.6768474727869034, "l_a": 1.7393324971199036, "l_c": 1.7010287195444107, "z_loss": 62.35650157928467, "lr": 4.039261362471236e-08, "wd": 0.01, "grad_norm": 0.08943790197372437, "tokens_per_second": 1410656.8161344077, "eta_seconds": 10.7782075881958, "data_wait_seconds": 3.5315752029418945e-05, "mix": {"fineweb_hq": 0.50957, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08235}}
{"step": 72460, "total_steps": 72479, "tokens": 37989908480, "loss": 4.348206698894501, "l_mix": 2.695660501718521, "l_a": 2.780023545026779, "l_c": 2.728463500738144, "z_loss": 48.96877479553223, "lr": 1.7338623351864646e-08, "wd": 0.01, "grad_norm": 0.08446713536977768, "tokens_per_second": 1418525.8128341103, "eta_seconds": 7.022411513328553, "data_wait_seconds": 1.6868114471435547e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}
{"step": 72470, "total_steps": 72479, "tokens": 37995151360, "loss": 4.130343556404114, "l_mix": 2.560019224882126, "l_a": 2.6426404118537903, "l_c": 2.5917737185955048, "z_loss": 49.224584102630615, "lr": 3.890391215477517e-09, "wd": 0.01, "grad_norm": 0.08168847858905792, "tokens_per_second": 1436824.4982533194, "eta_seconds": 3.284042000770569, "data_wait_seconds": 2.187490463256836e-05, "mix": {"fineweb_hq": 0.50958, "dclm": 0.20766, "cosmopedia_v2": 0.20043, "finemath": 0.08234}}